2 * Generic OPP Interface
4 * Copyright (C) 2009-2010 Texas Instruments Incorporated.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
14 #include <linux/cpu.h>
15 #include <linux/kernel.h>
16 #include <linux/errno.h>
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/device.h>
20 #include <linux/list.h>
21 #include <linux/rculist.h>
22 #include <linux/rcupdate.h>
23 #include <linux/pm_opp.h>
25 #include <linux/export.h>
28 * Internal data structure organization with the OPP layer library is as
31 * |- device 1 (represents voltage domain 1)
32 * | |- opp 1 (availability, freq, voltage)
36 * |- device 2 (represents the next voltage domain)
38 * `- device m (represents mth voltage domain)
39 * device 1, 2.. are represented by dev_opp structure while each opp
40 * is represented by the opp structure.
44 * struct dev_pm_opp - Generic OPP description structure
45 * @node: opp list node. The nodes are maintained throughout the lifetime
46 * of boot. It is expected only an optimal set of OPPs are
47 * added to the library by the SoC framework.
48 * RCU usage: opp list is traversed with RCU locks. node
49 * modification is possible realtime, hence the modifications
50 * are protected by the dev_opp_list_lock for integrity.
51 * IMPORTANT: the opp nodes should be maintained in increasing
53 * @dynamic: not-created from static DT entries.
54 * @available: true/false - marks if this OPP as available or not
55 * @turbo: true if turbo (boost) OPP
56 * @rate: Frequency in hertz
57 * @u_volt: Target voltage in microvolts corresponding to this OPP
58 * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP
59 * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP
60 * @u_amp: Maximum current drawn by the device in microamperes
61 * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's
62 * frequency from any other OPP's frequency.
63 * @dev_opp: points back to the device_opp struct this opp belongs to
64 * @rcu_head: RCU callback head used for deferred freeing
65 * @np: OPP's device node.
67 * This structure stores the OPP information for a given device.
70 struct list_head node
;
78 unsigned long u_volt_min
;
79 unsigned long u_volt_max
;
81 unsigned long clock_latency_ns
;
83 struct device_opp
*dev_opp
;
84 struct rcu_head rcu_head
;
86 struct device_node
*np
;
90 * struct device_list_opp - devices managed by 'struct device_opp'
92 * @dev: device to which the struct object belongs
93 * @rcu_head: RCU callback head used for deferred freeing
95 * This is an internal data structure maintaining the list of devices that are
96 * managed by 'struct device_opp'.
98 struct device_list_opp
{
99 struct list_head node
;
100 const struct device
*dev
;
101 struct rcu_head rcu_head
;
105 * struct device_opp - Device opp structure
106 * @node: list node - contains the devices with OPPs that
107 * have been registered. Nodes once added are not modified in this
109 * RCU usage: nodes are not modified in the list of device_opp,
110 * however addition is possible and is secured by dev_opp_list_lock
111 * @srcu_head: notifier head to notify the OPP availability changes.
112 * @rcu_head: RCU callback head used for deferred freeing
113 * @dev_list: list of devices that share these OPPs
114 * @opp_list: list of opps
115 * @np: struct device_node pointer for opp's DT node.
116 * @shared_opp: OPP is shared between multiple devices.
118 * This is an internal data structure maintaining the link to opps attached to
119 * a device. This structure is not meant to be shared to users as it is
120 * meant for book keeping and private to OPP library.
122 * Because the opp structures can be used from both rcu and srcu readers, we
123 * need to wait for the grace period of both of them before freeing any
124 * resources. And so we have used kfree_rcu() from within call_srcu() handlers.
127 struct list_head node
;
129 struct srcu_notifier_head srcu_head
;
130 struct rcu_head rcu_head
;
131 struct list_head dev_list
;
132 struct list_head opp_list
;
134 struct device_node
*np
;
135 unsigned long clock_latency_ns_max
;
137 struct dev_pm_opp
*suspend_opp
;
141 * The root of the list of all devices. All device_opp structures branch off
142 * from here, with each device_opp containing the list of opp it supports in
143 * various states of availability.
145 static LIST_HEAD(dev_opp_list
);
146 /* Lock to allow exclusive modification to the device and opp lists */
147 static DEFINE_MUTEX(dev_opp_list_lock
);
149 #define opp_rcu_lockdep_assert() \
151 RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
152 !lockdep_is_held(&dev_opp_list_lock), \
153 "Missing rcu_read_lock() or " \
154 "dev_opp_list_lock protection"); \
157 static struct device_list_opp
*_find_list_dev(const struct device
*dev
,
158 struct device_opp
*dev_opp
)
160 struct device_list_opp
*list_dev
;
162 list_for_each_entry(list_dev
, &dev_opp
->dev_list
, node
)
163 if (list_dev
->dev
== dev
)
169 static struct device_opp
*_managed_opp(const struct device_node
*np
)
171 struct device_opp
*dev_opp
;
173 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
) {
174 if (dev_opp
->np
== np
) {
176 * Multiple devices can point to the same OPP table and
177 * so will have same node-pointer, np.
179 * But the OPPs will be considered as shared only if the
180 * OPP table contains a "opp-shared" property.
182 return dev_opp
->shared_opp
? dev_opp
: NULL
;
190 * _find_device_opp() - find device_opp struct using device pointer
191 * @dev: device pointer used to lookup device OPPs
193 * Search list of device OPPs for one containing matching device. Does a RCU
194 * reader operation to grab the pointer needed.
196 * Return: pointer to 'struct device_opp' if found, otherwise -ENODEV or
197 * -EINVAL based on type of error.
199 * Locking: This function must be called under rcu_read_lock(). device_opp
200 * is a RCU protected pointer. This means that device_opp is valid as long
201 * as we are under RCU lock.
203 static struct device_opp
*_find_device_opp(struct device
*dev
)
205 struct device_opp
*dev_opp
;
207 if (IS_ERR_OR_NULL(dev
)) {
208 pr_err("%s: Invalid parameters\n", __func__
);
209 return ERR_PTR(-EINVAL
);
212 list_for_each_entry_rcu(dev_opp
, &dev_opp_list
, node
)
213 if (_find_list_dev(dev
, dev_opp
))
216 return ERR_PTR(-ENODEV
);
220 * dev_pm_opp_get_voltage() - Gets the voltage corresponding to an available opp
221 * @opp: opp for which voltage has to be returned for
223 * Return: voltage in micro volt corresponding to the opp, else
226 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
227 * protected pointer. This means that opp which could have been fetched by
228 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
229 * under RCU lock. The pointer returned by the opp_find_freq family must be
230 * used in the same section as the usage of this function with the pointer
231 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
234 unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp
*opp
)
236 struct dev_pm_opp
*tmp_opp
;
239 opp_rcu_lockdep_assert();
241 tmp_opp
= rcu_dereference(opp
);
242 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
243 pr_err("%s: Invalid parameters\n", __func__
);
249 EXPORT_SYMBOL_GPL(dev_pm_opp_get_voltage
);
252 * dev_pm_opp_get_freq() - Gets the frequency corresponding to an available opp
253 * @opp: opp for which frequency has to be returned for
255 * Return: frequency in hertz corresponding to the opp, else
258 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
259 * protected pointer. This means that opp which could have been fetched by
260 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
261 * under RCU lock. The pointer returned by the opp_find_freq family must be
262 * used in the same section as the usage of this function with the pointer
263 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
266 unsigned long dev_pm_opp_get_freq(struct dev_pm_opp
*opp
)
268 struct dev_pm_opp
*tmp_opp
;
271 opp_rcu_lockdep_assert();
273 tmp_opp
= rcu_dereference(opp
);
274 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
)
275 pr_err("%s: Invalid parameters\n", __func__
);
281 EXPORT_SYMBOL_GPL(dev_pm_opp_get_freq
);
284 * dev_pm_opp_is_turbo() - Returns if opp is turbo OPP or not
285 * @opp: opp for which turbo mode is being verified
287 * Turbo OPPs are not for normal use, and can be enabled (under certain
288 * conditions) for short duration of times to finish high throughput work
289 * quickly. Running on them for longer times may overheat the chip.
291 * Return: true if opp is turbo opp, else false.
293 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
294 * protected pointer. This means that opp which could have been fetched by
295 * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are
296 * under RCU lock. The pointer returned by the opp_find_freq family must be
297 * used in the same section as the usage of this function with the pointer
298 * prior to unlocking with rcu_read_unlock() to maintain the integrity of the
301 bool dev_pm_opp_is_turbo(struct dev_pm_opp
*opp
)
303 struct dev_pm_opp
*tmp_opp
;
305 opp_rcu_lockdep_assert();
307 tmp_opp
= rcu_dereference(opp
);
308 if (IS_ERR_OR_NULL(tmp_opp
) || !tmp_opp
->available
) {
309 pr_err("%s: Invalid parameters\n", __func__
);
313 return tmp_opp
->turbo
;
315 EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo
);
318 * dev_pm_opp_get_max_clock_latency() - Get max clock latency in nanoseconds
319 * @dev: device for which we do this operation
321 * Return: This function returns the max clock latency in nanoseconds.
323 * Locking: This function takes rcu_read_lock().
325 unsigned long dev_pm_opp_get_max_clock_latency(struct device
*dev
)
327 struct device_opp
*dev_opp
;
328 unsigned long clock_latency_ns
;
332 dev_opp
= _find_device_opp(dev
);
334 clock_latency_ns
= 0;
336 clock_latency_ns
= dev_opp
->clock_latency_ns_max
;
339 return clock_latency_ns
;
341 EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency
);
344 * dev_pm_opp_get_suspend_opp() - Get suspend opp
345 * @dev: device for which we do this operation
347 * Return: This function returns pointer to the suspend opp if it is
348 * defined and available, otherwise it returns NULL.
350 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
351 * protected pointer. The reason for the same is that the opp pointer which is
352 * returned will remain valid for use with opp_get_{voltage, freq} only while
353 * under the locked area. The pointer returned must be used prior to unlocking
354 * with rcu_read_unlock() to maintain the integrity of the pointer.
356 struct dev_pm_opp
*dev_pm_opp_get_suspend_opp(struct device
*dev
)
358 struct device_opp
*dev_opp
;
360 opp_rcu_lockdep_assert();
362 dev_opp
= _find_device_opp(dev
);
363 if (IS_ERR(dev_opp
) || !dev_opp
->suspend_opp
||
364 !dev_opp
->suspend_opp
->available
)
367 return dev_opp
->suspend_opp
;
369 EXPORT_SYMBOL_GPL(dev_pm_opp_get_suspend_opp
);
372 * dev_pm_opp_get_opp_count() - Get number of opps available in the opp list
373 * @dev: device for which we do this operation
375 * Return: This function returns the number of available opps if there are any,
376 * else returns 0 if none or the corresponding error value.
378 * Locking: This function takes rcu_read_lock().
380 int dev_pm_opp_get_opp_count(struct device
*dev
)
382 struct device_opp
*dev_opp
;
383 struct dev_pm_opp
*temp_opp
;
388 dev_opp
= _find_device_opp(dev
);
389 if (IS_ERR(dev_opp
)) {
390 count
= PTR_ERR(dev_opp
);
391 dev_err(dev
, "%s: device OPP not found (%d)\n",
396 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
397 if (temp_opp
->available
)
405 EXPORT_SYMBOL_GPL(dev_pm_opp_get_opp_count
);
408 * dev_pm_opp_find_freq_exact() - search for an exact frequency
409 * @dev: device for which we do this operation
410 * @freq: frequency to search for
411 * @available: true/false - match for available opp
413 * Return: Searches for exact match in the opp list and returns pointer to the
414 * matching opp if found, else returns ERR_PTR in case of error and should
415 * be handled using IS_ERR. Error return values can be:
416 * EINVAL: for bad pointer
417 * ERANGE: no match found for search
418 * ENODEV: if device not found in list of registered devices
420 * Note: available is a modifier for the search. if available=true, then the
421 * match is for exact matching frequency and is available in the stored OPP
422 * table. if false, the match is for exact frequency which is not available.
424 * This provides a mechanism to enable an opp which is not available currently
425 * or the opposite as well.
427 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
428 * protected pointer. The reason for the same is that the opp pointer which is
429 * returned will remain valid for use with opp_get_{voltage, freq} only while
430 * under the locked area. The pointer returned must be used prior to unlocking
431 * with rcu_read_unlock() to maintain the integrity of the pointer.
433 struct dev_pm_opp
*dev_pm_opp_find_freq_exact(struct device
*dev
,
437 struct device_opp
*dev_opp
;
438 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
440 opp_rcu_lockdep_assert();
442 dev_opp
= _find_device_opp(dev
);
443 if (IS_ERR(dev_opp
)) {
444 int r
= PTR_ERR(dev_opp
);
445 dev_err(dev
, "%s: device OPP not found (%d)\n", __func__
, r
);
449 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
450 if (temp_opp
->available
== available
&&
451 temp_opp
->rate
== freq
) {
459 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_exact
);
462 * dev_pm_opp_find_freq_ceil() - Search for an rounded ceil freq
463 * @dev: device for which we do this operation
464 * @freq: Start frequency
466 * Search for the matching ceil *available* OPP from a starting freq
469 * Return: matching *opp and refreshes *freq accordingly, else returns
470 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
472 * EINVAL: for bad pointer
473 * ERANGE: no match found for search
474 * ENODEV: if device not found in list of registered devices
476 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
477 * protected pointer. The reason for the same is that the opp pointer which is
478 * returned will remain valid for use with opp_get_{voltage, freq} only while
479 * under the locked area. The pointer returned must be used prior to unlocking
480 * with rcu_read_unlock() to maintain the integrity of the pointer.
482 struct dev_pm_opp
*dev_pm_opp_find_freq_ceil(struct device
*dev
,
485 struct device_opp
*dev_opp
;
486 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
488 opp_rcu_lockdep_assert();
491 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
492 return ERR_PTR(-EINVAL
);
495 dev_opp
= _find_device_opp(dev
);
497 return ERR_CAST(dev_opp
);
499 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
500 if (temp_opp
->available
&& temp_opp
->rate
>= *freq
) {
509 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_ceil
);
512 * dev_pm_opp_find_freq_floor() - Search for a rounded floor freq
513 * @dev: device for which we do this operation
514 * @freq: Start frequency
516 * Search for the matching floor *available* OPP from a starting freq
519 * Return: matching *opp and refreshes *freq accordingly, else returns
520 * ERR_PTR in case of error and should be handled using IS_ERR. Error return
522 * EINVAL: for bad pointer
523 * ERANGE: no match found for search
524 * ENODEV: if device not found in list of registered devices
526 * Locking: This function must be called under rcu_read_lock(). opp is a rcu
527 * protected pointer. The reason for the same is that the opp pointer which is
528 * returned will remain valid for use with opp_get_{voltage, freq} only while
529 * under the locked area. The pointer returned must be used prior to unlocking
530 * with rcu_read_unlock() to maintain the integrity of the pointer.
532 struct dev_pm_opp
*dev_pm_opp_find_freq_floor(struct device
*dev
,
535 struct device_opp
*dev_opp
;
536 struct dev_pm_opp
*temp_opp
, *opp
= ERR_PTR(-ERANGE
);
538 opp_rcu_lockdep_assert();
541 dev_err(dev
, "%s: Invalid argument freq=%p\n", __func__
, freq
);
542 return ERR_PTR(-EINVAL
);
545 dev_opp
= _find_device_opp(dev
);
547 return ERR_CAST(dev_opp
);
549 list_for_each_entry_rcu(temp_opp
, &dev_opp
->opp_list
, node
) {
550 if (temp_opp
->available
) {
551 /* go to the next node, before choosing prev */
552 if (temp_opp
->rate
> *freq
)
563 EXPORT_SYMBOL_GPL(dev_pm_opp_find_freq_floor
);
565 /* List-dev Helpers */
566 static void _kfree_list_dev_rcu(struct rcu_head
*head
)
568 struct device_list_opp
*list_dev
;
570 list_dev
= container_of(head
, struct device_list_opp
, rcu_head
);
571 kfree_rcu(list_dev
, rcu_head
);
574 static void _remove_list_dev(struct device_list_opp
*list_dev
,
575 struct device_opp
*dev_opp
)
577 list_del(&list_dev
->node
);
578 call_srcu(&dev_opp
->srcu_head
.srcu
, &list_dev
->rcu_head
,
579 _kfree_list_dev_rcu
);
582 static struct device_list_opp
*_add_list_dev(const struct device
*dev
,
583 struct device_opp
*dev_opp
)
585 struct device_list_opp
*list_dev
;
587 list_dev
= kzalloc(sizeof(*list_dev
), GFP_KERNEL
);
591 /* Initialize list-dev */
593 list_add_rcu(&list_dev
->node
, &dev_opp
->dev_list
);
599 * _add_device_opp() - Find device OPP table or allocate a new one
600 * @dev: device for which we do this operation
602 * It tries to find an existing table first, if it couldn't find one, it
603 * allocates a new OPP table and returns that.
605 * Return: valid device_opp pointer if success, else NULL.
607 static struct device_opp
*_add_device_opp(struct device
*dev
)
609 struct device_opp
*dev_opp
;
610 struct device_list_opp
*list_dev
;
612 /* Check for existing list for 'dev' first */
613 dev_opp
= _find_device_opp(dev
);
614 if (!IS_ERR(dev_opp
))
618 * Allocate a new device OPP table. In the infrequent case where a new
619 * device is needed to be added, we pay this penalty.
621 dev_opp
= kzalloc(sizeof(*dev_opp
), GFP_KERNEL
);
625 INIT_LIST_HEAD(&dev_opp
->dev_list
);
627 list_dev
= _add_list_dev(dev
, dev_opp
);
633 srcu_init_notifier_head(&dev_opp
->srcu_head
);
634 INIT_LIST_HEAD(&dev_opp
->opp_list
);
636 /* Secure the device list modification */
637 list_add_rcu(&dev_opp
->node
, &dev_opp_list
);
642 * _kfree_device_rcu() - Free device_opp RCU handler
645 static void _kfree_device_rcu(struct rcu_head
*head
)
647 struct device_opp
*device_opp
= container_of(head
, struct device_opp
, rcu_head
);
649 kfree_rcu(device_opp
, rcu_head
);
653 * _remove_device_opp() - Removes a device OPP table
654 * @dev_opp: device OPP table to be removed.
656 * Removes/frees device OPP table it it doesn't contain any OPPs.
658 static void _remove_device_opp(struct device_opp
*dev_opp
)
660 struct device_list_opp
*list_dev
;
662 if (!list_empty(&dev_opp
->opp_list
))
665 list_dev
= list_first_entry(&dev_opp
->dev_list
, struct device_list_opp
,
668 _remove_list_dev(list_dev
, dev_opp
);
670 /* dev_list must be empty now */
671 WARN_ON(!list_empty(&dev_opp
->dev_list
));
673 list_del_rcu(&dev_opp
->node
);
674 call_srcu(&dev_opp
->srcu_head
.srcu
, &dev_opp
->rcu_head
,
679 * _kfree_opp_rcu() - Free OPP RCU handler
682 static void _kfree_opp_rcu(struct rcu_head
*head
)
684 struct dev_pm_opp
*opp
= container_of(head
, struct dev_pm_opp
, rcu_head
);
686 kfree_rcu(opp
, rcu_head
);
690 * _opp_remove() - Remove an OPP from a table definition
691 * @dev_opp: points back to the device_opp struct this opp belongs to
692 * @opp: pointer to the OPP to remove
693 * @notify: OPP_EVENT_REMOVE notification should be sent or not
695 * This function removes an opp definition from the opp list.
697 * Locking: The internal device_opp and opp structures are RCU protected.
698 * It is assumed that the caller holds required mutex for an RCU updater
701 static void _opp_remove(struct device_opp
*dev_opp
,
702 struct dev_pm_opp
*opp
, bool notify
)
705 * Notify the changes in the availability of the operable
706 * frequency/voltage list.
709 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_REMOVE
, opp
);
710 list_del_rcu(&opp
->node
);
711 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
713 _remove_device_opp(dev_opp
);
717 * dev_pm_opp_remove() - Remove an OPP from OPP list
718 * @dev: device for which we do this operation
719 * @freq: OPP to remove with matching 'freq'
721 * This function removes an opp from the opp list.
723 * Locking: The internal device_opp and opp structures are RCU protected.
724 * Hence this function internally uses RCU updater strategy with mutex locks
725 * to keep the integrity of the internal data structures. Callers should ensure
726 * that this function is *NOT* called under RCU protection or in contexts where
727 * mutex cannot be locked.
729 void dev_pm_opp_remove(struct device
*dev
, unsigned long freq
)
731 struct dev_pm_opp
*opp
;
732 struct device_opp
*dev_opp
;
735 /* Hold our list modification lock here */
736 mutex_lock(&dev_opp_list_lock
);
738 dev_opp
= _find_device_opp(dev
);
742 list_for_each_entry(opp
, &dev_opp
->opp_list
, node
) {
743 if (opp
->rate
== freq
) {
750 dev_warn(dev
, "%s: Couldn't find OPP with freq: %lu\n",
755 _opp_remove(dev_opp
, opp
, true);
757 mutex_unlock(&dev_opp_list_lock
);
759 EXPORT_SYMBOL_GPL(dev_pm_opp_remove
);
761 static struct dev_pm_opp
*_allocate_opp(struct device
*dev
,
762 struct device_opp
**dev_opp
)
764 struct dev_pm_opp
*opp
;
766 /* allocate new OPP node */
767 opp
= kzalloc(sizeof(*opp
), GFP_KERNEL
);
771 INIT_LIST_HEAD(&opp
->node
);
773 *dev_opp
= _add_device_opp(dev
);
782 static int _opp_add(struct device
*dev
, struct dev_pm_opp
*new_opp
,
783 struct device_opp
*dev_opp
)
785 struct dev_pm_opp
*opp
;
786 struct list_head
*head
= &dev_opp
->opp_list
;
789 * Insert new OPP in order of increasing frequency and discard if
792 * Need to use &dev_opp->opp_list in the condition part of the 'for'
793 * loop, don't replace it with head otherwise it will become an infinite
796 list_for_each_entry_rcu(opp
, &dev_opp
->opp_list
, node
) {
797 if (new_opp
->rate
> opp
->rate
) {
802 if (new_opp
->rate
< opp
->rate
)
806 dev_warn(dev
, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n",
807 __func__
, opp
->rate
, opp
->u_volt
, opp
->available
,
808 new_opp
->rate
, new_opp
->u_volt
, new_opp
->available
);
810 return opp
->available
&& new_opp
->u_volt
== opp
->u_volt
?
814 new_opp
->dev_opp
= dev_opp
;
815 list_add_rcu(&new_opp
->node
, head
);
821 * _opp_add_dynamic() - Allocate a dynamic OPP.
822 * @dev: device for which we do this operation
823 * @freq: Frequency in Hz for this OPP
824 * @u_volt: Voltage in uVolts for this OPP
825 * @dynamic: Dynamically added OPPs.
827 * This function adds an opp definition to the opp list and returns status.
828 * The opp is made available by default and it can be controlled using
829 * dev_pm_opp_enable/disable functions and may be removed by dev_pm_opp_remove.
831 * NOTE: "dynamic" parameter impacts OPPs added by the of_init_opp_table and
832 * freed by of_free_opp_table.
834 * Locking: The internal device_opp and opp structures are RCU protected.
835 * Hence this function internally uses RCU updater strategy with mutex locks
836 * to keep the integrity of the internal data structures. Callers should ensure
837 * that this function is *NOT* called under RCU protection or in contexts where
838 * mutex cannot be locked.
842 * Duplicate OPPs (both freq and volt are same) and opp->available
843 * -EEXIST Freq are same and volt are different OR
844 * Duplicate OPPs (both freq and volt are same) and !opp->available
845 * -ENOMEM Memory allocation failure
847 static int _opp_add_dynamic(struct device
*dev
, unsigned long freq
,
848 long u_volt
, bool dynamic
)
850 struct device_opp
*dev_opp
;
851 struct dev_pm_opp
*new_opp
;
854 /* Hold our list modification lock here */
855 mutex_lock(&dev_opp_list_lock
);
857 new_opp
= _allocate_opp(dev
, &dev_opp
);
863 /* populate the opp table */
864 new_opp
->rate
= freq
;
865 new_opp
->u_volt
= u_volt
;
866 new_opp
->available
= true;
867 new_opp
->dynamic
= dynamic
;
869 ret
= _opp_add(dev
, new_opp
, dev_opp
);
873 mutex_unlock(&dev_opp_list_lock
);
876 * Notify the changes in the availability of the operable
877 * frequency/voltage list.
879 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
883 _opp_remove(dev_opp
, new_opp
, false);
885 mutex_unlock(&dev_opp_list_lock
);
889 /* TODO: Support multiple regulators */
890 static int opp_get_microvolt(struct dev_pm_opp
*opp
, struct device
*dev
)
892 u32 microvolt
[3] = {0};
895 /* Missing property isn't a problem, but an invalid entry is */
896 if (!of_find_property(opp
->np
, "opp-microvolt", NULL
))
899 count
= of_property_count_u32_elems(opp
->np
, "opp-microvolt");
901 dev_err(dev
, "%s: Invalid opp-microvolt property (%d)\n",
906 /* There can be one or three elements here */
907 if (count
!= 1 && count
!= 3) {
908 dev_err(dev
, "%s: Invalid number of elements in opp-microvolt property (%d)\n",
913 ret
= of_property_read_u32_array(opp
->np
, "opp-microvolt", microvolt
,
916 dev_err(dev
, "%s: error parsing opp-microvolt: %d\n", __func__
,
921 opp
->u_volt
= microvolt
[0];
922 opp
->u_volt_min
= microvolt
[1];
923 opp
->u_volt_max
= microvolt
[2];
929 * _opp_add_static_v2() - Allocate static OPPs (As per 'v2' DT bindings)
930 * @dev: device for which we do this operation
933 * This function adds an opp definition to the opp list and returns status. The
934 * opp can be controlled using dev_pm_opp_enable/disable functions and may be
935 * removed by dev_pm_opp_remove.
937 * Locking: The internal device_opp and opp structures are RCU protected.
938 * Hence this function internally uses RCU updater strategy with mutex locks
939 * to keep the integrity of the internal data structures. Callers should ensure
940 * that this function is *NOT* called under RCU protection or in contexts where
941 * mutex cannot be locked.
945 * Duplicate OPPs (both freq and volt are same) and opp->available
946 * -EEXIST Freq are same and volt are different OR
947 * Duplicate OPPs (both freq and volt are same) and !opp->available
948 * -ENOMEM Memory allocation failure
949 * -EINVAL Failed parsing the OPP node
951 static int _opp_add_static_v2(struct device
*dev
, struct device_node
*np
)
953 struct device_opp
*dev_opp
;
954 struct dev_pm_opp
*new_opp
;
959 /* Hold our list modification lock here */
960 mutex_lock(&dev_opp_list_lock
);
962 new_opp
= _allocate_opp(dev
, &dev_opp
);
968 ret
= of_property_read_u64(np
, "opp-hz", &rate
);
970 dev_err(dev
, "%s: opp-hz not found\n", __func__
);
975 * Rate is defined as an unsigned long in clk API, and so casting
976 * explicitly to its type. Must be fixed once rate is 64 bit
977 * guaranteed in clk API.
979 new_opp
->rate
= (unsigned long)rate
;
980 new_opp
->turbo
= of_property_read_bool(np
, "turbo-mode");
983 new_opp
->dynamic
= false;
984 new_opp
->available
= true;
986 if (!of_property_read_u32(np
, "clock-latency-ns", &val
))
987 new_opp
->clock_latency_ns
= val
;
989 ret
= opp_get_microvolt(new_opp
, dev
);
993 if (!of_property_read_u32(new_opp
->np
, "opp-microamp", &val
))
994 new_opp
->u_amp
= val
;
996 ret
= _opp_add(dev
, new_opp
, dev_opp
);
1000 /* OPP to select on device suspend */
1001 if (of_property_read_bool(np
, "opp-suspend")) {
1002 if (dev_opp
->suspend_opp
)
1003 dev_warn(dev
, "%s: Multiple suspend OPPs found (%lu %lu)\n",
1004 __func__
, dev_opp
->suspend_opp
->rate
,
1007 dev_opp
->suspend_opp
= new_opp
;
1010 if (new_opp
->clock_latency_ns
> dev_opp
->clock_latency_ns_max
)
1011 dev_opp
->clock_latency_ns_max
= new_opp
->clock_latency_ns
;
1013 mutex_unlock(&dev_opp_list_lock
);
1015 pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n",
1016 __func__
, new_opp
->turbo
, new_opp
->rate
, new_opp
->u_volt
,
1017 new_opp
->u_volt_min
, new_opp
->u_volt_max
,
1018 new_opp
->clock_latency_ns
);
1021 * Notify the changes in the availability of the operable
1022 * frequency/voltage list.
1024 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ADD
, new_opp
);
1028 _opp_remove(dev_opp
, new_opp
, false);
1030 mutex_unlock(&dev_opp_list_lock
);
1035 * dev_pm_opp_add() - Add an OPP table from a table definitions
1036 * @dev: device for which we do this operation
1037 * @freq: Frequency in Hz for this OPP
1038 * @u_volt: Voltage in uVolts for this OPP
1040 * This function adds an opp definition to the opp list and returns status.
1041 * The opp is made available by default and it can be controlled using
1042 * dev_pm_opp_enable/disable functions.
1044 * Locking: The internal device_opp and opp structures are RCU protected.
1045 * Hence this function internally uses RCU updater strategy with mutex locks
1046 * to keep the integrity of the internal data structures. Callers should ensure
1047 * that this function is *NOT* called under RCU protection or in contexts where
1048 * mutex cannot be locked.
1052 * Duplicate OPPs (both freq and volt are same) and opp->available
1053 * -EEXIST Freq are same and volt are different OR
1054 * Duplicate OPPs (both freq and volt are same) and !opp->available
1055 * -ENOMEM Memory allocation failure
1057 int dev_pm_opp_add(struct device
*dev
, unsigned long freq
, unsigned long u_volt
)
1059 return _opp_add_dynamic(dev
, freq
, u_volt
, true);
1061 EXPORT_SYMBOL_GPL(dev_pm_opp_add
);
1064 * _opp_set_availability() - helper to set the availability of an opp
1065 * @dev: device for which we do this operation
1066 * @freq: OPP frequency to modify availability
1067 * @availability_req: availability status requested for this opp
1069 * Set the availability of an OPP with an RCU operation, opp_{enable,disable}
1070 * share a common logic which is isolated here.
1072 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1073 * copy operation, returns 0 if no modification was done OR modification was
1076 * Locking: The internal device_opp and opp structures are RCU protected.
1077 * Hence this function internally uses RCU updater strategy with mutex locks to
1078 * keep the integrity of the internal data structures. Callers should ensure
1079 * that this function is *NOT* called under RCU protection or in contexts where
1080 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1082 static int _opp_set_availability(struct device
*dev
, unsigned long freq
,
1083 bool availability_req
)
1085 struct device_opp
*dev_opp
;
1086 struct dev_pm_opp
*new_opp
, *tmp_opp
, *opp
= ERR_PTR(-ENODEV
);
1089 /* keep the node allocated */
1090 new_opp
= kmalloc(sizeof(*new_opp
), GFP_KERNEL
);
1094 mutex_lock(&dev_opp_list_lock
);
1096 /* Find the device_opp */
1097 dev_opp
= _find_device_opp(dev
);
1098 if (IS_ERR(dev_opp
)) {
1099 r
= PTR_ERR(dev_opp
);
1100 dev_warn(dev
, "%s: Device OPP not found (%d)\n", __func__
, r
);
1104 /* Do we have the frequency? */
1105 list_for_each_entry(tmp_opp
, &dev_opp
->opp_list
, node
) {
1106 if (tmp_opp
->rate
== freq
) {
1116 /* Is update really needed? */
1117 if (opp
->available
== availability_req
)
1119 /* copy the old data over */
1122 /* plug in new node */
1123 new_opp
->available
= availability_req
;
1125 list_replace_rcu(&opp
->node
, &new_opp
->node
);
1126 mutex_unlock(&dev_opp_list_lock
);
1127 call_srcu(&dev_opp
->srcu_head
.srcu
, &opp
->rcu_head
, _kfree_opp_rcu
);
1129 /* Notify the change of the OPP availability */
1130 if (availability_req
)
1131 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_ENABLE
,
1134 srcu_notifier_call_chain(&dev_opp
->srcu_head
, OPP_EVENT_DISABLE
,
1140 mutex_unlock(&dev_opp_list_lock
);
1146 * dev_pm_opp_enable() - Enable a specific OPP
1147 * @dev: device for which we do this operation
1148 * @freq: OPP frequency to enable
1150 * Enables a provided opp. If the operation is valid, this returns 0, else the
1151 * corresponding error value. It is meant to be used for users an OPP available
1152 * after being temporarily made unavailable with dev_pm_opp_disable.
1154 * Locking: The internal device_opp and opp structures are RCU protected.
1155 * Hence this function indirectly uses RCU and mutex locks to keep the
1156 * integrity of the internal data structures. Callers should ensure that
1157 * this function is *NOT* called under RCU protection or in contexts where
1158 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1160 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1161 * copy operation, returns 0 if no modification was done OR modification was
1164 int dev_pm_opp_enable(struct device
*dev
, unsigned long freq
)
1166 return _opp_set_availability(dev
, freq
, true);
1168 EXPORT_SYMBOL_GPL(dev_pm_opp_enable
);
1171 * dev_pm_opp_disable() - Disable a specific OPP
1172 * @dev: device for which we do this operation
1173 * @freq: OPP frequency to disable
1175 * Disables a provided opp. If the operation is valid, this returns
1176 * 0, else the corresponding error value. It is meant to be a temporary
1177 * control by users to make this OPP not available until the circumstances are
1178 * right to make it available again (with a call to dev_pm_opp_enable).
1180 * Locking: The internal device_opp and opp structures are RCU protected.
1181 * Hence this function indirectly uses RCU and mutex locks to keep the
1182 * integrity of the internal data structures. Callers should ensure that
1183 * this function is *NOT* called under RCU protection or in contexts where
1184 * mutex locking or synchronize_rcu() blocking calls cannot be used.
1186 * Return: -EINVAL for bad pointers, -ENOMEM if no memory available for the
1187 * copy operation, returns 0 if no modification was done OR modification was
1190 int dev_pm_opp_disable(struct device
*dev
, unsigned long freq
)
1192 return _opp_set_availability(dev
, freq
, false);
1194 EXPORT_SYMBOL_GPL(dev_pm_opp_disable
);
1197 * dev_pm_opp_get_notifier() - find notifier_head of the device with opp
1198 * @dev: device pointer used to lookup device OPPs.
1200 * Return: pointer to notifier head if found, otherwise -ENODEV or
1201 * -EINVAL based on type of error casted as pointer. value must be checked
1202 * with IS_ERR to determine valid pointer or error result.
1204 * Locking: This function must be called under rcu_read_lock(). dev_opp is a RCU
1205 * protected pointer. The reason for the same is that the opp pointer which is
1206 * returned will remain valid for use with opp_get_{voltage, freq} only while
1207 * under the locked area. The pointer returned must be used prior to unlocking
1208 * with rcu_read_unlock() to maintain the integrity of the pointer.
1210 struct srcu_notifier_head
*dev_pm_opp_get_notifier(struct device
*dev
)
1212 struct device_opp
*dev_opp
= _find_device_opp(dev
);
1214 if (IS_ERR(dev_opp
))
1215 return ERR_CAST(dev_opp
); /* matching type */
1217 return &dev_opp
->srcu_head
;
1219 EXPORT_SYMBOL_GPL(dev_pm_opp_get_notifier
);
1223 * of_free_opp_table() - Free OPP table entries created from static DT entries
1224 * @dev: device pointer used to lookup device OPPs.
1226 * Free OPPs created using static entries present in DT.
1228 * Locking: The internal device_opp and opp structures are RCU protected.
1229 * Hence this function indirectly uses RCU updater strategy with mutex locks
1230 * to keep the integrity of the internal data structures. Callers should ensure
1231 * that this function is *NOT* called under RCU protection or in contexts where
1232 * mutex cannot be locked.
1234 void of_free_opp_table(struct device
*dev
)
1236 struct device_opp
*dev_opp
;
1237 struct dev_pm_opp
*opp
, *tmp
;
1239 /* Hold our list modification lock here */
1240 mutex_lock(&dev_opp_list_lock
);
1242 /* Check for existing list for 'dev' */
1243 dev_opp
= _find_device_opp(dev
);
1244 if (IS_ERR(dev_opp
)) {
1245 int error
= PTR_ERR(dev_opp
);
1247 if (error
!= -ENODEV
)
1248 WARN(1, "%s: dev_opp: %d\n",
1249 IS_ERR_OR_NULL(dev
) ?
1250 "Invalid device" : dev_name(dev
),
1255 /* Find if dev_opp manages a single device */
1256 if (list_is_singular(&dev_opp
->dev_list
)) {
1257 /* Free static OPPs */
1258 list_for_each_entry_safe(opp
, tmp
, &dev_opp
->opp_list
, node
) {
1260 _opp_remove(dev_opp
, opp
, true);
1263 _remove_list_dev(_find_list_dev(dev
, dev_opp
), dev_opp
);
1267 mutex_unlock(&dev_opp_list_lock
);
1269 EXPORT_SYMBOL_GPL(of_free_opp_table
);
1271 void of_cpumask_free_opp_table(cpumask_var_t cpumask
)
1273 struct device
*cpu_dev
;
1276 WARN_ON(cpumask_empty(cpumask
));
1278 for_each_cpu(cpu
, cpumask
) {
1279 cpu_dev
= get_cpu_device(cpu
);
1281 pr_err("%s: failed to get cpu%d device\n", __func__
,
1286 of_free_opp_table(cpu_dev
);
1289 EXPORT_SYMBOL_GPL(of_cpumask_free_opp_table
);
1291 /* Returns opp descriptor node from its phandle. Caller must do of_node_put() */
1292 static struct device_node
*
1293 _of_get_opp_desc_node_from_prop(struct device
*dev
, const struct property
*prop
)
1295 struct device_node
*opp_np
;
1297 opp_np
= of_find_node_by_phandle(be32_to_cpup(prop
->value
));
1299 dev_err(dev
, "%s: Prop: %s contains invalid opp desc phandle\n",
1300 __func__
, prop
->name
);
1301 return ERR_PTR(-EINVAL
);
1307 /* Returns opp descriptor node for a device. Caller must do of_node_put() */
1308 static struct device_node
*_of_get_opp_desc_node(struct device
*dev
)
1310 const struct property
*prop
;
1312 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1314 return ERR_PTR(-ENODEV
);
1316 return ERR_PTR(-ENODATA
);
1319 * TODO: Support for multiple OPP tables.
1321 * There should be only ONE phandle present in "operating-points-v2"
1324 if (prop
->length
!= sizeof(__be32
)) {
1325 dev_err(dev
, "%s: Invalid opp desc phandle\n", __func__
);
1326 return ERR_PTR(-EINVAL
);
1329 return _of_get_opp_desc_node_from_prop(dev
, prop
);
1332 /* Initializes OPP tables based on new bindings */
1333 static int _of_init_opp_table_v2(struct device
*dev
,
1334 const struct property
*prop
)
1336 struct device_node
*opp_np
, *np
;
1337 struct device_opp
*dev_opp
;
1338 int ret
= 0, count
= 0;
1344 opp_np
= _of_get_opp_desc_node_from_prop(dev
, prop
);
1346 return PTR_ERR(opp_np
);
1348 dev_opp
= _managed_opp(opp_np
);
1350 /* OPPs are already managed */
1351 if (!_add_list_dev(dev
, dev_opp
))
1356 /* We have opp-list node now, iterate over it and add OPPs */
1357 for_each_available_child_of_node(opp_np
, np
) {
1360 ret
= _opp_add_static_v2(dev
, np
);
1362 dev_err(dev
, "%s: Failed to add OPP, %d\n", __func__
,
1368 /* There should be one of more OPP defined */
1369 if (WARN_ON(!count
)) {
1374 dev_opp
= _find_device_opp(dev
);
1375 if (WARN_ON(IS_ERR(dev_opp
))) {
1376 ret
= PTR_ERR(dev_opp
);
1380 dev_opp
->np
= opp_np
;
1381 dev_opp
->shared_opp
= of_property_read_bool(opp_np
, "opp-shared");
1383 of_node_put(opp_np
);
1387 of_free_opp_table(dev
);
1389 of_node_put(opp_np
);
1394 /* Initializes OPP tables based on old-deprecated bindings */
1395 static int _of_init_opp_table_v1(struct device
*dev
)
1397 const struct property
*prop
;
1401 prop
= of_find_property(dev
->of_node
, "operating-points", NULL
);
1408 * Each OPP is a set of tuples consisting of frequency and
1409 * voltage like <freq-kHz vol-uV>.
1411 nr
= prop
->length
/ sizeof(u32
);
1413 dev_err(dev
, "%s: Invalid OPP list\n", __func__
);
1419 unsigned long freq
= be32_to_cpup(val
++) * 1000;
1420 unsigned long volt
= be32_to_cpup(val
++);
1422 if (_opp_add_dynamic(dev
, freq
, volt
, false))
1423 dev_warn(dev
, "%s: Failed to add OPP %ld\n",
1432 * of_init_opp_table() - Initialize opp table from device tree
1433 * @dev: device pointer used to lookup device OPPs.
1435 * Register the initial OPP table with the OPP library for given device.
1437 * Locking: The internal device_opp and opp structures are RCU protected.
1438 * Hence this function indirectly uses RCU updater strategy with mutex locks
1439 * to keep the integrity of the internal data structures. Callers should ensure
1440 * that this function is *NOT* called under RCU protection or in contexts where
1441 * mutex cannot be locked.
1445 * Duplicate OPPs (both freq and volt are same) and opp->available
1446 * -EEXIST Freq are same and volt are different OR
1447 * Duplicate OPPs (both freq and volt are same) and !opp->available
1448 * -ENOMEM Memory allocation failure
1449 * -ENODEV when 'operating-points' property is not found or is invalid data
1451 * -ENODATA when empty 'operating-points' property is found
1452 * -EINVAL when invalid entries are found in opp-v2 table
1454 int of_init_opp_table(struct device
*dev
)
1456 const struct property
*prop
;
1459 * OPPs have two version of bindings now. The older one is deprecated,
1460 * try for the new binding first.
1462 prop
= of_find_property(dev
->of_node
, "operating-points-v2", NULL
);
1465 * Try old-deprecated bindings for backward compatibility with
1468 return _of_init_opp_table_v1(dev
);
1471 return _of_init_opp_table_v2(dev
, prop
);
1473 EXPORT_SYMBOL_GPL(of_init_opp_table
);
1475 int of_cpumask_init_opp_table(cpumask_var_t cpumask
)
1477 struct device
*cpu_dev
;
1480 WARN_ON(cpumask_empty(cpumask
));
1482 for_each_cpu(cpu
, cpumask
) {
1483 cpu_dev
= get_cpu_device(cpu
);
1485 pr_err("%s: failed to get cpu%d device\n", __func__
,
1490 ret
= of_init_opp_table(cpu_dev
);
1492 pr_err("%s: couldn't find opp table for cpu:%d, %d\n",
1493 __func__
, cpu
, ret
);
1495 /* Free all other OPPs */
1496 of_cpumask_free_opp_table(cpumask
);
1503 EXPORT_SYMBOL_GPL(of_cpumask_init_opp_table
);
1505 /* Required only for V1 bindings, as v2 can manage it from DT itself */
1506 int set_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1508 struct device_list_opp
*list_dev
;
1509 struct device_opp
*dev_opp
;
1515 dev_opp
= _find_device_opp(cpu_dev
);
1516 if (IS_ERR(dev_opp
)) {
1518 goto out_rcu_read_unlock
;
1521 for_each_cpu(cpu
, cpumask
) {
1522 if (cpu
== cpu_dev
->id
)
1525 dev
= get_cpu_device(cpu
);
1527 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1532 list_dev
= _add_list_dev(dev
, dev_opp
);
1534 dev_err(dev
, "%s: failed to add list-dev for cpu%d device\n",
1539 out_rcu_read_unlock
:
1544 EXPORT_SYMBOL_GPL(set_cpus_sharing_opps
);
1547 * Works only for OPP v2 bindings.
1549 * cpumask should be already set to mask of cpu_dev->id.
1550 * Returns -ENOENT if operating-points-v2 bindings aren't supported.
1552 int of_get_cpus_sharing_opps(struct device
*cpu_dev
, cpumask_var_t cpumask
)
1554 struct device_node
*np
, *tmp_np
;
1555 struct device
*tcpu_dev
;
1558 /* Get OPP descriptor node */
1559 np
= _of_get_opp_desc_node(cpu_dev
);
1561 dev_dbg(cpu_dev
, "%s: Couldn't find opp node: %ld\n", __func__
,
1566 /* OPPs are shared ? */
1567 if (!of_property_read_bool(np
, "opp-shared"))
1570 for_each_possible_cpu(cpu
) {
1571 if (cpu
== cpu_dev
->id
)
1574 tcpu_dev
= get_cpu_device(cpu
);
1576 dev_err(cpu_dev
, "%s: failed to get cpu%d device\n",
1582 /* Get OPP descriptor node */
1583 tmp_np
= _of_get_opp_desc_node(tcpu_dev
);
1584 if (IS_ERR(tmp_np
)) {
1585 dev_err(tcpu_dev
, "%s: Couldn't find opp node: %ld\n",
1586 __func__
, PTR_ERR(tmp_np
));
1587 ret
= PTR_ERR(tmp_np
);
1591 /* CPUs are sharing opp node */
1593 cpumask_set_cpu(cpu
, cpumask
);
1595 of_node_put(tmp_np
);
1602 EXPORT_SYMBOL_GPL(of_get_cpus_sharing_opps
);