2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1 (4) /* L1 state */
33 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
37 u32 l0s
; /* L0s latency (nsec) */
38 u32 l1
; /* L1 latency (nsec) */
41 struct pcie_link_state
{
42 struct pci_dev
*pdev
; /* Upstream component of the Link */
43 struct pcie_link_state
*root
; /* pointer to the root port link */
44 struct pcie_link_state
*parent
; /* pointer to the parent Link state */
45 struct list_head sibling
; /* node in link_list */
46 struct list_head children
; /* list of child link states */
47 struct list_head link
; /* node in parent's children list */
50 u32 aspm_support
:3; /* Supported ASPM state */
51 u32 aspm_enabled
:3; /* Enabled ASPM state */
52 u32 aspm_capable
:3; /* Capable ASPM state with latency */
53 u32 aspm_default
:3; /* Default ASPM state by BIOS */
54 u32 aspm_disable
:3; /* Disabled ASPM state */
57 u32 clkpm_capable
:1; /* Clock PM capable? */
58 u32 clkpm_enabled
:1; /* Current Clock PM state */
59 u32 clkpm_default
:1; /* Default Clock PM state by BIOS */
62 struct aspm_latency latency_up
; /* Upstream direction exit latency */
63 struct aspm_latency latency_dw
; /* Downstream direction exit latency */
65 * Endpoint acceptable latencies. A pcie downstream port only
66 * has one slot under it, so at most there are 8 functions.
68 struct aspm_latency acceptable
[8];
71 static int aspm_disabled
, aspm_force
;
72 static bool aspm_support_enabled
= true;
73 static DEFINE_MUTEX(aspm_lock
);
74 static LIST_HEAD(link_list
);
76 #define POLICY_DEFAULT 0 /* BIOS default setting */
77 #define POLICY_PERFORMANCE 1 /* high performance */
78 #define POLICY_POWERSAVE 2 /* high power saving */
80 #ifdef CONFIG_PCIEASPM_PERFORMANCE
81 static int aspm_policy
= POLICY_PERFORMANCE
;
82 #elif defined CONFIG_PCIEASPM_POWERSAVE
83 static int aspm_policy
= POLICY_POWERSAVE
;
85 static int aspm_policy
;
88 static const char *policy_str
[] = {
89 [POLICY_DEFAULT
] = "default",
90 [POLICY_PERFORMANCE
] = "performance",
91 [POLICY_POWERSAVE
] = "powersave"
94 #define LINK_RETRAIN_TIMEOUT HZ
96 static int policy_to_aspm_state(struct pcie_link_state
*link
)
98 switch (aspm_policy
) {
99 case POLICY_PERFORMANCE
:
100 /* Disable ASPM and Clock PM */
102 case POLICY_POWERSAVE
:
103 /* Enable ASPM L0s/L1 */
104 return ASPM_STATE_ALL
;
106 return link
->aspm_default
;
111 static int policy_to_clkpm_state(struct pcie_link_state
*link
)
113 switch (aspm_policy
) {
114 case POLICY_PERFORMANCE
:
115 /* Disable ASPM and Clock PM */
117 case POLICY_POWERSAVE
:
118 /* Disable Clock PM */
121 return link
->clkpm_default
;
126 static void pcie_set_clkpm_nocheck(struct pcie_link_state
*link
, int enable
)
130 struct pci_dev
*child
;
131 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
133 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
134 pos
= pci_pcie_cap(child
);
137 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
139 reg16
|= PCI_EXP_LNKCTL_CLKREQ_EN
;
141 reg16
&= ~PCI_EXP_LNKCTL_CLKREQ_EN
;
142 pci_write_config_word(child
, pos
+ PCI_EXP_LNKCTL
, reg16
);
144 link
->clkpm_enabled
= !!enable
;
147 static void pcie_set_clkpm(struct pcie_link_state
*link
, int enable
)
149 /* Don't enable Clock PM if the link is not Clock PM capable */
150 if (!link
->clkpm_capable
&& enable
)
152 /* Need nothing if the specified equals to current state */
153 if (link
->clkpm_enabled
== enable
)
155 pcie_set_clkpm_nocheck(link
, enable
);
158 static void pcie_clkpm_cap_init(struct pcie_link_state
*link
, int blacklist
)
160 int pos
, capable
= 1, enabled
= 1;
163 struct pci_dev
*child
;
164 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
166 /* All functions should have the same cap and state, take the worst */
167 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
168 pos
= pci_pcie_cap(child
);
171 pci_read_config_dword(child
, pos
+ PCI_EXP_LNKCAP
, ®32
);
172 if (!(reg32
& PCI_EXP_LNKCAP_CLKPM
)) {
177 pci_read_config_word(child
, pos
+ PCI_EXP_LNKCTL
, ®16
);
178 if (!(reg16
& PCI_EXP_LNKCTL_CLKREQ_EN
))
181 link
->clkpm_enabled
= enabled
;
182 link
->clkpm_default
= enabled
;
183 link
->clkpm_capable
= (blacklist
) ? 0 : capable
;
187 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
188 * could use common clock. If they are, configure them to use the
189 * common clock. That will reduce the ASPM state exit latency.
191 static void pcie_aspm_configure_common_clock(struct pcie_link_state
*link
)
193 int ppos
, cpos
, same_clock
= 1;
194 u16 reg16
, parent_reg
, child_reg
[8];
195 unsigned long start_jiffies
;
196 struct pci_dev
*child
, *parent
= link
->pdev
;
197 struct pci_bus
*linkbus
= parent
->subordinate
;
199 * All functions of a slot should have the same Slot Clock
200 * Configuration, so just check one function
202 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
203 BUG_ON(!pci_is_pcie(child
));
205 /* Check downstream component if bit Slot Clock Configuration is 1 */
206 cpos
= pci_pcie_cap(child
);
207 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKSTA
, ®16
);
208 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
211 /* Check upstream component if bit Slot Clock Configuration is 1 */
212 ppos
= pci_pcie_cap(parent
);
213 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
214 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
217 /* Configure downstream component, all functions */
218 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
219 cpos
= pci_pcie_cap(child
);
220 pci_read_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, ®16
);
221 child_reg
[PCI_FUNC(child
->devfn
)] = reg16
;
223 reg16
|= PCI_EXP_LNKCTL_CCC
;
225 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
226 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
, reg16
);
229 /* Configure upstream component */
230 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, ®16
);
233 reg16
|= PCI_EXP_LNKCTL_CCC
;
235 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
236 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
239 reg16
|= PCI_EXP_LNKCTL_RL
;
240 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, reg16
);
242 /* Wait for link training end. Break out after waiting for timeout */
243 start_jiffies
= jiffies
;
245 pci_read_config_word(parent
, ppos
+ PCI_EXP_LNKSTA
, ®16
);
246 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
248 if (time_after(jiffies
, start_jiffies
+ LINK_RETRAIN_TIMEOUT
))
252 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
255 /* Training failed. Restore common clock configurations */
256 dev_printk(KERN_ERR
, &parent
->dev
,
257 "ASPM: Could not configure common clock\n");
258 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
259 cpos
= pci_pcie_cap(child
);
260 pci_write_config_word(child
, cpos
+ PCI_EXP_LNKCTL
,
261 child_reg
[PCI_FUNC(child
->devfn
)]);
263 pci_write_config_word(parent
, ppos
+ PCI_EXP_LNKCTL
, parent_reg
);
266 /* Convert L0s latency encoding to ns */
267 static u32
calc_l0s_latency(u32 encoding
)
270 return (5 * 1000); /* > 4us */
271 return (64 << encoding
);
274 /* Convert L0s acceptable latency encoding to ns */
275 static u32
calc_l0s_acceptable(u32 encoding
)
279 return (64 << encoding
);
282 /* Convert L1 latency encoding to ns */
283 static u32
calc_l1_latency(u32 encoding
)
286 return (65 * 1000); /* > 64us */
287 return (1000 << encoding
);
290 /* Convert L1 acceptable latency encoding to ns */
291 static u32
calc_l1_acceptable(u32 encoding
)
295 return (1000 << encoding
);
298 struct aspm_register_info
{
301 u32 latency_encoding_l0s
;
302 u32 latency_encoding_l1
;
305 static void pcie_get_aspm_reg(struct pci_dev
*pdev
,
306 struct aspm_register_info
*info
)
312 pos
= pci_pcie_cap(pdev
);
313 pci_read_config_dword(pdev
, pos
+ PCI_EXP_LNKCAP
, ®32
);
314 info
->support
= (reg32
& PCI_EXP_LNKCAP_ASPMS
) >> 10;
315 info
->latency_encoding_l0s
= (reg32
& PCI_EXP_LNKCAP_L0SEL
) >> 12;
316 info
->latency_encoding_l1
= (reg32
& PCI_EXP_LNKCAP_L1EL
) >> 15;
317 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
318 info
->enabled
= reg16
& PCI_EXP_LNKCTL_ASPMC
;
321 static void pcie_aspm_check_latency(struct pci_dev
*endpoint
)
323 u32 latency
, l1_switch_latency
= 0;
324 struct aspm_latency
*acceptable
;
325 struct pcie_link_state
*link
;
327 /* Device not in D0 doesn't need latency check */
328 if ((endpoint
->current_state
!= PCI_D0
) &&
329 (endpoint
->current_state
!= PCI_UNKNOWN
))
332 link
= endpoint
->bus
->self
->link_state
;
333 acceptable
= &link
->acceptable
[PCI_FUNC(endpoint
->devfn
)];
336 /* Check upstream direction L0s latency */
337 if ((link
->aspm_capable
& ASPM_STATE_L0S_UP
) &&
338 (link
->latency_up
.l0s
> acceptable
->l0s
))
339 link
->aspm_capable
&= ~ASPM_STATE_L0S_UP
;
341 /* Check downstream direction L0s latency */
342 if ((link
->aspm_capable
& ASPM_STATE_L0S_DW
) &&
343 (link
->latency_dw
.l0s
> acceptable
->l0s
))
344 link
->aspm_capable
&= ~ASPM_STATE_L0S_DW
;
347 * Every switch on the path to root complex need 1
348 * more microsecond for L1. Spec doesn't mention L0s.
350 latency
= max_t(u32
, link
->latency_up
.l1
, link
->latency_dw
.l1
);
351 if ((link
->aspm_capable
& ASPM_STATE_L1
) &&
352 (latency
+ l1_switch_latency
> acceptable
->l1
))
353 link
->aspm_capable
&= ~ASPM_STATE_L1
;
354 l1_switch_latency
+= 1000;
360 static void pcie_aspm_cap_init(struct pcie_link_state
*link
, int blacklist
)
362 struct pci_dev
*child
, *parent
= link
->pdev
;
363 struct pci_bus
*linkbus
= parent
->subordinate
;
364 struct aspm_register_info upreg
, dwreg
;
367 /* Set enabled/disable so that we will disable ASPM later */
368 link
->aspm_enabled
= ASPM_STATE_ALL
;
369 link
->aspm_disable
= ASPM_STATE_ALL
;
373 /* Configure common clock before checking latencies */
374 pcie_aspm_configure_common_clock(link
);
376 /* Get upstream/downstream components' register state */
377 pcie_get_aspm_reg(parent
, &upreg
);
378 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
379 pcie_get_aspm_reg(child
, &dwreg
);
384 * Note that we must not enable L0s in either direction on a
385 * given link unless components on both sides of the link each
388 if (dwreg
.support
& upreg
.support
& PCIE_LINK_STATE_L0S
)
389 link
->aspm_support
|= ASPM_STATE_L0S
;
390 if (dwreg
.enabled
& PCIE_LINK_STATE_L0S
)
391 link
->aspm_enabled
|= ASPM_STATE_L0S_UP
;
392 if (upreg
.enabled
& PCIE_LINK_STATE_L0S
)
393 link
->aspm_enabled
|= ASPM_STATE_L0S_DW
;
394 link
->latency_up
.l0s
= calc_l0s_latency(upreg
.latency_encoding_l0s
);
395 link
->latency_dw
.l0s
= calc_l0s_latency(dwreg
.latency_encoding_l0s
);
398 if (upreg
.support
& dwreg
.support
& PCIE_LINK_STATE_L1
)
399 link
->aspm_support
|= ASPM_STATE_L1
;
400 if (upreg
.enabled
& dwreg
.enabled
& PCIE_LINK_STATE_L1
)
401 link
->aspm_enabled
|= ASPM_STATE_L1
;
402 link
->latency_up
.l1
= calc_l1_latency(upreg
.latency_encoding_l1
);
403 link
->latency_dw
.l1
= calc_l1_latency(dwreg
.latency_encoding_l1
);
405 /* Save default state */
406 link
->aspm_default
= link
->aspm_enabled
;
408 /* Setup initial capable state. Will be updated later */
409 link
->aspm_capable
= link
->aspm_support
;
411 * If the downstream component has pci bridge function, don't
414 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
415 if (child
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
) {
416 link
->aspm_disable
= ASPM_STATE_ALL
;
421 /* Get and check endpoint acceptable latencies */
422 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
425 struct aspm_latency
*acceptable
=
426 &link
->acceptable
[PCI_FUNC(child
->devfn
)];
428 if (child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
&&
429 child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
)
432 pos
= pci_pcie_cap(child
);
433 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
434 /* Calculate endpoint L0s acceptable latency */
435 encoding
= (reg32
& PCI_EXP_DEVCAP_L0S
) >> 6;
436 acceptable
->l0s
= calc_l0s_acceptable(encoding
);
437 /* Calculate endpoint L1 acceptable latency */
438 encoding
= (reg32
& PCI_EXP_DEVCAP_L1
) >> 9;
439 acceptable
->l1
= calc_l1_acceptable(encoding
);
441 pcie_aspm_check_latency(child
);
445 static void pcie_config_aspm_dev(struct pci_dev
*pdev
, u32 val
)
448 int pos
= pci_pcie_cap(pdev
);
450 pci_read_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, ®16
);
453 pci_write_config_word(pdev
, pos
+ PCI_EXP_LNKCTL
, reg16
);
456 static void pcie_config_aspm_link(struct pcie_link_state
*link
, u32 state
)
458 u32 upstream
= 0, dwstream
= 0;
459 struct pci_dev
*child
, *parent
= link
->pdev
;
460 struct pci_bus
*linkbus
= parent
->subordinate
;
462 /* Nothing to do if the link is already in the requested state */
463 state
&= (link
->aspm_capable
& ~link
->aspm_disable
);
464 if (link
->aspm_enabled
== state
)
466 /* Convert ASPM state to upstream/downstream ASPM register state */
467 if (state
& ASPM_STATE_L0S_UP
)
468 dwstream
|= PCIE_LINK_STATE_L0S
;
469 if (state
& ASPM_STATE_L0S_DW
)
470 upstream
|= PCIE_LINK_STATE_L0S
;
471 if (state
& ASPM_STATE_L1
) {
472 upstream
|= PCIE_LINK_STATE_L1
;
473 dwstream
|= PCIE_LINK_STATE_L1
;
476 * Spec 2.0 suggests all functions should be configured the
477 * same setting for ASPM. Enabling ASPM L1 should be done in
478 * upstream component first and then downstream, and vice
479 * versa for disabling ASPM L1. Spec doesn't mention L0S.
481 if (state
& ASPM_STATE_L1
)
482 pcie_config_aspm_dev(parent
, upstream
);
483 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
484 pcie_config_aspm_dev(child
, dwstream
);
485 if (!(state
& ASPM_STATE_L1
))
486 pcie_config_aspm_dev(parent
, upstream
);
488 link
->aspm_enabled
= state
;
491 static void pcie_config_aspm_path(struct pcie_link_state
*link
)
494 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
499 static void free_link_state(struct pcie_link_state
*link
)
501 link
->pdev
->link_state
= NULL
;
505 static int pcie_aspm_sanity_check(struct pci_dev
*pdev
)
507 struct pci_dev
*child
;
515 * Some functions in a slot might not all be PCIe functions,
516 * very strange. Disable ASPM for the whole slot
518 list_for_each_entry(child
, &pdev
->subordinate
->devices
, bus_list
) {
519 pos
= pci_pcie_cap(child
);
523 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
524 * RBER bit to determine if a function is 1.1 version device
526 pci_read_config_dword(child
, pos
+ PCI_EXP_DEVCAP
, ®32
);
527 if (!(reg32
& PCI_EXP_DEVCAP_RBER
) && !aspm_force
) {
528 dev_printk(KERN_INFO
, &child
->dev
, "disabling ASPM"
529 " on pre-1.1 PCIe device. You can enable it"
530 " with 'pcie_aspm=force'\n");
537 static struct pcie_link_state
*alloc_pcie_link_state(struct pci_dev
*pdev
)
539 struct pcie_link_state
*link
;
541 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
544 INIT_LIST_HEAD(&link
->sibling
);
545 INIT_LIST_HEAD(&link
->children
);
546 INIT_LIST_HEAD(&link
->link
);
548 if (pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
) {
549 struct pcie_link_state
*parent
;
550 parent
= pdev
->bus
->parent
->self
->link_state
;
555 link
->parent
= parent
;
556 list_add(&link
->link
, &parent
->children
);
558 /* Setup a pointer to the root port link */
562 link
->root
= link
->parent
->root
;
564 list_add(&link
->sibling
, &link_list
);
565 pdev
->link_state
= link
;
570 * pcie_aspm_init_link_state: Initiate PCI express link state.
571 * It is called after the pcie and its children devices are scaned.
572 * @pdev: the root port or switch downstream port
574 void pcie_aspm_init_link_state(struct pci_dev
*pdev
)
576 struct pcie_link_state
*link
;
577 int blacklist
= !!pcie_aspm_sanity_check(pdev
);
579 if (!pci_is_pcie(pdev
) || pdev
->link_state
)
581 if (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
582 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
)
585 /* VIA has a strange chipset, root port is under a bridge */
586 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
&&
590 down_read(&pci_bus_sem
);
591 if (list_empty(&pdev
->subordinate
->devices
))
594 mutex_lock(&aspm_lock
);
595 link
= alloc_pcie_link_state(pdev
);
599 * Setup initial ASPM state. Note that we need to configure
600 * upstream links also because capable state of them can be
601 * update through pcie_aspm_cap_init().
603 pcie_aspm_cap_init(link
, blacklist
);
605 /* Setup initial Clock PM state */
606 pcie_clkpm_cap_init(link
, blacklist
);
609 * At this stage drivers haven't had an opportunity to change the
610 * link policy setting. Enabling ASPM on broken hardware can cripple
611 * it even before the driver has had a chance to disable ASPM, so
612 * default to a safe level right now. If we're enabling ASPM beyond
613 * the BIOS's expectation, we'll do so once pci_enable_device() is
616 if (aspm_policy
!= POLICY_POWERSAVE
) {
617 pcie_config_aspm_path(link
);
618 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
622 mutex_unlock(&aspm_lock
);
624 up_read(&pci_bus_sem
);
627 /* Recheck latencies and update aspm_capable for links under the root */
628 static void pcie_update_aspm_capable(struct pcie_link_state
*root
)
630 struct pcie_link_state
*link
;
631 BUG_ON(root
->parent
);
632 list_for_each_entry(link
, &link_list
, sibling
) {
633 if (link
->root
!= root
)
635 link
->aspm_capable
= link
->aspm_support
;
637 list_for_each_entry(link
, &link_list
, sibling
) {
638 struct pci_dev
*child
;
639 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
640 if (link
->root
!= root
)
642 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
643 if ((child
->pcie_type
!= PCI_EXP_TYPE_ENDPOINT
) &&
644 (child
->pcie_type
!= PCI_EXP_TYPE_LEG_END
))
646 pcie_aspm_check_latency(child
);
651 /* @pdev: the endpoint device */
652 void pcie_aspm_exit_link_state(struct pci_dev
*pdev
)
654 struct pci_dev
*parent
= pdev
->bus
->self
;
655 struct pcie_link_state
*link
, *root
, *parent_link
;
657 if (!pci_is_pcie(pdev
) || !parent
|| !parent
->link_state
)
659 if ((parent
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
660 (parent
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
663 down_read(&pci_bus_sem
);
664 mutex_lock(&aspm_lock
);
666 * All PCIe functions are in one slot, remove one function will remove
667 * the whole slot, so just wait until we are the last function left.
669 if (!list_is_last(&pdev
->bus_list
, &parent
->subordinate
->devices
))
672 link
= parent
->link_state
;
674 parent_link
= link
->parent
;
676 /* All functions are removed, so just disable ASPM for the link */
677 pcie_config_aspm_link(link
, 0);
678 list_del(&link
->sibling
);
679 list_del(&link
->link
);
680 /* Clock PM is for endpoint device */
681 free_link_state(link
);
683 /* Recheck latencies and configure upstream links */
685 pcie_update_aspm_capable(root
);
686 pcie_config_aspm_path(parent_link
);
689 mutex_unlock(&aspm_lock
);
690 up_read(&pci_bus_sem
);
693 /* @pdev: the root port or switch downstream port */
694 void pcie_aspm_pm_state_change(struct pci_dev
*pdev
)
696 struct pcie_link_state
*link
= pdev
->link_state
;
698 if (aspm_disabled
|| !pci_is_pcie(pdev
) || !link
)
700 if ((pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
701 (pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
704 * Devices changed PM state, we should recheck if latency
705 * meets all functions' requirement
707 down_read(&pci_bus_sem
);
708 mutex_lock(&aspm_lock
);
709 pcie_update_aspm_capable(link
->root
);
710 pcie_config_aspm_path(link
);
711 mutex_unlock(&aspm_lock
);
712 up_read(&pci_bus_sem
);
715 void pcie_aspm_powersave_config_link(struct pci_dev
*pdev
)
717 struct pcie_link_state
*link
= pdev
->link_state
;
719 if (aspm_disabled
|| !pci_is_pcie(pdev
) || !link
)
722 if (aspm_policy
!= POLICY_POWERSAVE
)
725 if ((pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
) &&
726 (pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
))
729 down_read(&pci_bus_sem
);
730 mutex_lock(&aspm_lock
);
731 pcie_config_aspm_path(link
);
732 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
733 mutex_unlock(&aspm_lock
);
734 up_read(&pci_bus_sem
);
738 * pci_disable_link_state - disable pci device's link state, so the link will
739 * never enter specific states
741 static void __pci_disable_link_state(struct pci_dev
*pdev
, int state
, bool sem
,
744 struct pci_dev
*parent
= pdev
->bus
->self
;
745 struct pcie_link_state
*link
;
747 if (aspm_disabled
&& !force
)
750 if (!pci_is_pcie(pdev
))
753 if (pdev
->pcie_type
== PCI_EXP_TYPE_ROOT_PORT
||
754 pdev
->pcie_type
== PCI_EXP_TYPE_DOWNSTREAM
)
756 if (!parent
|| !parent
->link_state
)
760 down_read(&pci_bus_sem
);
761 mutex_lock(&aspm_lock
);
762 link
= parent
->link_state
;
763 if (state
& PCIE_LINK_STATE_L0S
)
764 link
->aspm_disable
|= ASPM_STATE_L0S
;
765 if (state
& PCIE_LINK_STATE_L1
)
766 link
->aspm_disable
|= ASPM_STATE_L1
;
767 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
769 if (state
& PCIE_LINK_STATE_CLKPM
) {
770 link
->clkpm_capable
= 0;
771 pcie_set_clkpm(link
, 0);
773 mutex_unlock(&aspm_lock
);
775 up_read(&pci_bus_sem
);
778 void pci_disable_link_state_locked(struct pci_dev
*pdev
, int state
)
780 __pci_disable_link_state(pdev
, state
, false, false);
782 EXPORT_SYMBOL(pci_disable_link_state_locked
);
784 void pci_disable_link_state(struct pci_dev
*pdev
, int state
)
786 __pci_disable_link_state(pdev
, state
, true, false);
788 EXPORT_SYMBOL(pci_disable_link_state
);
790 void pcie_clear_aspm(struct pci_bus
*bus
)
792 struct pci_dev
*child
;
795 * Clear any ASPM setup that the firmware has carried out on this bus
797 list_for_each_entry(child
, &bus
->devices
, bus_list
) {
798 __pci_disable_link_state(child
, PCIE_LINK_STATE_L0S
|
800 PCIE_LINK_STATE_CLKPM
,
805 static int pcie_aspm_set_policy(const char *val
, struct kernel_param
*kp
)
808 struct pcie_link_state
*link
;
812 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
813 if (!strncmp(val
, policy_str
[i
], strlen(policy_str
[i
])))
815 if (i
>= ARRAY_SIZE(policy_str
))
817 if (i
== aspm_policy
)
820 down_read(&pci_bus_sem
);
821 mutex_lock(&aspm_lock
);
823 list_for_each_entry(link
, &link_list
, sibling
) {
824 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
825 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
827 mutex_unlock(&aspm_lock
);
828 up_read(&pci_bus_sem
);
832 static int pcie_aspm_get_policy(char *buffer
, struct kernel_param
*kp
)
835 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
836 if (i
== aspm_policy
)
837 cnt
+= sprintf(buffer
+ cnt
, "[%s] ", policy_str
[i
]);
839 cnt
+= sprintf(buffer
+ cnt
, "%s ", policy_str
[i
]);
843 module_param_call(policy
, pcie_aspm_set_policy
, pcie_aspm_get_policy
,
846 #ifdef CONFIG_PCIEASPM_DEBUG
847 static ssize_t
link_state_show(struct device
*dev
,
848 struct device_attribute
*attr
,
851 struct pci_dev
*pci_device
= to_pci_dev(dev
);
852 struct pcie_link_state
*link_state
= pci_device
->link_state
;
854 return sprintf(buf
, "%d\n", link_state
->aspm_enabled
);
857 static ssize_t
link_state_store(struct device
*dev
,
858 struct device_attribute
*attr
,
862 struct pci_dev
*pdev
= to_pci_dev(dev
);
863 struct pcie_link_state
*link
, *root
= pdev
->link_state
->root
;
864 u32 val
= buf
[0] - '0', state
= 0;
868 if (n
< 1 || val
> 3)
871 /* Convert requested state to ASPM state */
872 if (val
& PCIE_LINK_STATE_L0S
)
873 state
|= ASPM_STATE_L0S
;
874 if (val
& PCIE_LINK_STATE_L1
)
875 state
|= ASPM_STATE_L1
;
877 down_read(&pci_bus_sem
);
878 mutex_lock(&aspm_lock
);
879 list_for_each_entry(link
, &link_list
, sibling
) {
880 if (link
->root
!= root
)
882 pcie_config_aspm_link(link
, state
);
884 mutex_unlock(&aspm_lock
);
885 up_read(&pci_bus_sem
);
889 static ssize_t
clk_ctl_show(struct device
*dev
,
890 struct device_attribute
*attr
,
893 struct pci_dev
*pci_device
= to_pci_dev(dev
);
894 struct pcie_link_state
*link_state
= pci_device
->link_state
;
896 return sprintf(buf
, "%d\n", link_state
->clkpm_enabled
);
899 static ssize_t
clk_ctl_store(struct device
*dev
,
900 struct device_attribute
*attr
,
904 struct pci_dev
*pdev
= to_pci_dev(dev
);
911 down_read(&pci_bus_sem
);
912 mutex_lock(&aspm_lock
);
913 pcie_set_clkpm_nocheck(pdev
->link_state
, !!state
);
914 mutex_unlock(&aspm_lock
);
915 up_read(&pci_bus_sem
);
920 static DEVICE_ATTR(link_state
, 0644, link_state_show
, link_state_store
);
921 static DEVICE_ATTR(clk_ctl
, 0644, clk_ctl_show
, clk_ctl_store
);
923 static char power_group
[] = "power";
924 void pcie_aspm_create_sysfs_dev_files(struct pci_dev
*pdev
)
926 struct pcie_link_state
*link_state
= pdev
->link_state
;
928 if (!pci_is_pcie(pdev
) ||
929 (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
930 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
933 if (link_state
->aspm_support
)
934 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
935 &dev_attr_link_state
.attr
, power_group
);
936 if (link_state
->clkpm_capable
)
937 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
938 &dev_attr_clk_ctl
.attr
, power_group
);
941 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev
*pdev
)
943 struct pcie_link_state
*link_state
= pdev
->link_state
;
945 if (!pci_is_pcie(pdev
) ||
946 (pdev
->pcie_type
!= PCI_EXP_TYPE_ROOT_PORT
&&
947 pdev
->pcie_type
!= PCI_EXP_TYPE_DOWNSTREAM
) || !link_state
)
950 if (link_state
->aspm_support
)
951 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
952 &dev_attr_link_state
.attr
, power_group
);
953 if (link_state
->clkpm_capable
)
954 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
955 &dev_attr_clk_ctl
.attr
, power_group
);
959 static int __init
pcie_aspm_disable(char *str
)
961 if (!strcmp(str
, "off")) {
962 aspm_policy
= POLICY_DEFAULT
;
964 aspm_support_enabled
= false;
965 printk(KERN_INFO
"PCIe ASPM is disabled\n");
966 } else if (!strcmp(str
, "force")) {
968 printk(KERN_INFO
"PCIe ASPM is forcibly enabled\n");
973 __setup("pcie_aspm=", pcie_aspm_disable
);
975 void pcie_no_aspm(void)
978 * Disabling ASPM is intended to prevent the kernel from modifying
979 * existing hardware state, not to clear existing state. To that end:
980 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
981 * (b) prevent userspace from changing policy
984 aspm_policy
= POLICY_DEFAULT
;
990 * pcie_aspm_enabled - is PCIe ASPM enabled?
992 * Returns true if ASPM has not been disabled by the command-line option
995 int pcie_aspm_enabled(void)
997 return !aspm_disabled
;
999 EXPORT_SYMBOL(pcie_aspm_enabled
);
1001 bool pcie_aspm_support_enabled(void)
1003 return aspm_support_enabled
;
1005 EXPORT_SYMBOL(pcie_aspm_support_enabled
);