2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1 (4) /* L1 state */
33 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
34 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
35 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
36 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
37 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
38 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
39 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
41 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
42 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
46 * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter
47 * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details).
48 * Not sure is there is a way to "calculate" this on the fly, but maybe we
49 * could turn it into a parameter in future. This value has been taken from
50 * the following files from Intel's coreboot (which is the only code I found
52 * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html
53 * https://review.coreboot.org/#/c/8832/
55 #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30))
58 u32 l0s
; /* L0s latency (nsec) */
59 u32 l1
; /* L1 latency (nsec) */
62 struct pcie_link_state
{
63 struct pci_dev
*pdev
; /* Upstream component of the Link */
64 struct pci_dev
*downstream
; /* Downstream component, function 0 */
65 struct pcie_link_state
*root
; /* pointer to the root port link */
66 struct pcie_link_state
*parent
; /* pointer to the parent Link state */
67 struct list_head sibling
; /* node in link_list */
68 struct list_head children
; /* list of child link states */
69 struct list_head link
; /* node in parent's children list */
72 u32 aspm_support
:7; /* Supported ASPM state */
73 u32 aspm_enabled
:7; /* Enabled ASPM state */
74 u32 aspm_capable
:7; /* Capable ASPM state with latency */
75 u32 aspm_default
:7; /* Default ASPM state by BIOS */
76 u32 aspm_disable
:7; /* Disabled ASPM state */
79 u32 clkpm_capable
:1; /* Clock PM capable? */
80 u32 clkpm_enabled
:1; /* Current Clock PM state */
81 u32 clkpm_default
:1; /* Default Clock PM state by BIOS */
84 struct aspm_latency latency_up
; /* Upstream direction exit latency */
85 struct aspm_latency latency_dw
; /* Downstream direction exit latency */
87 * Endpoint acceptable latencies. A pcie downstream port only
88 * has one slot under it, so at most there are 8 functions.
90 struct aspm_latency acceptable
[8];
92 /* L1 PM Substate info */
94 u32 up_cap_ptr
; /* L1SS cap ptr in upstream dev */
95 u32 dw_cap_ptr
; /* L1SS cap ptr in downstream dev */
96 u32 ctl1
; /* value to be programmed in ctl1 */
97 u32 ctl2
; /* value to be programmed in ctl2 */
101 static int aspm_disabled
, aspm_force
;
102 static bool aspm_support_enabled
= true;
103 static DEFINE_MUTEX(aspm_lock
);
104 static LIST_HEAD(link_list
);
106 #define POLICY_DEFAULT 0 /* BIOS default setting */
107 #define POLICY_PERFORMANCE 1 /* high performance */
108 #define POLICY_POWERSAVE 2 /* high power saving */
109 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
111 #ifdef CONFIG_PCIEASPM_PERFORMANCE
112 static int aspm_policy
= POLICY_PERFORMANCE
;
113 #elif defined CONFIG_PCIEASPM_POWERSAVE
114 static int aspm_policy
= POLICY_POWERSAVE
;
115 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
116 static int aspm_policy
= POLICY_POWER_SUPERSAVE
;
118 static int aspm_policy
;
121 static const char *policy_str
[] = {
122 [POLICY_DEFAULT
] = "default",
123 [POLICY_PERFORMANCE
] = "performance",
124 [POLICY_POWERSAVE
] = "powersave",
125 [POLICY_POWER_SUPERSAVE
] = "powersupersave"
128 #define LINK_RETRAIN_TIMEOUT HZ
130 static int policy_to_aspm_state(struct pcie_link_state
*link
)
132 switch (aspm_policy
) {
133 case POLICY_PERFORMANCE
:
134 /* Disable ASPM and Clock PM */
136 case POLICY_POWERSAVE
:
137 /* Enable ASPM L0s/L1 */
138 return (ASPM_STATE_L0S
| ASPM_STATE_L1
);
139 case POLICY_POWER_SUPERSAVE
:
140 /* Enable Everything */
141 return ASPM_STATE_ALL
;
143 return link
->aspm_default
;
148 static int policy_to_clkpm_state(struct pcie_link_state
*link
)
150 switch (aspm_policy
) {
151 case POLICY_PERFORMANCE
:
152 /* Disable ASPM and Clock PM */
154 case POLICY_POWERSAVE
:
155 case POLICY_POWER_SUPERSAVE
:
156 /* Enable Clock PM */
159 return link
->clkpm_default
;
164 static void pcie_set_clkpm_nocheck(struct pcie_link_state
*link
, int enable
)
166 struct pci_dev
*child
;
167 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
168 u32 val
= enable
? PCI_EXP_LNKCTL_CLKREQ_EN
: 0;
170 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
171 pcie_capability_clear_and_set_word(child
, PCI_EXP_LNKCTL
,
172 PCI_EXP_LNKCTL_CLKREQ_EN
,
174 link
->clkpm_enabled
= !!enable
;
177 static void pcie_set_clkpm(struct pcie_link_state
*link
, int enable
)
179 /* Don't enable Clock PM if the link is not Clock PM capable */
180 if (!link
->clkpm_capable
)
182 /* Need nothing if the specified equals to current state */
183 if (link
->clkpm_enabled
== enable
)
185 pcie_set_clkpm_nocheck(link
, enable
);
188 static void pcie_clkpm_cap_init(struct pcie_link_state
*link
, int blacklist
)
190 int capable
= 1, enabled
= 1;
193 struct pci_dev
*child
;
194 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
196 /* All functions should have the same cap and state, take the worst */
197 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
198 pcie_capability_read_dword(child
, PCI_EXP_LNKCAP
, ®32
);
199 if (!(reg32
& PCI_EXP_LNKCAP_CLKPM
)) {
204 pcie_capability_read_word(child
, PCI_EXP_LNKCTL
, ®16
);
205 if (!(reg16
& PCI_EXP_LNKCTL_CLKREQ_EN
))
208 link
->clkpm_enabled
= enabled
;
209 link
->clkpm_default
= enabled
;
210 link
->clkpm_capable
= (blacklist
) ? 0 : capable
;
214 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
215 * could use common clock. If they are, configure them to use the
216 * common clock. That will reduce the ASPM state exit latency.
218 static void pcie_aspm_configure_common_clock(struct pcie_link_state
*link
)
221 u16 reg16
, parent_reg
, child_reg
[8];
222 unsigned long start_jiffies
;
223 struct pci_dev
*child
, *parent
= link
->pdev
;
224 struct pci_bus
*linkbus
= parent
->subordinate
;
226 * All functions of a slot should have the same Slot Clock
227 * Configuration, so just check one function
229 child
= list_entry(linkbus
->devices
.next
, struct pci_dev
, bus_list
);
230 BUG_ON(!pci_is_pcie(child
));
232 /* Check downstream component if bit Slot Clock Configuration is 1 */
233 pcie_capability_read_word(child
, PCI_EXP_LNKSTA
, ®16
);
234 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
237 /* Check upstream component if bit Slot Clock Configuration is 1 */
238 pcie_capability_read_word(parent
, PCI_EXP_LNKSTA
, ®16
);
239 if (!(reg16
& PCI_EXP_LNKSTA_SLC
))
242 /* Configure downstream component, all functions */
243 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
244 pcie_capability_read_word(child
, PCI_EXP_LNKCTL
, ®16
);
245 child_reg
[PCI_FUNC(child
->devfn
)] = reg16
;
247 reg16
|= PCI_EXP_LNKCTL_CCC
;
249 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
250 pcie_capability_write_word(child
, PCI_EXP_LNKCTL
, reg16
);
253 /* Configure upstream component */
254 pcie_capability_read_word(parent
, PCI_EXP_LNKCTL
, ®16
);
257 reg16
|= PCI_EXP_LNKCTL_CCC
;
259 reg16
&= ~PCI_EXP_LNKCTL_CCC
;
260 pcie_capability_write_word(parent
, PCI_EXP_LNKCTL
, reg16
);
263 reg16
|= PCI_EXP_LNKCTL_RL
;
264 pcie_capability_write_word(parent
, PCI_EXP_LNKCTL
, reg16
);
266 /* Wait for link training end. Break out after waiting for timeout */
267 start_jiffies
= jiffies
;
269 pcie_capability_read_word(parent
, PCI_EXP_LNKSTA
, ®16
);
270 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
272 if (time_after(jiffies
, start_jiffies
+ LINK_RETRAIN_TIMEOUT
))
276 if (!(reg16
& PCI_EXP_LNKSTA_LT
))
279 /* Training failed. Restore common clock configurations */
280 dev_err(&parent
->dev
, "ASPM: Could not configure common clock\n");
281 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
282 pcie_capability_write_word(child
, PCI_EXP_LNKCTL
,
283 child_reg
[PCI_FUNC(child
->devfn
)]);
284 pcie_capability_write_word(parent
, PCI_EXP_LNKCTL
, parent_reg
);
287 /* Convert L0s latency encoding to ns */
288 static u32
calc_l0s_latency(u32 encoding
)
291 return (5 * 1000); /* > 4us */
292 return (64 << encoding
);
295 /* Convert L0s acceptable latency encoding to ns */
296 static u32
calc_l0s_acceptable(u32 encoding
)
300 return (64 << encoding
);
303 /* Convert L1 latency encoding to ns */
304 static u32
calc_l1_latency(u32 encoding
)
307 return (65 * 1000); /* > 64us */
308 return (1000 << encoding
);
311 /* Convert L1 acceptable latency encoding to ns */
312 static u32
calc_l1_acceptable(u32 encoding
)
316 return (1000 << encoding
);
319 /* Convert L1SS T_pwr encoding to usec */
320 static u32
calc_l1ss_pwron(struct pci_dev
*pdev
, u32 scale
, u32 val
)
330 dev_err(&pdev
->dev
, "%s: Invalid T_PwrOn scale: %u\n",
335 struct aspm_register_info
{
338 u32 latency_encoding_l0s
;
339 u32 latency_encoding_l1
;
348 static void pcie_get_aspm_reg(struct pci_dev
*pdev
,
349 struct aspm_register_info
*info
)
354 pcie_capability_read_dword(pdev
, PCI_EXP_LNKCAP
, ®32
);
355 info
->support
= (reg32
& PCI_EXP_LNKCAP_ASPMS
) >> 10;
356 info
->latency_encoding_l0s
= (reg32
& PCI_EXP_LNKCAP_L0SEL
) >> 12;
357 info
->latency_encoding_l1
= (reg32
& PCI_EXP_LNKCAP_L1EL
) >> 15;
358 pcie_capability_read_word(pdev
, PCI_EXP_LNKCTL
, ®16
);
359 info
->enabled
= reg16
& PCI_EXP_LNKCTL_ASPMC
;
361 /* Read L1 PM substate capabilities */
362 info
->l1ss_cap
= info
->l1ss_ctl1
= info
->l1ss_ctl2
= 0;
363 info
->l1ss_cap_ptr
= pci_find_ext_capability(pdev
, PCI_EXT_CAP_ID_L1SS
);
364 if (!info
->l1ss_cap_ptr
)
366 pci_read_config_dword(pdev
, info
->l1ss_cap_ptr
+ PCI_L1SS_CAP
,
368 if (!(info
->l1ss_cap
& PCI_L1SS_CAP_L1_PM_SS
)) {
372 pci_read_config_dword(pdev
, info
->l1ss_cap_ptr
+ PCI_L1SS_CTL1
,
374 pci_read_config_dword(pdev
, info
->l1ss_cap_ptr
+ PCI_L1SS_CTL2
,
378 static void pcie_aspm_check_latency(struct pci_dev
*endpoint
)
380 u32 latency
, l1_switch_latency
= 0;
381 struct aspm_latency
*acceptable
;
382 struct pcie_link_state
*link
;
384 /* Device not in D0 doesn't need latency check */
385 if ((endpoint
->current_state
!= PCI_D0
) &&
386 (endpoint
->current_state
!= PCI_UNKNOWN
))
389 link
= endpoint
->bus
->self
->link_state
;
390 acceptable
= &link
->acceptable
[PCI_FUNC(endpoint
->devfn
)];
393 /* Check upstream direction L0s latency */
394 if ((link
->aspm_capable
& ASPM_STATE_L0S_UP
) &&
395 (link
->latency_up
.l0s
> acceptable
->l0s
))
396 link
->aspm_capable
&= ~ASPM_STATE_L0S_UP
;
398 /* Check downstream direction L0s latency */
399 if ((link
->aspm_capable
& ASPM_STATE_L0S_DW
) &&
400 (link
->latency_dw
.l0s
> acceptable
->l0s
))
401 link
->aspm_capable
&= ~ASPM_STATE_L0S_DW
;
404 * Every switch on the path to root complex need 1
405 * more microsecond for L1. Spec doesn't mention L0s.
407 * The exit latencies for L1 substates are not advertised
408 * by a device. Since the spec also doesn't mention a way
409 * to determine max latencies introduced by enabling L1
410 * substates on the components, it is not clear how to do
411 * a L1 substate exit latency check. We assume that the
412 * L1 exit latencies advertised by a device include L1
413 * substate latencies (and hence do not do any check).
415 latency
= max_t(u32
, link
->latency_up
.l1
, link
->latency_dw
.l1
);
416 if ((link
->aspm_capable
& ASPM_STATE_L1
) &&
417 (latency
+ l1_switch_latency
> acceptable
->l1
))
418 link
->aspm_capable
&= ~ASPM_STATE_L1
;
419 l1_switch_latency
+= 1000;
426 * The L1 PM substate capability is only implemented in function 0 in a
427 * multi function device.
429 static struct pci_dev
*pci_function_0(struct pci_bus
*linkbus
)
431 struct pci_dev
*child
;
433 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
434 if (PCI_FUNC(child
->devfn
) == 0)
439 /* Calculate L1.2 PM substate timing parameters */
440 static void aspm_calc_l1ss_info(struct pcie_link_state
*link
,
441 struct aspm_register_info
*upreg
,
442 struct aspm_register_info
*dwreg
)
444 u32 val1
, val2
, scale1
, scale2
;
446 link
->l1ss
.up_cap_ptr
= upreg
->l1ss_cap_ptr
;
447 link
->l1ss
.dw_cap_ptr
= dwreg
->l1ss_cap_ptr
;
448 link
->l1ss
.ctl1
= link
->l1ss
.ctl2
= 0;
450 if (!(link
->aspm_support
& ASPM_STATE_L1_2_MASK
))
453 /* Choose the greater of the two T_cmn_mode_rstr_time */
454 val1
= (upreg
->l1ss_cap
>> 8) & 0xFF;
455 val2
= (upreg
->l1ss_cap
>> 8) & 0xFF;
457 link
->l1ss
.ctl1
|= val1
<< 8;
459 link
->l1ss
.ctl1
|= val2
<< 8;
461 * We currently use LTR L1.2 threshold to be fixed constant picked from
464 link
->l1ss
.ctl1
|= LTR_L1_2_THRESHOLD_BITS
;
466 /* Choose the greater of the two T_pwr_on */
467 val1
= (upreg
->l1ss_cap
>> 19) & 0x1F;
468 scale1
= (upreg
->l1ss_cap
>> 16) & 0x03;
469 val2
= (dwreg
->l1ss_cap
>> 19) & 0x1F;
470 scale2
= (dwreg
->l1ss_cap
>> 16) & 0x03;
472 if (calc_l1ss_pwron(link
->pdev
, scale1
, val1
) >
473 calc_l1ss_pwron(link
->downstream
, scale2
, val2
))
474 link
->l1ss
.ctl2
|= scale1
| (val1
<< 3);
476 link
->l1ss
.ctl2
|= scale2
| (val2
<< 3);
479 static void pcie_aspm_cap_init(struct pcie_link_state
*link
, int blacklist
)
481 struct pci_dev
*child
= link
->downstream
, *parent
= link
->pdev
;
482 struct pci_bus
*linkbus
= parent
->subordinate
;
483 struct aspm_register_info upreg
, dwreg
;
486 /* Set enabled/disable so that we will disable ASPM later */
487 link
->aspm_enabled
= ASPM_STATE_ALL
;
488 link
->aspm_disable
= ASPM_STATE_ALL
;
492 /* Get upstream/downstream components' register state */
493 pcie_get_aspm_reg(parent
, &upreg
);
494 pcie_get_aspm_reg(child
, &dwreg
);
497 * If ASPM not supported, don't mess with the clocks and link,
500 if (!(upreg
.support
& dwreg
.support
))
503 /* Configure common clock before checking latencies */
504 pcie_aspm_configure_common_clock(link
);
507 * Re-read upstream/downstream components' register state
508 * after clock configuration
510 pcie_get_aspm_reg(parent
, &upreg
);
511 pcie_get_aspm_reg(child
, &dwreg
);
516 * Note that we must not enable L0s in either direction on a
517 * given link unless components on both sides of the link each
520 if (dwreg
.support
& upreg
.support
& PCIE_LINK_STATE_L0S
)
521 link
->aspm_support
|= ASPM_STATE_L0S
;
522 if (dwreg
.enabled
& PCIE_LINK_STATE_L0S
)
523 link
->aspm_enabled
|= ASPM_STATE_L0S_UP
;
524 if (upreg
.enabled
& PCIE_LINK_STATE_L0S
)
525 link
->aspm_enabled
|= ASPM_STATE_L0S_DW
;
526 link
->latency_up
.l0s
= calc_l0s_latency(upreg
.latency_encoding_l0s
);
527 link
->latency_dw
.l0s
= calc_l0s_latency(dwreg
.latency_encoding_l0s
);
530 if (upreg
.support
& dwreg
.support
& PCIE_LINK_STATE_L1
)
531 link
->aspm_support
|= ASPM_STATE_L1
;
532 if (upreg
.enabled
& dwreg
.enabled
& PCIE_LINK_STATE_L1
)
533 link
->aspm_enabled
|= ASPM_STATE_L1
;
534 link
->latency_up
.l1
= calc_l1_latency(upreg
.latency_encoding_l1
);
535 link
->latency_dw
.l1
= calc_l1_latency(dwreg
.latency_encoding_l1
);
537 /* Setup L1 substate */
538 if (upreg
.l1ss_cap
& dwreg
.l1ss_cap
& PCI_L1SS_CAP_ASPM_L1_1
)
539 link
->aspm_support
|= ASPM_STATE_L1_1
;
540 if (upreg
.l1ss_cap
& dwreg
.l1ss_cap
& PCI_L1SS_CAP_ASPM_L1_2
)
541 link
->aspm_support
|= ASPM_STATE_L1_2
;
542 if (upreg
.l1ss_cap
& dwreg
.l1ss_cap
& PCI_L1SS_CAP_PCIPM_L1_1
)
543 link
->aspm_support
|= ASPM_STATE_L1_1_PCIPM
;
544 if (upreg
.l1ss_cap
& dwreg
.l1ss_cap
& PCI_L1SS_CAP_PCIPM_L1_2
)
545 link
->aspm_support
|= ASPM_STATE_L1_2_PCIPM
;
547 if (upreg
.l1ss_ctl1
& dwreg
.l1ss_ctl1
& PCI_L1SS_CTL1_ASPM_L1_1
)
548 link
->aspm_enabled
|= ASPM_STATE_L1_1
;
549 if (upreg
.l1ss_ctl1
& dwreg
.l1ss_ctl1
& PCI_L1SS_CTL1_ASPM_L1_2
)
550 link
->aspm_enabled
|= ASPM_STATE_L1_2
;
551 if (upreg
.l1ss_ctl1
& dwreg
.l1ss_ctl1
& PCI_L1SS_CTL1_PCIPM_L1_1
)
552 link
->aspm_enabled
|= ASPM_STATE_L1_1_PCIPM
;
553 if (upreg
.l1ss_ctl1
& dwreg
.l1ss_ctl1
& PCI_L1SS_CTL1_PCIPM_L1_2
)
554 link
->aspm_enabled
|= ASPM_STATE_L1_2_PCIPM
;
556 if (link
->aspm_support
& ASPM_STATE_L1SS
)
557 aspm_calc_l1ss_info(link
, &upreg
, &dwreg
);
559 /* Save default state */
560 link
->aspm_default
= link
->aspm_enabled
;
562 /* Setup initial capable state. Will be updated later */
563 link
->aspm_capable
= link
->aspm_support
;
565 * If the downstream component has pci bridge function, don't
568 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
569 if (pci_pcie_type(child
) == PCI_EXP_TYPE_PCI_BRIDGE
) {
570 link
->aspm_disable
= ASPM_STATE_ALL
;
575 /* Get and check endpoint acceptable latencies */
576 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
578 struct aspm_latency
*acceptable
=
579 &link
->acceptable
[PCI_FUNC(child
->devfn
)];
581 if (pci_pcie_type(child
) != PCI_EXP_TYPE_ENDPOINT
&&
582 pci_pcie_type(child
) != PCI_EXP_TYPE_LEG_END
)
585 pcie_capability_read_dword(child
, PCI_EXP_DEVCAP
, ®32
);
586 /* Calculate endpoint L0s acceptable latency */
587 encoding
= (reg32
& PCI_EXP_DEVCAP_L0S
) >> 6;
588 acceptable
->l0s
= calc_l0s_acceptable(encoding
);
589 /* Calculate endpoint L1 acceptable latency */
590 encoding
= (reg32
& PCI_EXP_DEVCAP_L1
) >> 9;
591 acceptable
->l1
= calc_l1_acceptable(encoding
);
593 pcie_aspm_check_latency(child
);
597 static void pci_clear_and_set_dword(struct pci_dev
*pdev
, int pos
,
602 pci_read_config_dword(pdev
, pos
, &val
);
605 pci_write_config_dword(pdev
, pos
, val
);
608 /* Configure the ASPM L1 substates */
609 static void pcie_config_aspm_l1ss(struct pcie_link_state
*link
, u32 state
)
612 struct pci_dev
*child
= link
->downstream
, *parent
= link
->pdev
;
613 u32 up_cap_ptr
= link
->l1ss
.up_cap_ptr
;
614 u32 dw_cap_ptr
= link
->l1ss
.dw_cap_ptr
;
616 enable_req
= (link
->aspm_enabled
^ state
) & state
;
619 * Here are the rules specified in the PCIe spec for enabling L1SS:
620 * - When enabling L1.x, enable bit at parent first, then at child
621 * - When disabling L1.x, disable bit at child first, then at parent
622 * - When enabling ASPM L1.x, need to disable L1
623 * (at child followed by parent).
624 * - The ASPM/PCIPM L1.2 must be disabled while programming timing
627 * To keep it simple, disable all L1SS bits first, and later enable
631 /* Disable all L1 substates */
632 pci_clear_and_set_dword(child
, dw_cap_ptr
+ PCI_L1SS_CTL1
,
633 PCI_L1SS_CTL1_L1SS_MASK
, 0);
634 pci_clear_and_set_dword(parent
, up_cap_ptr
+ PCI_L1SS_CTL1
,
635 PCI_L1SS_CTL1_L1SS_MASK
, 0);
637 * If needed, disable L1, and it gets enabled later
638 * in pcie_config_aspm_link().
640 if (enable_req
& (ASPM_STATE_L1_1
| ASPM_STATE_L1_2
)) {
641 pcie_capability_clear_and_set_word(child
, PCI_EXP_LNKCTL
,
642 PCI_EXP_LNKCTL_ASPM_L1
, 0);
643 pcie_capability_clear_and_set_word(parent
, PCI_EXP_LNKCTL
,
644 PCI_EXP_LNKCTL_ASPM_L1
, 0);
647 if (enable_req
& ASPM_STATE_L1_2_MASK
) {
649 /* Program T_pwr_on in both ports */
650 pci_write_config_dword(parent
, up_cap_ptr
+ PCI_L1SS_CTL2
,
652 pci_write_config_dword(child
, dw_cap_ptr
+ PCI_L1SS_CTL2
,
655 /* Program T_cmn_mode in parent */
656 pci_clear_and_set_dword(parent
, up_cap_ptr
+ PCI_L1SS_CTL1
,
657 0xFF00, link
->l1ss
.ctl1
);
659 /* Program LTR L1.2 threshold in both ports */
660 pci_clear_and_set_dword(parent
, dw_cap_ptr
+ PCI_L1SS_CTL1
,
661 0xE3FF0000, link
->l1ss
.ctl1
);
662 pci_clear_and_set_dword(child
, dw_cap_ptr
+ PCI_L1SS_CTL1
,
663 0xE3FF0000, link
->l1ss
.ctl1
);
667 if (state
& ASPM_STATE_L1_1
)
668 val
|= PCI_L1SS_CTL1_ASPM_L1_1
;
669 if (state
& ASPM_STATE_L1_2
)
670 val
|= PCI_L1SS_CTL1_ASPM_L1_2
;
671 if (state
& ASPM_STATE_L1_1_PCIPM
)
672 val
|= PCI_L1SS_CTL1_PCIPM_L1_1
;
673 if (state
& ASPM_STATE_L1_2_PCIPM
)
674 val
|= PCI_L1SS_CTL1_PCIPM_L1_2
;
676 /* Enable what we need to enable */
677 pci_clear_and_set_dword(parent
, up_cap_ptr
+ PCI_L1SS_CTL1
,
678 PCI_L1SS_CAP_L1_PM_SS
, val
);
679 pci_clear_and_set_dword(child
, dw_cap_ptr
+ PCI_L1SS_CTL1
,
680 PCI_L1SS_CAP_L1_PM_SS
, val
);
683 static void pcie_config_aspm_dev(struct pci_dev
*pdev
, u32 val
)
685 pcie_capability_clear_and_set_word(pdev
, PCI_EXP_LNKCTL
,
686 PCI_EXP_LNKCTL_ASPMC
, val
);
689 static void pcie_config_aspm_link(struct pcie_link_state
*link
, u32 state
)
691 u32 upstream
= 0, dwstream
= 0;
692 struct pci_dev
*child
= link
->downstream
, *parent
= link
->pdev
;
693 struct pci_bus
*linkbus
= parent
->subordinate
;
695 /* Enable only the states that were not explicitly disabled */
696 state
&= (link
->aspm_capable
& ~link
->aspm_disable
);
698 /* Can't enable any substates if L1 is not enabled */
699 if (!(state
& ASPM_STATE_L1
))
700 state
&= ~ASPM_STATE_L1SS
;
702 /* Spec says both ports must be in D0 before enabling PCI PM substates*/
703 if (parent
->current_state
!= PCI_D0
|| child
->current_state
!= PCI_D0
) {
704 state
&= ~ASPM_STATE_L1_SS_PCIPM
;
705 state
|= (link
->aspm_enabled
& ASPM_STATE_L1_SS_PCIPM
);
708 /* Nothing to do if the link is already in the requested state */
709 if (link
->aspm_enabled
== state
)
711 /* Convert ASPM state to upstream/downstream ASPM register state */
712 if (state
& ASPM_STATE_L0S_UP
)
713 dwstream
|= PCI_EXP_LNKCTL_ASPM_L0S
;
714 if (state
& ASPM_STATE_L0S_DW
)
715 upstream
|= PCI_EXP_LNKCTL_ASPM_L0S
;
716 if (state
& ASPM_STATE_L1
) {
717 upstream
|= PCI_EXP_LNKCTL_ASPM_L1
;
718 dwstream
|= PCI_EXP_LNKCTL_ASPM_L1
;
721 if (link
->aspm_capable
& ASPM_STATE_L1SS
)
722 pcie_config_aspm_l1ss(link
, state
);
725 * Spec 2.0 suggests all functions should be configured the
726 * same setting for ASPM. Enabling ASPM L1 should be done in
727 * upstream component first and then downstream, and vice
728 * versa for disabling ASPM L1. Spec doesn't mention L0S.
730 if (state
& ASPM_STATE_L1
)
731 pcie_config_aspm_dev(parent
, upstream
);
732 list_for_each_entry(child
, &linkbus
->devices
, bus_list
)
733 pcie_config_aspm_dev(child
, dwstream
);
734 if (!(state
& ASPM_STATE_L1
))
735 pcie_config_aspm_dev(parent
, upstream
);
737 link
->aspm_enabled
= state
;
740 static void pcie_config_aspm_path(struct pcie_link_state
*link
)
743 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
748 static void free_link_state(struct pcie_link_state
*link
)
750 link
->pdev
->link_state
= NULL
;
754 static int pcie_aspm_sanity_check(struct pci_dev
*pdev
)
756 struct pci_dev
*child
;
760 * Some functions in a slot might not all be PCIe functions,
761 * very strange. Disable ASPM for the whole slot
763 list_for_each_entry(child
, &pdev
->subordinate
->devices
, bus_list
) {
764 if (!pci_is_pcie(child
))
768 * If ASPM is disabled then we're not going to change
769 * the BIOS state. It's safe to continue even if it's a
777 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
778 * RBER bit to determine if a function is 1.1 version device
780 pcie_capability_read_dword(child
, PCI_EXP_DEVCAP
, ®32
);
781 if (!(reg32
& PCI_EXP_DEVCAP_RBER
) && !aspm_force
) {
782 dev_info(&child
->dev
, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
789 static struct pcie_link_state
*alloc_pcie_link_state(struct pci_dev
*pdev
)
791 struct pcie_link_state
*link
;
793 link
= kzalloc(sizeof(*link
), GFP_KERNEL
);
797 INIT_LIST_HEAD(&link
->sibling
);
798 INIT_LIST_HEAD(&link
->children
);
799 INIT_LIST_HEAD(&link
->link
);
801 link
->downstream
= pci_function_0(pdev
->subordinate
);
804 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
807 if (pci_pcie_type(pdev
) == PCI_EXP_TYPE_ROOT_PORT
||
808 pci_pcie_type(pdev
) == PCI_EXP_TYPE_PCIE_BRIDGE
) {
811 struct pcie_link_state
*parent
;
813 parent
= pdev
->bus
->parent
->self
->link_state
;
819 link
->parent
= parent
;
820 link
->root
= link
->parent
->root
;
821 list_add(&link
->link
, &parent
->children
);
824 list_add(&link
->sibling
, &link_list
);
825 pdev
->link_state
= link
;
830 * pcie_aspm_init_link_state: Initiate PCI express link state.
831 * It is called after the pcie and its children devices are scanned.
832 * @pdev: the root port or switch downstream port
834 void pcie_aspm_init_link_state(struct pci_dev
*pdev
)
836 struct pcie_link_state
*link
;
837 int blacklist
= !!pcie_aspm_sanity_check(pdev
);
839 if (!aspm_support_enabled
)
842 if (pdev
->link_state
)
846 * We allocate pcie_link_state for the component on the upstream
847 * end of a Link, so there's nothing to do unless this device has a
848 * Link on its secondary side.
850 if (!pdev
->has_secondary_link
)
853 /* VIA has a strange chipset, root port is under a bridge */
854 if (pci_pcie_type(pdev
) == PCI_EXP_TYPE_ROOT_PORT
&&
858 down_read(&pci_bus_sem
);
859 if (list_empty(&pdev
->subordinate
->devices
))
862 mutex_lock(&aspm_lock
);
863 link
= alloc_pcie_link_state(pdev
);
867 * Setup initial ASPM state. Note that we need to configure
868 * upstream links also because capable state of them can be
869 * update through pcie_aspm_cap_init().
871 pcie_aspm_cap_init(link
, blacklist
);
873 /* Setup initial Clock PM state */
874 pcie_clkpm_cap_init(link
, blacklist
);
877 * At this stage drivers haven't had an opportunity to change the
878 * link policy setting. Enabling ASPM on broken hardware can cripple
879 * it even before the driver has had a chance to disable ASPM, so
880 * default to a safe level right now. If we're enabling ASPM beyond
881 * the BIOS's expectation, we'll do so once pci_enable_device() is
884 if (aspm_policy
!= POLICY_POWERSAVE
&&
885 aspm_policy
!= POLICY_POWER_SUPERSAVE
) {
886 pcie_config_aspm_path(link
);
887 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
891 mutex_unlock(&aspm_lock
);
893 up_read(&pci_bus_sem
);
896 /* Recheck latencies and update aspm_capable for links under the root */
897 static void pcie_update_aspm_capable(struct pcie_link_state
*root
)
899 struct pcie_link_state
*link
;
900 BUG_ON(root
->parent
);
901 list_for_each_entry(link
, &link_list
, sibling
) {
902 if (link
->root
!= root
)
904 link
->aspm_capable
= link
->aspm_support
;
906 list_for_each_entry(link
, &link_list
, sibling
) {
907 struct pci_dev
*child
;
908 struct pci_bus
*linkbus
= link
->pdev
->subordinate
;
909 if (link
->root
!= root
)
911 list_for_each_entry(child
, &linkbus
->devices
, bus_list
) {
912 if ((pci_pcie_type(child
) != PCI_EXP_TYPE_ENDPOINT
) &&
913 (pci_pcie_type(child
) != PCI_EXP_TYPE_LEG_END
))
915 pcie_aspm_check_latency(child
);
920 /* @pdev: the endpoint device */
921 void pcie_aspm_exit_link_state(struct pci_dev
*pdev
)
923 struct pci_dev
*parent
= pdev
->bus
->self
;
924 struct pcie_link_state
*link
, *root
, *parent_link
;
926 if (!parent
|| !parent
->link_state
)
929 down_read(&pci_bus_sem
);
930 mutex_lock(&aspm_lock
);
932 * All PCIe functions are in one slot, remove one function will remove
933 * the whole slot, so just wait until we are the last function left.
935 if (!list_is_last(&pdev
->bus_list
, &parent
->subordinate
->devices
))
938 link
= parent
->link_state
;
940 parent_link
= link
->parent
;
942 /* All functions are removed, so just disable ASPM for the link */
943 pcie_config_aspm_link(link
, 0);
944 list_del(&link
->sibling
);
945 list_del(&link
->link
);
946 /* Clock PM is for endpoint device */
947 free_link_state(link
);
949 /* Recheck latencies and configure upstream links */
951 pcie_update_aspm_capable(root
);
952 pcie_config_aspm_path(parent_link
);
955 mutex_unlock(&aspm_lock
);
956 up_read(&pci_bus_sem
);
959 /* @pdev: the root port or switch downstream port */
960 void pcie_aspm_pm_state_change(struct pci_dev
*pdev
)
962 struct pcie_link_state
*link
= pdev
->link_state
;
964 if (aspm_disabled
|| !link
)
967 * Devices changed PM state, we should recheck if latency
968 * meets all functions' requirement
970 down_read(&pci_bus_sem
);
971 mutex_lock(&aspm_lock
);
972 pcie_update_aspm_capable(link
->root
);
973 pcie_config_aspm_path(link
);
974 mutex_unlock(&aspm_lock
);
975 up_read(&pci_bus_sem
);
978 void pcie_aspm_powersave_config_link(struct pci_dev
*pdev
)
980 struct pcie_link_state
*link
= pdev
->link_state
;
982 if (aspm_disabled
|| !link
)
985 if (aspm_policy
!= POLICY_POWERSAVE
&&
986 aspm_policy
!= POLICY_POWER_SUPERSAVE
)
989 down_read(&pci_bus_sem
);
990 mutex_lock(&aspm_lock
);
991 pcie_config_aspm_path(link
);
992 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
993 mutex_unlock(&aspm_lock
);
994 up_read(&pci_bus_sem
);
997 static void __pci_disable_link_state(struct pci_dev
*pdev
, int state
, bool sem
)
999 struct pci_dev
*parent
= pdev
->bus
->self
;
1000 struct pcie_link_state
*link
;
1002 if (!pci_is_pcie(pdev
))
1005 if (pdev
->has_secondary_link
)
1007 if (!parent
|| !parent
->link_state
)
1011 * A driver requested that ASPM be disabled on this device, but
1012 * if we don't have permission to manage ASPM (e.g., on ACPI
1013 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1014 * the _OSC method), we can't honor that request. Windows has
1015 * a similar mechanism using "PciASPMOptOut", which is also
1016 * ignored in this situation.
1018 if (aspm_disabled
) {
1019 dev_warn(&pdev
->dev
, "can't disable ASPM; OS doesn't have ASPM control\n");
1024 down_read(&pci_bus_sem
);
1025 mutex_lock(&aspm_lock
);
1026 link
= parent
->link_state
;
1027 if (state
& PCIE_LINK_STATE_L0S
)
1028 link
->aspm_disable
|= ASPM_STATE_L0S
;
1029 if (state
& PCIE_LINK_STATE_L1
)
1030 link
->aspm_disable
|= ASPM_STATE_L1
;
1031 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
1033 if (state
& PCIE_LINK_STATE_CLKPM
) {
1034 link
->clkpm_capable
= 0;
1035 pcie_set_clkpm(link
, 0);
1037 mutex_unlock(&aspm_lock
);
1039 up_read(&pci_bus_sem
);
1042 void pci_disable_link_state_locked(struct pci_dev
*pdev
, int state
)
1044 __pci_disable_link_state(pdev
, state
, false);
1046 EXPORT_SYMBOL(pci_disable_link_state_locked
);
1049 * pci_disable_link_state - Disable device's link state, so the link will
1050 * never enter specific states. Note that if the BIOS didn't grant ASPM
1051 * control to the OS, this does nothing because we can't touch the LNKCTL
1055 * @state: ASPM link state to disable
1057 void pci_disable_link_state(struct pci_dev
*pdev
, int state
)
1059 __pci_disable_link_state(pdev
, state
, true);
1061 EXPORT_SYMBOL(pci_disable_link_state
);
1063 static int pcie_aspm_set_policy(const char *val
, struct kernel_param
*kp
)
1066 struct pcie_link_state
*link
;
1070 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
1071 if (!strncmp(val
, policy_str
[i
], strlen(policy_str
[i
])))
1073 if (i
>= ARRAY_SIZE(policy_str
))
1075 if (i
== aspm_policy
)
1078 down_read(&pci_bus_sem
);
1079 mutex_lock(&aspm_lock
);
1081 list_for_each_entry(link
, &link_list
, sibling
) {
1082 pcie_config_aspm_link(link
, policy_to_aspm_state(link
));
1083 pcie_set_clkpm(link
, policy_to_clkpm_state(link
));
1085 mutex_unlock(&aspm_lock
);
1086 up_read(&pci_bus_sem
);
1090 static int pcie_aspm_get_policy(char *buffer
, struct kernel_param
*kp
)
1093 for (i
= 0; i
< ARRAY_SIZE(policy_str
); i
++)
1094 if (i
== aspm_policy
)
1095 cnt
+= sprintf(buffer
+ cnt
, "[%s] ", policy_str
[i
]);
1097 cnt
+= sprintf(buffer
+ cnt
, "%s ", policy_str
[i
]);
1101 module_param_call(policy
, pcie_aspm_set_policy
, pcie_aspm_get_policy
,
1104 #ifdef CONFIG_PCIEASPM_DEBUG
1105 static ssize_t
link_state_show(struct device
*dev
,
1106 struct device_attribute
*attr
,
1109 struct pci_dev
*pci_device
= to_pci_dev(dev
);
1110 struct pcie_link_state
*link_state
= pci_device
->link_state
;
1112 return sprintf(buf
, "%d\n", link_state
->aspm_enabled
);
1115 static ssize_t
link_state_store(struct device
*dev
,
1116 struct device_attribute
*attr
,
1120 struct pci_dev
*pdev
= to_pci_dev(dev
);
1121 struct pcie_link_state
*link
, *root
= pdev
->link_state
->root
;
1127 if (kstrtouint(buf
, 10, &state
))
1129 if ((state
& ~ASPM_STATE_ALL
) != 0)
1132 down_read(&pci_bus_sem
);
1133 mutex_lock(&aspm_lock
);
1134 list_for_each_entry(link
, &link_list
, sibling
) {
1135 if (link
->root
!= root
)
1137 pcie_config_aspm_link(link
, state
);
1139 mutex_unlock(&aspm_lock
);
1140 up_read(&pci_bus_sem
);
1144 static ssize_t
clk_ctl_show(struct device
*dev
,
1145 struct device_attribute
*attr
,
1148 struct pci_dev
*pci_device
= to_pci_dev(dev
);
1149 struct pcie_link_state
*link_state
= pci_device
->link_state
;
1151 return sprintf(buf
, "%d\n", link_state
->clkpm_enabled
);
1154 static ssize_t
clk_ctl_store(struct device
*dev
,
1155 struct device_attribute
*attr
,
1159 struct pci_dev
*pdev
= to_pci_dev(dev
);
1162 if (strtobool(buf
, &state
))
1165 down_read(&pci_bus_sem
);
1166 mutex_lock(&aspm_lock
);
1167 pcie_set_clkpm_nocheck(pdev
->link_state
, state
);
1168 mutex_unlock(&aspm_lock
);
1169 up_read(&pci_bus_sem
);
1174 static DEVICE_ATTR_RW(link_state
);
1175 static DEVICE_ATTR_RW(clk_ctl
);
1177 static char power_group
[] = "power";
1178 void pcie_aspm_create_sysfs_dev_files(struct pci_dev
*pdev
)
1180 struct pcie_link_state
*link_state
= pdev
->link_state
;
1185 if (link_state
->aspm_support
)
1186 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
1187 &dev_attr_link_state
.attr
, power_group
);
1188 if (link_state
->clkpm_capable
)
1189 sysfs_add_file_to_group(&pdev
->dev
.kobj
,
1190 &dev_attr_clk_ctl
.attr
, power_group
);
1193 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev
*pdev
)
1195 struct pcie_link_state
*link_state
= pdev
->link_state
;
1200 if (link_state
->aspm_support
)
1201 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
1202 &dev_attr_link_state
.attr
, power_group
);
1203 if (link_state
->clkpm_capable
)
1204 sysfs_remove_file_from_group(&pdev
->dev
.kobj
,
1205 &dev_attr_clk_ctl
.attr
, power_group
);
1209 static int __init
pcie_aspm_disable(char *str
)
1211 if (!strcmp(str
, "off")) {
1212 aspm_policy
= POLICY_DEFAULT
;
1214 aspm_support_enabled
= false;
1215 printk(KERN_INFO
"PCIe ASPM is disabled\n");
1216 } else if (!strcmp(str
, "force")) {
1218 printk(KERN_INFO
"PCIe ASPM is forcibly enabled\n");
1223 __setup("pcie_aspm=", pcie_aspm_disable
);
1225 void pcie_no_aspm(void)
1228 * Disabling ASPM is intended to prevent the kernel from modifying
1229 * existing hardware state, not to clear existing state. To that end:
1230 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
1231 * (b) prevent userspace from changing policy
1234 aspm_policy
= POLICY_DEFAULT
;
1239 bool pcie_aspm_support_enabled(void)
1241 return aspm_support_enabled
;
1243 EXPORT_SYMBOL(pcie_aspm_support_enabled
);