drivers/infiniband/hw/hfi1/aspm.h

   1 /*
   2  * Copyright(c) 2015-2017 Intel Corporation.
   3  *
   4  * This file is provided under a dual BSD/GPLv2 license.  When using or
   5  * redistributing this file, you may do so under either license.
   6  *
   7  * GPL LICENSE SUMMARY
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of version 2 of the GNU General Public License as
  11  * published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  *
  18  * BSD LICENSE
  19  *
  20  * Redistribution and use in source and binary forms, with or without
  21  * modification, are permitted provided that the following conditions
  22  * are met:
  23  *
  24  *  - Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  *  - Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in
  28  *    the documentation and/or other materials provided with the
  29  *    distribution.
  30  *  - Neither the name of Intel Corporation nor the names of its
  31  *    contributors may be used to endorse or promote products derived
  32  *    from this software without specific prior written permission.
  33  *
  34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45  *
  46  */
  47 #ifndef _ASPM_H
  48 #define _ASPM_H
  49
  50 #include "hfi.h"
  51
  52 extern uint aspm_mode;
  53
  54 enum aspm_mode {
  55         ASPM_MODE_DISABLED = 0, /* ASPM always disabled, performance mode */
  56         ASPM_MODE_ENABLED = 1,  /* ASPM always enabled, power saving mode */
  57         ASPM_MODE_DYNAMIC = 2,  /* ASPM enabled/disabled dynamically */
  58 };
  59
  60 /* Time after which the timer interrupt will re-enable ASPM */
  61 #define ASPM_TIMER_MS 1000
  62 /* Time for which interrupts are ignored after a timer has been scheduled */
  63 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
  64 /* Two interrupts within this time trigger ASPM disable */
  65 #define ASPM_TRIGGER_MS 1
  66 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
  67 #define ASPM_L1_SUPPORTED(reg) \
  68         (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
  69
  70 static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
  71 {
  72         struct pci_dev *parent = dd->pcidev->bus->self;
  73         u32 up, dn;
  74
  75         /*
  76          * If the driver does not have access to the upstream component,
  77          * it cannot support ASPM L1 at all.
  78          */
  79         if (!parent)
  80                 return false;
  81
  82         pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
  83         dn = ASPM_L1_SUPPORTED(dn);
  84
  85         pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
  86         up = ASPM_L1_SUPPORTED(up);
  87
  88         /* ASPM works on A-step but is reported as not supported */
  89         return (!!dn || is_ax(dd)) && !!up;
  90 }
  91
  92 /* Set L1 entrance latency for slower entry to L1 */
  93 static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
  94 {
  95         u32 l1_ent_lat = 0x4u;
  96         u32 reg32;
  97
  98         pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, &reg32);
  99         reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
 100         reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
 101         pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
 102 }
 103
 104 static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd)
 105 {
 106         struct pci_dev *parent = dd->pcidev->bus->self;
 107
 108         /*
 109          * If the driver does not have access to the upstream component,
 110          * it cannot support ASPM L1 at all.
 111          */
 112         if (!parent)
 113                 return;
 114
 115         /* Enable ASPM L1 first in upstream component and then downstream */
 116         pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
 117                                            PCI_EXP_LNKCTL_ASPMC,
 118                                            PCI_EXP_LNKCTL_ASPM_L1);
 119         pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
 120                                            PCI_EXP_LNKCTL_ASPMC,
 121                                            PCI_EXP_LNKCTL_ASPM_L1);
 122 }
 123
 124 static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd)
 125 {
 126         struct pci_dev *parent = dd->pcidev->bus->self;
 127
 128         /* Disable ASPM L1 first in downstream component and then upstream */
 129         pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
 130                                            PCI_EXP_LNKCTL_ASPMC, 0x0);
 131         if (parent)
 132                 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
 133                                                    PCI_EXP_LNKCTL_ASPMC, 0x0);
 134 }
 135
 136 static inline void aspm_enable(struct hfi1_devdata *dd)
 137 {
 138         if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
 139             !dd->aspm_supported)
 140                 return;
 141
 142         aspm_hw_enable_l1(dd);
 143         dd->aspm_enabled = true;
 144 }
 145
 146 static inline void aspm_disable(struct hfi1_devdata *dd)
 147 {
 148         if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
 149                 return;
 150
 151         aspm_hw_disable_l1(dd);
 152         dd->aspm_enabled = false;
 153 }
 154
 155 static inline void aspm_disable_inc(struct hfi1_devdata *dd)
 156 {
 157         unsigned long flags;
 158
 159         spin_lock_irqsave(&dd->aspm_lock, flags);
 160         aspm_disable(dd);
 161         atomic_inc(&dd->aspm_disabled_cnt);
 162         spin_unlock_irqrestore(&dd->aspm_lock, flags);
 163 }
 164
 165 static inline void aspm_enable_dec(struct hfi1_devdata *dd)
 166 {
 167         unsigned long flags;
 168
 169         spin_lock_irqsave(&dd->aspm_lock, flags);
 170         if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
 171                 aspm_enable(dd);
 172         spin_unlock_irqrestore(&dd->aspm_lock, flags);
 173 }
 174
 175 /* ASPM processing for each receive context interrupt */
 176 static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
 177 {
 178         bool restart_timer;
 179         bool close_interrupts;
 180         unsigned long flags;
 181         ktime_t now, prev;
 182
 183         /* Quickest exit for minimum impact */
 184         if (!rcd->aspm_intr_supported)
 185                 return;
 186
 187         spin_lock_irqsave(&rcd->aspm_lock, flags);
 188         /* PSM contexts are open */
 189         if (!rcd->aspm_intr_enable)
 190                 goto unlock;
 191
 192         prev = rcd->aspm_ts_last_intr;
 193         now = ktime_get();
 194         rcd->aspm_ts_last_intr = now;
 195
 196         /* An interrupt pair close together in time */
 197         close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
 198
 199         /* Don't push out our timer till this much time has elapsed */
 200         restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
 201                                     ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
 202         restart_timer = restart_timer && close_interrupts;
 203
 204         /* Disable ASPM and schedule timer */
 205         if (rcd->aspm_enabled && close_interrupts) {
 206                 aspm_disable_inc(rcd->dd);
 207                 rcd->aspm_enabled = false;
 208                 restart_timer = true;
 209         }
 210
 211         if (restart_timer) {
 212                 mod_timer(&rcd->aspm_timer,
 213                           jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
 214                 rcd->aspm_ts_timer_sched = now;
 215         }
 216 unlock:
 217         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 218 }
 219
 220 /* Timer function for re-enabling ASPM in the absence of interrupt activity */
 221 static inline void aspm_ctx_timer_function(struct timer_list *t)
 222 {
 223         struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
 224         unsigned long flags;
 225
 226         spin_lock_irqsave(&rcd->aspm_lock, flags);
 227         aspm_enable_dec(rcd->dd);
 228         rcd->aspm_enabled = true;
 229         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 230 }
 231
 232 /*
 233  * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
 234  * are open.
 235  */
 236 static inline void aspm_disable_all(struct hfi1_devdata *dd)
 237 {
 238         struct hfi1_ctxtdata *rcd;
 239         unsigned long flags;
 240         u16 i;
 241
 242         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 243                 rcd = hfi1_rcd_get_by_index(dd, i);
 244                 if (rcd) {
 245                         del_timer_sync(&rcd->aspm_timer);
 246                         spin_lock_irqsave(&rcd->aspm_lock, flags);
 247                         rcd->aspm_intr_enable = false;
 248                         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 249                         hfi1_rcd_put(rcd);
 250                 }
 251         }
 252
 253         aspm_disable(dd);
 254         atomic_set(&dd->aspm_disabled_cnt, 0);
 255 }
 256
 257 /* Re-enable interrupt processing for verbs contexts */
 258 static inline void aspm_enable_all(struct hfi1_devdata *dd)
 259 {
 260         struct hfi1_ctxtdata *rcd;
 261         unsigned long flags;
 262         u16 i;
 263
 264         aspm_enable(dd);
 265
 266         if (aspm_mode != ASPM_MODE_DYNAMIC)
 267                 return;
 268
 269         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 270                 rcd = hfi1_rcd_get_by_index(dd, i);
 271                 if (rcd) {
 272                         spin_lock_irqsave(&rcd->aspm_lock, flags);
 273                         rcd->aspm_intr_enable = true;
 274                         rcd->aspm_enabled = true;
 275                         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 276                         hfi1_rcd_put(rcd);
 277                 }
 278         }
 279 }
 280
 281 static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
 282 {
 283         spin_lock_init(&rcd->aspm_lock);
 284         timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
 285         rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
 286                 aspm_mode == ASPM_MODE_DYNAMIC &&
 287                 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
 288 }
 289
 290 static inline void aspm_init(struct hfi1_devdata *dd)
 291 {
 292         struct hfi1_ctxtdata *rcd;
 293         u16 i;
 294
 295         spin_lock_init(&dd->aspm_lock);
 296         dd->aspm_supported = aspm_hw_l1_supported(dd);
 297
 298         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 299                 rcd = hfi1_rcd_get_by_index(dd, i);
 300                 if (rcd)
 301                         aspm_ctx_init(rcd);
 302                 hfi1_rcd_put(rcd);
 303         }
 304
 305         /* Start with ASPM disabled */
 306         aspm_hw_set_l1_ent_latency(dd);
 307         dd->aspm_enabled = false;
 308         aspm_hw_disable_l1(dd);
 309
 310         /* Now turn on ASPM if configured */
 311         aspm_enable_all(dd);
 312 }
 313
 314 static inline void aspm_exit(struct hfi1_devdata *dd)
 315 {
 316         aspm_disable_all(dd);
 317
 318         /* Turn on ASPM on exit to conserve power */
 319         aspm_enable(dd);
 320 }
 321
 322 #endif /* _ASPM_H */