2 * Copyright(c) 2015-2017 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 extern uint aspm_mode;
55 ASPM_MODE_DISABLED = 0, /* ASPM always disabled, performance mode */
56 ASPM_MODE_ENABLED = 1, /* ASPM always enabled, power saving mode */
57 ASPM_MODE_DYNAMIC = 2, /* ASPM enabled/disabled dynamically */
60 /* Time after which the timer interrupt will re-enable ASPM */
61 #define ASPM_TIMER_MS 1000
62 /* Time for which interrupts are ignored after a timer has been scheduled */
63 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
64 /* Two interrupts within this time trigger ASPM disable */
65 #define ASPM_TRIGGER_MS 1
66 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
67 #define ASPM_L1_SUPPORTED(reg) \
68 (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
70 static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
72 struct pci_dev *parent = dd->pcidev->bus->self;
76 * If the driver does not have access to the upstream component,
77 * it cannot support ASPM L1 at all.
82 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
83 dn = ASPM_L1_SUPPORTED(dn);
85 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
86 up = ASPM_L1_SUPPORTED(up);
88 /* ASPM works on A-step but is reported as not supported */
89 return (!!dn || is_ax(dd)) && !!up;
92 /* Set L1 entrance latency for slower entry to L1 */
93 static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
95 u32 l1_ent_lat = 0x4u;
98 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
99 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
100 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
101 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
104 static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd)
106 struct pci_dev *parent = dd->pcidev->bus->self;
109 * If the driver does not have access to the upstream component,
110 * it cannot support ASPM L1 at all.
115 /* Enable ASPM L1 first in upstream component and then downstream */
116 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
117 PCI_EXP_LNKCTL_ASPMC,
118 PCI_EXP_LNKCTL_ASPM_L1);
119 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
120 PCI_EXP_LNKCTL_ASPMC,
121 PCI_EXP_LNKCTL_ASPM_L1);
124 static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd)
126 struct pci_dev *parent = dd->pcidev->bus->self;
128 /* Disable ASPM L1 first in downstream component and then upstream */
129 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
130 PCI_EXP_LNKCTL_ASPMC, 0x0);
132 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
133 PCI_EXP_LNKCTL_ASPMC, 0x0);
136 static inline void aspm_enable(struct hfi1_devdata *dd)
138 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
142 aspm_hw_enable_l1(dd);
143 dd->aspm_enabled = true;
146 static inline void aspm_disable(struct hfi1_devdata *dd)
148 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
151 aspm_hw_disable_l1(dd);
152 dd->aspm_enabled = false;
155 static inline void aspm_disable_inc(struct hfi1_devdata *dd)
159 spin_lock_irqsave(&dd->aspm_lock, flags);
161 atomic_inc(&dd->aspm_disabled_cnt);
162 spin_unlock_irqrestore(&dd->aspm_lock, flags);
165 static inline void aspm_enable_dec(struct hfi1_devdata *dd)
169 spin_lock_irqsave(&dd->aspm_lock, flags);
170 if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
172 spin_unlock_irqrestore(&dd->aspm_lock, flags);
175 /* ASPM processing for each receive context interrupt */
176 static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
179 bool close_interrupts;
183 /* Quickest exit for minimum impact */
184 if (!rcd->aspm_intr_supported)
187 spin_lock_irqsave(&rcd->aspm_lock, flags);
188 /* PSM contexts are open */
189 if (!rcd->aspm_intr_enable)
192 prev = rcd->aspm_ts_last_intr;
194 rcd->aspm_ts_last_intr = now;
196 /* An interrupt pair close together in time */
197 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
199 /* Don't push out our timer till this much time has elapsed */
200 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
201 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
202 restart_timer = restart_timer && close_interrupts;
204 /* Disable ASPM and schedule timer */
205 if (rcd->aspm_enabled && close_interrupts) {
206 aspm_disable_inc(rcd->dd);
207 rcd->aspm_enabled = false;
208 restart_timer = true;
212 mod_timer(&rcd->aspm_timer,
213 jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
214 rcd->aspm_ts_timer_sched = now;
217 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
220 /* Timer function for re-enabling ASPM in the absence of interrupt activity */
221 static inline void aspm_ctx_timer_function(unsigned long data)
223 struct hfi1_ctxtdata *rcd = (struct hfi1_ctxtdata *)data;
226 spin_lock_irqsave(&rcd->aspm_lock, flags);
227 aspm_enable_dec(rcd->dd);
228 rcd->aspm_enabled = true;
229 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
233 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
236 static inline void aspm_disable_all(struct hfi1_devdata *dd)
238 struct hfi1_ctxtdata *rcd;
242 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
243 rcd = hfi1_rcd_get_by_index(dd, i);
245 del_timer_sync(&rcd->aspm_timer);
246 spin_lock_irqsave(&rcd->aspm_lock, flags);
247 rcd->aspm_intr_enable = false;
248 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
254 atomic_set(&dd->aspm_disabled_cnt, 0);
257 /* Re-enable interrupt processing for verbs contexts */
258 static inline void aspm_enable_all(struct hfi1_devdata *dd)
260 struct hfi1_ctxtdata *rcd;
266 if (aspm_mode != ASPM_MODE_DYNAMIC)
269 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
270 rcd = hfi1_rcd_get_by_index(dd, i);
272 spin_lock_irqsave(&rcd->aspm_lock, flags);
273 rcd->aspm_intr_enable = true;
274 rcd->aspm_enabled = true;
275 spin_unlock_irqrestore(&rcd->aspm_lock, flags);
281 static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
283 spin_lock_init(&rcd->aspm_lock);
284 setup_timer(&rcd->aspm_timer, aspm_ctx_timer_function,
286 rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
287 aspm_mode == ASPM_MODE_DYNAMIC &&
288 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
291 static inline void aspm_init(struct hfi1_devdata *dd)
293 struct hfi1_ctxtdata *rcd;
296 spin_lock_init(&dd->aspm_lock);
297 dd->aspm_supported = aspm_hw_l1_supported(dd);
299 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
300 rcd = hfi1_rcd_get_by_index(dd, i);
306 /* Start with ASPM disabled */
307 aspm_hw_set_l1_ent_latency(dd);
308 dd->aspm_enabled = false;
309 aspm_hw_disable_l1(dd);
311 /* Now turn on ASPM if configured */
315 static inline void aspm_exit(struct hfi1_devdata *dd)
317 aspm_disable_all(dd);
319 /* Turn on ASPM on exit to conserve power */