GNU Linux-libre 4.14.295-gnu1
[releases.git] / drivers / usb / host / xhci.c
1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22
23 #include <linux/pci.h>
24 #include <linux/iopoll.h>
25 #include <linux/irq.h>
26 #include <linux/log2.h>
27 #include <linux/module.h>
28 #include <linux/moduleparam.h>
29 #include <linux/slab.h>
30 #include <linux/dmi.h>
31 #include <linux/dma-mapping.h>
32
33 #include "xhci.h"
34 #include "xhci-trace.h"
35 #include "xhci-mtk.h"
36
37 #define DRIVER_AUTHOR "Sarah Sharp"
38 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
39
40 #define PORT_WAKE_BITS  (PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
41
42 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
43 static int link_quirk;
44 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
45 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
46
47 static unsigned long long quirks;
48 module_param(quirks, ullong, S_IRUGO);
49 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
50
51 static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
52 {
53         struct xhci_segment *seg = ring->first_seg;
54
55         if (!td || !td->start_seg)
56                 return false;
57         do {
58                 if (seg == td->start_seg)
59                         return true;
60                 seg = seg->next;
61         } while (seg && seg != ring->first_seg);
62
63         return false;
64 }
65
66 /*
67  * xhci_handshake - spin reading hc until handshake completes or fails
68  * @ptr: address of hc register to be read
69  * @mask: bits to look at in result of read
70  * @done: value of those bits when handshake succeeds
71  * @usec: timeout in microseconds
72  *
73  * Returns negative errno, or zero on success
74  *
75  * Success happens when the "mask" bits have the specified value (hardware
76  * handshake done).  There are two failure modes:  "usec" have passed (major
77  * hardware flakeout), or the register reads as all-ones (hardware removed).
78  */
79 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, u64 timeout_us)
80 {
81         u32     result;
82         int     ret;
83
84         ret = readl_poll_timeout_atomic(ptr, result,
85                                         (result & mask) == done ||
86                                         result == U32_MAX,
87                                         1, timeout_us);
88         if (result == U32_MAX)          /* card removed */
89                 return -ENODEV;
90
91         return ret;
92 }
93
94 /*
95  * Disable interrupts and begin the xHCI halting process.
96  */
97 void xhci_quiesce(struct xhci_hcd *xhci)
98 {
99         u32 halted;
100         u32 cmd;
101         u32 mask;
102
103         mask = ~(XHCI_IRQS);
104         halted = readl(&xhci->op_regs->status) & STS_HALT;
105         if (!halted)
106                 mask &= ~CMD_RUN;
107
108         cmd = readl(&xhci->op_regs->command);
109         cmd &= mask;
110         writel(cmd, &xhci->op_regs->command);
111 }
112
113 /*
114  * Force HC into halt state.
115  *
116  * Disable any IRQs and clear the run/stop bit.
117  * HC will complete any current and actively pipelined transactions, and
118  * should halt within 16 ms of the run/stop bit being cleared.
119  * Read HC Halted bit in the status register to see when the HC is finished.
120  */
121 int xhci_halt(struct xhci_hcd *xhci)
122 {
123         int ret;
124         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
125         xhci_quiesce(xhci);
126
127         ret = xhci_handshake(&xhci->op_regs->status,
128                         STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
129         if (ret) {
130                 xhci_warn(xhci, "Host halt failed, %d\n", ret);
131                 return ret;
132         }
133         xhci->xhc_state |= XHCI_STATE_HALTED;
134         xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
135         return ret;
136 }
137
138 /*
139  * Set the run bit and wait for the host to be running.
140  */
141 int xhci_start(struct xhci_hcd *xhci)
142 {
143         u32 temp;
144         int ret;
145
146         temp = readl(&xhci->op_regs->command);
147         temp |= (CMD_RUN);
148         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
149                         temp);
150         writel(temp, &xhci->op_regs->command);
151
152         /*
153          * Wait for the HCHalted Status bit to be 0 to indicate the host is
154          * running.
155          */
156         ret = xhci_handshake(&xhci->op_regs->status,
157                         STS_HALT, 0, XHCI_MAX_HALT_USEC);
158         if (ret == -ETIMEDOUT)
159                 xhci_err(xhci, "Host took too long to start, "
160                                 "waited %u microseconds.\n",
161                                 XHCI_MAX_HALT_USEC);
162         if (!ret) {
163                 /* clear state flags. Including dying, halted or removing */
164                 xhci->xhc_state = 0;
165                 xhci->run_graceperiod = jiffies + msecs_to_jiffies(500);
166         }
167
168         return ret;
169 }
170
171 /*
172  * Reset a halted HC.
173  *
174  * This resets pipelines, timers, counters, state machines, etc.
175  * Transactions will be terminated immediately, and operational registers
176  * will be set to their defaults.
177  */
178 int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us)
179 {
180         u32 command;
181         u32 state;
182         int ret, i;
183
184         state = readl(&xhci->op_regs->status);
185
186         if (state == ~(u32)0) {
187                 xhci_warn(xhci, "Host not accessible, reset failed.\n");
188                 return -ENODEV;
189         }
190
191         if ((state & STS_HALT) == 0) {
192                 xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
193                 return 0;
194         }
195
196         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
197         command = readl(&xhci->op_regs->command);
198         command |= CMD_RESET;
199         writel(command, &xhci->op_regs->command);
200
201         /* Existing Intel xHCI controllers require a delay of 1 mS,
202          * after setting the CMD_RESET bit, and before accessing any
203          * HC registers. This allows the HC to complete the
204          * reset operation and be ready for HC register access.
205          * Without this delay, the subsequent HC register access,
206          * may result in a system hang very rarely.
207          */
208         if (xhci->quirks & XHCI_INTEL_HOST)
209                 udelay(1000);
210
211         ret = xhci_handshake(&xhci->op_regs->command, CMD_RESET, 0, timeout_us);
212         if (ret)
213                 return ret;
214
215         if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
216                 usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
217
218         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
219                          "Wait for controller to be ready for doorbell rings");
220         /*
221          * xHCI cannot write to any doorbells or operational registers other
222          * than status until the "Controller Not Ready" flag is cleared.
223          */
224         ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us);
225
226         for (i = 0; i < 2; i++) {
227                 xhci->bus_state[i].port_c_suspend = 0;
228                 xhci->bus_state[i].suspended_ports = 0;
229                 xhci->bus_state[i].resuming_ports = 0;
230         }
231
232         return ret;
233 }
234
235
236 #ifdef CONFIG_USB_PCI
237 /*
238  * Set up MSI
239  */
240 static int xhci_setup_msi(struct xhci_hcd *xhci)
241 {
242         int ret;
243         /*
244          * TODO:Check with MSI Soc for sysdev
245          */
246         struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
247
248         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
249         if (ret < 0) {
250                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
251                                 "failed to allocate MSI entry");
252                 return ret;
253         }
254
255         ret = request_irq(pdev->irq, xhci_msi_irq,
256                                 0, "xhci_hcd", xhci_to_hcd(xhci));
257         if (ret) {
258                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
259                                 "disable MSI interrupt");
260                 pci_free_irq_vectors(pdev);
261         }
262
263         return ret;
264 }
265
266 /*
267  * Set up MSI-X
268  */
269 static int xhci_setup_msix(struct xhci_hcd *xhci)
270 {
271         int i, ret = 0;
272         struct usb_hcd *hcd = xhci_to_hcd(xhci);
273         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
274
275         /*
276          * calculate number of msi-x vectors supported.
277          * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
278          *   with max number of interrupters based on the xhci HCSPARAMS1.
279          * - num_online_cpus: maximum msi-x vectors per CPUs core.
280          *   Add additional 1 vector to ensure always available interrupt.
281          */
282         xhci->msix_count = min(num_online_cpus() + 1,
283                                 HCS_MAX_INTRS(xhci->hcs_params1));
284
285         ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
286                         PCI_IRQ_MSIX);
287         if (ret < 0) {
288                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
289                                 "Failed to enable MSI-X");
290                 return ret;
291         }
292
293         for (i = 0; i < xhci->msix_count; i++) {
294                 ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
295                                 "xhci_hcd", xhci_to_hcd(xhci));
296                 if (ret)
297                         goto disable_msix;
298         }
299
300         hcd->msix_enabled = 1;
301         return ret;
302
303 disable_msix:
304         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
305         while (--i >= 0)
306                 free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
307         pci_free_irq_vectors(pdev);
308         return ret;
309 }
310
311 /* Free any IRQs and disable MSI-X */
312 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
313 {
314         struct usb_hcd *hcd = xhci_to_hcd(xhci);
315         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
316
317         if (xhci->quirks & XHCI_PLAT)
318                 return;
319
320         /* return if using legacy interrupt */
321         if (hcd->irq > 0)
322                 return;
323
324         if (hcd->msix_enabled) {
325                 int i;
326
327                 for (i = 0; i < xhci->msix_count; i++)
328                         free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
329         } else {
330                 free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
331         }
332
333         pci_free_irq_vectors(pdev);
334         hcd->msix_enabled = 0;
335 }
336
337 static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
338 {
339         struct usb_hcd *hcd = xhci_to_hcd(xhci);
340
341         if (hcd->msix_enabled) {
342                 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
343                 int i;
344
345                 for (i = 0; i < xhci->msix_count; i++)
346                         synchronize_irq(pci_irq_vector(pdev, i));
347         }
348 }
349
350 static int xhci_try_enable_msi(struct usb_hcd *hcd)
351 {
352         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
353         struct pci_dev  *pdev;
354         int ret;
355
356         /* The xhci platform device has set up IRQs through usb_add_hcd. */
357         if (xhci->quirks & XHCI_PLAT)
358                 return 0;
359
360         pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
361         /*
362          * Some Fresco Logic host controllers advertise MSI, but fail to
363          * generate interrupts.  Don't even try to enable MSI.
364          */
365         if (xhci->quirks & XHCI_BROKEN_MSI)
366                 goto legacy_irq;
367
368         /* unregister the legacy interrupt */
369         if (hcd->irq)
370                 free_irq(hcd->irq, hcd);
371         hcd->irq = 0;
372
373         ret = xhci_setup_msix(xhci);
374         if (ret)
375                 /* fall back to msi*/
376                 ret = xhci_setup_msi(xhci);
377
378         if (!ret) {
379                 hcd->msi_enabled = 1;
380                 return 0;
381         }
382
383         if (!pdev->irq) {
384                 xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
385                 return -EINVAL;
386         }
387
388  legacy_irq:
389         if (!strlen(hcd->irq_descr))
390                 snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
391                          hcd->driver->description, hcd->self.busnum);
392
393         /* fall back to legacy interrupt*/
394         ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
395                         hcd->irq_descr, hcd);
396         if (ret) {
397                 xhci_err(xhci, "request interrupt %d failed\n",
398                                 pdev->irq);
399                 return ret;
400         }
401         hcd->irq = pdev->irq;
402         return 0;
403 }
404
405 #else
406
407 static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
408 {
409         return 0;
410 }
411
412 static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
413 {
414 }
415
416 static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
417 {
418 }
419
420 #endif
421
422 static void compliance_mode_recovery(unsigned long arg)
423 {
424         struct xhci_hcd *xhci;
425         struct usb_hcd *hcd;
426         u32 temp;
427         int i;
428
429         xhci = (struct xhci_hcd *)arg;
430
431         for (i = 0; i < xhci->num_usb3_ports; i++) {
432                 temp = readl(xhci->usb3_ports[i]);
433                 if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
434                         /*
435                          * Compliance Mode Detected. Letting USB Core
436                          * handle the Warm Reset
437                          */
438                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
439                                         "Compliance mode detected->port %d",
440                                         i + 1);
441                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
442                                         "Attempting compliance mode recovery");
443                         hcd = xhci->shared_hcd;
444
445                         if (hcd->state == HC_STATE_SUSPENDED)
446                                 usb_hcd_resume_root_hub(hcd);
447
448                         usb_hcd_poll_rh_status(hcd);
449                 }
450         }
451
452         if (xhci->port_status_u0 != ((1 << xhci->num_usb3_ports)-1))
453                 mod_timer(&xhci->comp_mode_recovery_timer,
454                         jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
455 }
456
457 /*
458  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
459  * that causes ports behind that hardware to enter compliance mode sometimes.
460  * The quirk creates a timer that polls every 2 seconds the link state of
461  * each host controller's port and recovers it by issuing a Warm reset
462  * if Compliance mode is detected, otherwise the port will become "dead" (no
463  * device connections or disconnections will be detected anymore). Becasue no
464  * status event is generated when entering compliance mode (per xhci spec),
465  * this quirk is needed on systems that have the failing hardware installed.
466  */
467 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
468 {
469         xhci->port_status_u0 = 0;
470         setup_timer(&xhci->comp_mode_recovery_timer,
471                     compliance_mode_recovery, (unsigned long)xhci);
472         xhci->comp_mode_recovery_timer.expires = jiffies +
473                         msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
474
475         add_timer(&xhci->comp_mode_recovery_timer);
476         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
477                         "Compliance mode recovery timer initialized");
478 }
479
480 /*
481  * This function identifies the systems that have installed the SN65LVPE502CP
482  * USB3.0 re-driver and that need the Compliance Mode Quirk.
483  * Systems:
484  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
485  */
486 static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
487 {
488         const char *dmi_product_name, *dmi_sys_vendor;
489
490         dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
491         dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
492         if (!dmi_product_name || !dmi_sys_vendor)
493                 return false;
494
495         if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
496                 return false;
497
498         if (strstr(dmi_product_name, "Z420") ||
499                         strstr(dmi_product_name, "Z620") ||
500                         strstr(dmi_product_name, "Z820") ||
501                         strstr(dmi_product_name, "Z1 Workstation"))
502                 return true;
503
504         return false;
505 }
506
507 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
508 {
509         return (xhci->port_status_u0 == ((1 << xhci->num_usb3_ports)-1));
510 }
511
512
513 /*
514  * Initialize memory for HCD and xHC (one-time init).
515  *
516  * Program the PAGESIZE register, initialize the device context array, create
517  * device contexts (?), set up a command ring segment (or two?), create event
518  * ring (one for now).
519  */
520 static int xhci_init(struct usb_hcd *hcd)
521 {
522         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
523         int retval = 0;
524
525         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
526         spin_lock_init(&xhci->lock);
527         if (xhci->hci_version == 0x95 && link_quirk) {
528                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
529                                 "QUIRK: Not clearing Link TRB chain bits.");
530                 xhci->quirks |= XHCI_LINK_TRB_QUIRK;
531         } else {
532                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
533                                 "xHCI doesn't need link TRB QUIRK");
534         }
535         retval = xhci_mem_init(xhci, GFP_KERNEL);
536         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
537
538         /* Initializing Compliance Mode Recovery Data If Needed */
539         if (xhci_compliance_mode_recovery_timer_quirk_check()) {
540                 xhci->quirks |= XHCI_COMP_MODE_QUIRK;
541                 compliance_mode_recovery_timer_init(xhci);
542         }
543
544         return retval;
545 }
546
547 /*-------------------------------------------------------------------------*/
548
549
550 static int xhci_run_finished(struct xhci_hcd *xhci)
551 {
552         if (xhci_start(xhci)) {
553                 xhci_halt(xhci);
554                 return -ENODEV;
555         }
556         xhci->shared_hcd->state = HC_STATE_RUNNING;
557         xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
558
559         if (xhci->quirks & XHCI_NEC_HOST)
560                 xhci_ring_cmd_db(xhci);
561
562         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
563                         "Finished xhci_run for USB3 roothub");
564         return 0;
565 }
566
567 /*
568  * Start the HC after it was halted.
569  *
570  * This function is called by the USB core when the HC driver is added.
571  * Its opposite is xhci_stop().
572  *
573  * xhci_init() must be called once before this function can be called.
574  * Reset the HC, enable device slot contexts, program DCBAAP, and
575  * set command ring pointer and event ring pointer.
576  *
577  * Setup MSI-X vectors and enable interrupts.
578  */
579 int xhci_run(struct usb_hcd *hcd)
580 {
581         u32 temp;
582         u64 temp_64;
583         int ret;
584         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
585
586         /* Start the xHCI host controller running only after the USB 2.0 roothub
587          * is setup.
588          */
589
590         hcd->uses_new_polling = 1;
591         if (!usb_hcd_is_primary_hcd(hcd))
592                 return xhci_run_finished(xhci);
593
594         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
595
596         ret = xhci_try_enable_msi(hcd);
597         if (ret)
598                 return ret;
599
600         xhci_dbg_cmd_ptrs(xhci);
601
602         xhci_dbg(xhci, "ERST memory map follows:\n");
603         xhci_dbg_erst(xhci, &xhci->erst);
604         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
605         temp_64 &= ~ERST_PTR_MASK;
606         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
607                         "ERST deq = 64'h%0lx", (long unsigned int) temp_64);
608
609         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
610                         "// Set the interrupt modulation register");
611         temp = readl(&xhci->ir_set->irq_control);
612         temp &= ~ER_IRQ_INTERVAL_MASK;
613         /*
614          * the increment interval is 8 times as much as that defined
615          * in xHCI spec on MTK's controller
616          */
617         temp |= (u32) ((xhci->quirks & XHCI_MTK_HOST) ? 20 : 160);
618         writel(temp, &xhci->ir_set->irq_control);
619
620         /* Set the HCD state before we enable the irqs */
621         temp = readl(&xhci->op_regs->command);
622         temp |= (CMD_EIE);
623         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
624                         "// Enable interrupts, cmd = 0x%x.", temp);
625         writel(temp, &xhci->op_regs->command);
626
627         temp = readl(&xhci->ir_set->irq_pending);
628         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
629                         "// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
630                         xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
631         writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
632         xhci_print_ir_set(xhci, 0);
633
634         if (xhci->quirks & XHCI_NEC_HOST) {
635                 struct xhci_command *command;
636
637                 command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
638                 if (!command)
639                         return -ENOMEM;
640
641                 ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
642                                 TRB_TYPE(TRB_NEC_GET_FW));
643                 if (ret)
644                         xhci_free_command(xhci, command);
645         }
646         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
647                         "Finished xhci_run for USB2 roothub");
648         return 0;
649 }
650 EXPORT_SYMBOL_GPL(xhci_run);
651
652 /*
653  * Stop xHCI driver.
654  *
655  * This function is called by the USB core when the HC driver is removed.
656  * Its opposite is xhci_run().
657  *
658  * Disable device contexts, disable IRQs, and quiesce the HC.
659  * Reset the HC, finish any completed transactions, and cleanup memory.
660  */
661 static void xhci_stop(struct usb_hcd *hcd)
662 {
663         u32 temp;
664         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
665
666         mutex_lock(&xhci->mutex);
667
668         /* Only halt host and free memory after both hcds are removed */
669         if (!usb_hcd_is_primary_hcd(hcd)) {
670                 mutex_unlock(&xhci->mutex);
671                 return;
672         }
673
674         spin_lock_irq(&xhci->lock);
675         xhci->xhc_state |= XHCI_STATE_HALTED;
676         xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
677         xhci_halt(xhci);
678         xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
679         spin_unlock_irq(&xhci->lock);
680
681         xhci_cleanup_msix(xhci);
682
683         /* Deleting Compliance Mode Recovery Timer */
684         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
685                         (!(xhci_all_ports_seen_u0(xhci)))) {
686                 del_timer_sync(&xhci->comp_mode_recovery_timer);
687                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
688                                 "%s: compliance mode recovery timer deleted",
689                                 __func__);
690         }
691
692         if (xhci->quirks & XHCI_AMD_PLL_FIX)
693                 usb_amd_dev_put();
694
695         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
696                         "// Disabling event ring interrupts");
697         temp = readl(&xhci->op_regs->status);
698         writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
699         temp = readl(&xhci->ir_set->irq_pending);
700         writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
701         xhci_print_ir_set(xhci, 0);
702
703         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
704         xhci_mem_cleanup(xhci);
705         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
706                         "xhci_stop completed - status = %x",
707                         readl(&xhci->op_regs->status));
708         mutex_unlock(&xhci->mutex);
709 }
710
711 /*
712  * Shutdown HC (not bus-specific)
713  *
714  * This is called when the machine is rebooting or halting.  We assume that the
715  * machine will be powered off, and the HC's internal state will be reset.
716  * Don't bother to free memory.
717  *
718  * This will only ever be called with the main usb_hcd (the USB3 roothub).
719  */
720 void xhci_shutdown(struct usb_hcd *hcd)
721 {
722         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
723
724         if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
725                 usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
726
727         /* Don't poll the roothubs after shutdown. */
728         xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",
729                         __func__, hcd->self.busnum);
730         clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
731         del_timer_sync(&hcd->rh_timer);
732
733         if (xhci->shared_hcd) {
734                 clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
735                 del_timer_sync(&xhci->shared_hcd->rh_timer);
736         }
737
738         spin_lock_irq(&xhci->lock);
739         xhci_halt(xhci);
740         /* Workaround for spurious wakeups at shutdown with HSW */
741         if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
742                 xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
743         spin_unlock_irq(&xhci->lock);
744
745         xhci_cleanup_msix(xhci);
746
747         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
748                         "xhci_shutdown completed - status = %x",
749                         readl(&xhci->op_regs->status));
750 }
751 EXPORT_SYMBOL_GPL(xhci_shutdown);
752
753 #ifdef CONFIG_PM
754 static void xhci_save_registers(struct xhci_hcd *xhci)
755 {
756         xhci->s3.command = readl(&xhci->op_regs->command);
757         xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
758         xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
759         xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
760         xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
761         xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
762         xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
763         xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
764         xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
765 }
766
767 static void xhci_restore_registers(struct xhci_hcd *xhci)
768 {
769         writel(xhci->s3.command, &xhci->op_regs->command);
770         writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
771         xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
772         writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
773         writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
774         xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
775         xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
776         writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
777         writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
778 }
779
780 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
781 {
782         u64     val_64;
783
784         /* step 2: initialize command ring buffer */
785         val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
786         val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
787                 (xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
788                                       xhci->cmd_ring->dequeue) &
789                  (u64) ~CMD_RING_RSVD_BITS) |
790                 xhci->cmd_ring->cycle_state;
791         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
792                         "// Setting command ring address to 0x%llx",
793                         (long unsigned long) val_64);
794         xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
795 }
796
797 /*
798  * The whole command ring must be cleared to zero when we suspend the host.
799  *
800  * The host doesn't save the command ring pointer in the suspend well, so we
801  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
802  * aligned, because of the reserved bits in the command ring dequeue pointer
803  * register.  Therefore, we can't just set the dequeue pointer back in the
804  * middle of the ring (TRBs are 16-byte aligned).
805  */
806 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
807 {
808         struct xhci_ring *ring;
809         struct xhci_segment *seg;
810
811         ring = xhci->cmd_ring;
812         seg = ring->deq_seg;
813         do {
814                 memset(seg->trbs, 0,
815                         sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
816                 seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
817                         cpu_to_le32(~TRB_CYCLE);
818                 seg = seg->next;
819         } while (seg != ring->deq_seg);
820
821         /* Reset the software enqueue and dequeue pointers */
822         ring->deq_seg = ring->first_seg;
823         ring->dequeue = ring->first_seg->trbs;
824         ring->enq_seg = ring->deq_seg;
825         ring->enqueue = ring->dequeue;
826
827         ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
828         /*
829          * Ring is now zeroed, so the HW should look for change of ownership
830          * when the cycle bit is set to 1.
831          */
832         ring->cycle_state = 1;
833
834         /*
835          * Reset the hardware dequeue pointer.
836          * Yes, this will need to be re-written after resume, but we're paranoid
837          * and want to make sure the hardware doesn't access bogus memory
838          * because, say, the BIOS or an SMI started the host without changing
839          * the command ring pointers.
840          */
841         xhci_set_cmd_ring_deq(xhci);
842 }
843
844 static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
845 {
846         int port_index;
847         __le32 __iomem **port_array;
848         unsigned long flags;
849         u32 t1, t2;
850
851         spin_lock_irqsave(&xhci->lock, flags);
852
853         /* disable usb3 ports Wake bits */
854         port_index = xhci->num_usb3_ports;
855         port_array = xhci->usb3_ports;
856         while (port_index--) {
857                 t1 = readl(port_array[port_index]);
858                 t1 = xhci_port_state_to_neutral(t1);
859                 t2 = t1 & ~PORT_WAKE_BITS;
860                 if (t1 != t2)
861                         writel(t2, port_array[port_index]);
862         }
863
864         /* disable usb2 ports Wake bits */
865         port_index = xhci->num_usb2_ports;
866         port_array = xhci->usb2_ports;
867         while (port_index--) {
868                 t1 = readl(port_array[port_index]);
869                 t1 = xhci_port_state_to_neutral(t1);
870                 t2 = t1 & ~PORT_WAKE_BITS;
871                 if (t1 != t2)
872                         writel(t2, port_array[port_index]);
873         }
874
875         spin_unlock_irqrestore(&xhci->lock, flags);
876 }
877
878 static bool xhci_pending_portevent(struct xhci_hcd *xhci)
879 {
880         __le32 __iomem          **port_array;
881         int                     port_index;
882         u32                     status;
883         u32                     portsc;
884
885         status = readl(&xhci->op_regs->status);
886         if (status & STS_EINT)
887                 return true;
888         /*
889          * Checking STS_EINT is not enough as there is a lag between a change
890          * bit being set and the Port Status Change Event that it generated
891          * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.
892          */
893
894         port_index = xhci->num_usb2_ports;
895         port_array = xhci->usb2_ports;
896         while (port_index--) {
897                 portsc = readl(port_array[port_index]);
898                 if (portsc & PORT_CHANGE_MASK ||
899                     (portsc & PORT_PLS_MASK) == XDEV_RESUME)
900                         return true;
901         }
902         port_index = xhci->num_usb3_ports;
903         port_array = xhci->usb3_ports;
904         while (port_index--) {
905                 portsc = readl(port_array[port_index]);
906                 if (portsc & PORT_CHANGE_MASK ||
907                     (portsc & PORT_PLS_MASK) == XDEV_RESUME)
908                         return true;
909         }
910         return false;
911 }
912
913 /*
914  * Stop HC (not bus-specific)
915  *
916  * This is called when the machine transition into S3/S4 mode.
917  *
918  */
919 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
920 {
921         int                     rc = 0;
922         unsigned int            delay = XHCI_MAX_HALT_USEC * 2;
923         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
924         u32                     command;
925         u32                     res;
926
927         if (!hcd->state)
928                 return 0;
929
930         if (hcd->state != HC_STATE_SUSPENDED ||
931                         xhci->shared_hcd->state != HC_STATE_SUSPENDED)
932                 return -EINVAL;
933
934         /* Clear root port wake on bits if wakeup not allowed. */
935         if (!do_wakeup)
936                 xhci_disable_port_wake_on_bits(xhci);
937
938         /* Don't poll the roothubs on bus suspend. */
939         xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
940         clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
941         del_timer_sync(&hcd->rh_timer);
942         clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
943         del_timer_sync(&xhci->shared_hcd->rh_timer);
944
945         if (xhci->quirks & XHCI_SUSPEND_DELAY)
946                 usleep_range(1000, 1500);
947
948         spin_lock_irq(&xhci->lock);
949         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
950         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
951         /* step 1: stop endpoint */
952         /* skipped assuming that port suspend has done */
953
954         /* step 2: clear Run/Stop bit */
955         command = readl(&xhci->op_regs->command);
956         command &= ~CMD_RUN;
957         writel(command, &xhci->op_regs->command);
958
959         /* Some chips from Fresco Logic need an extraordinary delay */
960         delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
961
962         if (xhci_handshake(&xhci->op_regs->status,
963                       STS_HALT, STS_HALT, delay)) {
964                 xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
965                 spin_unlock_irq(&xhci->lock);
966                 return -ETIMEDOUT;
967         }
968         xhci_clear_command_ring(xhci);
969
970         /* step 3: save registers */
971         xhci_save_registers(xhci);
972
973         /* step 4: set CSS flag */
974         command = readl(&xhci->op_regs->command);
975         command |= CMD_CSS;
976         writel(command, &xhci->op_regs->command);
977         xhci->broken_suspend = 0;
978         if (xhci_handshake(&xhci->op_regs->status,
979                                 STS_SAVE, 0, 20 * 1000)) {
980         /*
981          * AMD SNPS xHC 3.0 occasionally does not clear the
982          * SSS bit of USBSTS and when driver tries to poll
983          * to see if the xHC clears BIT(8) which never happens
984          * and driver assumes that controller is not responding
985          * and times out. To workaround this, its good to check
986          * if SRE and HCE bits are not set (as per xhci
987          * Section 5.4.2) and bypass the timeout.
988          */
989                 res = readl(&xhci->op_regs->status);
990                 if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
991                     (((res & STS_SRE) == 0) &&
992                                 ((res & STS_HCE) == 0))) {
993                         xhci->broken_suspend = 1;
994                 } else {
995                         xhci_warn(xhci, "WARN: xHC save state timeout\n");
996                         spin_unlock_irq(&xhci->lock);
997                         return -ETIMEDOUT;
998                 }
999         }
1000         spin_unlock_irq(&xhci->lock);
1001
1002         /*
1003          * Deleting Compliance Mode Recovery Timer because the xHCI Host
1004          * is about to be suspended.
1005          */
1006         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1007                         (!(xhci_all_ports_seen_u0(xhci)))) {
1008                 del_timer_sync(&xhci->comp_mode_recovery_timer);
1009                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1010                                 "%s: compliance mode recovery timer deleted",
1011                                 __func__);
1012         }
1013
1014         /* step 5: remove core well power */
1015         /* synchronize irq when using MSI-X */
1016         xhci_msix_sync_irqs(xhci);
1017
1018         return rc;
1019 }
1020 EXPORT_SYMBOL_GPL(xhci_suspend);
1021
1022 /*
1023  * start xHC (not bus-specific)
1024  *
1025  * This is called when the machine transition from S3/S4 mode.
1026  *
1027  */
1028 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
1029 {
1030         u32                     command, temp = 0;
1031         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
1032         struct usb_hcd          *secondary_hcd;
1033         int                     retval = 0;
1034         bool                    comp_timer_running = false;
1035         bool                    pending_portevent = false;
1036         bool                    reinit_xhc = false;
1037
1038         if (!hcd->state)
1039                 return 0;
1040
1041         /* Wait a bit if either of the roothubs need to settle from the
1042          * transition into bus suspend.
1043          */
1044         if (time_before(jiffies, xhci->bus_state[0].next_statechange) ||
1045                         time_before(jiffies,
1046                                 xhci->bus_state[1].next_statechange))
1047                 msleep(100);
1048
1049         set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1050         set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1051
1052         spin_lock_irq(&xhci->lock);
1053
1054         if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend)
1055                 reinit_xhc = true;
1056
1057         if (!reinit_xhc) {
1058                 /*
1059                  * Some controllers might lose power during suspend, so wait
1060                  * for controller not ready bit to clear, just as in xHC init.
1061                  */
1062                 retval = xhci_handshake(&xhci->op_regs->status,
1063                                         STS_CNR, 0, 10 * 1000 * 1000);
1064                 if (retval) {
1065                         xhci_warn(xhci, "Controller not ready at resume %d\n",
1066                                   retval);
1067                         spin_unlock_irq(&xhci->lock);
1068                         return retval;
1069                 }
1070                 /* step 1: restore register */
1071                 xhci_restore_registers(xhci);
1072                 /* step 2: initialize command ring buffer */
1073                 xhci_set_cmd_ring_deq(xhci);
1074                 /* step 3: restore state and start state*/
1075                 /* step 3: set CRS flag */
1076                 command = readl(&xhci->op_regs->command);
1077                 command |= CMD_CRS;
1078                 writel(command, &xhci->op_regs->command);
1079                 /*
1080                  * Some controllers take up to 55+ ms to complete the controller
1081                  * restore so setting the timeout to 100ms. Xhci specification
1082                  * doesn't mention any timeout value.
1083                  */
1084                 if (xhci_handshake(&xhci->op_regs->status,
1085                               STS_RESTORE, 0, 100 * 1000)) {
1086                         xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1087                         spin_unlock_irq(&xhci->lock);
1088                         return -ETIMEDOUT;
1089                 }
1090         }
1091
1092         temp = readl(&xhci->op_regs->status);
1093
1094         /* re-initialize the HC on Restore Error, or Host Controller Error */
1095         if (temp & (STS_SRE | STS_HCE)) {
1096                 reinit_xhc = true;
1097                 xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp);
1098         }
1099
1100         if (reinit_xhc) {
1101                 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1102                                 !(xhci_all_ports_seen_u0(xhci))) {
1103                         del_timer_sync(&xhci->comp_mode_recovery_timer);
1104                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1105                                 "Compliance Mode Recovery Timer deleted!");
1106                 }
1107
1108                 /* Let the USB core know _both_ roothubs lost power. */
1109                 usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1110                 usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1111
1112                 xhci_dbg(xhci, "Stop HCD\n");
1113                 xhci_halt(xhci);
1114                 retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
1115                 spin_unlock_irq(&xhci->lock);
1116                 if (retval)
1117                         return retval;
1118                 xhci_cleanup_msix(xhci);
1119
1120                 xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1121                 temp = readl(&xhci->op_regs->status);
1122                 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
1123                 temp = readl(&xhci->ir_set->irq_pending);
1124                 writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
1125                 xhci_print_ir_set(xhci, 0);
1126
1127                 xhci_dbg(xhci, "cleaning up memory\n");
1128                 xhci_mem_cleanup(xhci);
1129                 xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1130                             readl(&xhci->op_regs->status));
1131
1132                 /* USB core calls the PCI reinit and start functions twice:
1133                  * first with the primary HCD, and then with the secondary HCD.
1134                  * If we don't do the same, the host will never be started.
1135                  */
1136                 if (!usb_hcd_is_primary_hcd(hcd))
1137                         secondary_hcd = hcd;
1138                 else
1139                         secondary_hcd = xhci->shared_hcd;
1140
1141                 xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1142                 retval = xhci_init(hcd->primary_hcd);
1143                 if (retval)
1144                         return retval;
1145                 comp_timer_running = true;
1146
1147                 xhci_dbg(xhci, "Start the primary HCD\n");
1148                 retval = xhci_run(hcd->primary_hcd);
1149                 if (!retval) {
1150                         xhci_dbg(xhci, "Start the secondary HCD\n");
1151                         retval = xhci_run(secondary_hcd);
1152                 }
1153                 hcd->state = HC_STATE_SUSPENDED;
1154                 xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1155                 goto done;
1156         }
1157
1158         /* step 4: set Run/Stop bit */
1159         command = readl(&xhci->op_regs->command);
1160         command |= CMD_RUN;
1161         writel(command, &xhci->op_regs->command);
1162         xhci_handshake(&xhci->op_regs->status, STS_HALT,
1163                   0, 250 * 1000);
1164
1165         /* step 5: walk topology and initialize portsc,
1166          * portpmsc and portli
1167          */
1168         /* this is done in bus_resume */
1169
1170         /* step 6: restart each of the previously
1171          * Running endpoints by ringing their doorbells
1172          */
1173
1174         spin_unlock_irq(&xhci->lock);
1175
1176  done:
1177         if (retval == 0) {
1178                 /*
1179                  * Resume roothubs only if there are pending events.
1180                  * USB 3 devices resend U3 LFPS wake after a 100ms delay if
1181                  * the first wake signalling failed, give it that chance.
1182                  */
1183                 pending_portevent = xhci_pending_portevent(xhci);
1184                 if (!pending_portevent) {
1185                         msleep(120);
1186                         pending_portevent = xhci_pending_portevent(xhci);
1187                 }
1188
1189                 if (pending_portevent) {
1190                         usb_hcd_resume_root_hub(xhci->shared_hcd);
1191                         usb_hcd_resume_root_hub(hcd);
1192                 }
1193         }
1194         /*
1195          * If system is subject to the Quirk, Compliance Mode Timer needs to
1196          * be re-initialized Always after a system resume. Ports are subject
1197          * to suffer the Compliance Mode issue again. It doesn't matter if
1198          * ports have entered previously to U0 before system's suspension.
1199          */
1200         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1201                 compliance_mode_recovery_timer_init(xhci);
1202
1203         if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
1204                 usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
1205
1206         /* Re-enable port polling. */
1207         xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1208         set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1209         usb_hcd_poll_rh_status(xhci->shared_hcd);
1210         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1211         usb_hcd_poll_rh_status(hcd);
1212
1213         return retval;
1214 }
1215 EXPORT_SYMBOL_GPL(xhci_resume);
1216 #endif  /* CONFIG_PM */
1217
1218 /*-------------------------------------------------------------------------*/
1219
1220 /**
1221  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1222  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1223  * value to right shift 1 for the bitmask.
1224  *
1225  * Index  = (epnum * 2) + direction - 1,
1226  * where direction = 0 for OUT, 1 for IN.
1227  * For control endpoints, the IN index is used (OUT index is unused), so
1228  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1229  */
1230 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1231 {
1232         unsigned int index;
1233         if (usb_endpoint_xfer_control(desc))
1234                 index = (unsigned int) (usb_endpoint_num(desc)*2);
1235         else
1236                 index = (unsigned int) (usb_endpoint_num(desc)*2) +
1237                         (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1238         return index;
1239 }
1240
1241 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
1242  * address from the XHCI endpoint index.
1243  */
1244 unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1245 {
1246         unsigned int number = DIV_ROUND_UP(ep_index, 2);
1247         unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1248         return direction | number;
1249 }
1250
1251 /* Find the flag for this endpoint (for use in the control context).  Use the
1252  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1253  * bit 1, etc.
1254  */
1255 static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1256 {
1257         return 1 << (xhci_get_endpoint_index(desc) + 1);
1258 }
1259
1260 /* Find the flag for this endpoint (for use in the control context).  Use the
1261  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1262  * bit 1, etc.
1263  */
1264 static unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
1265 {
1266         return 1 << (ep_index + 1);
1267 }
1268
1269 /* Compute the last valid endpoint context index.  Basically, this is the
1270  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1271  * we find the most significant bit set in the added contexts flags.
1272  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1273  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1274  */
1275 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1276 {
1277         return fls(added_ctxs) - 1;
1278 }
1279
1280 /* Returns 1 if the arguments are OK;
1281  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1282  */
1283 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1284                 struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1285                 const char *func) {
1286         struct xhci_hcd *xhci;
1287         struct xhci_virt_device *virt_dev;
1288
1289         if (!hcd || (check_ep && !ep) || !udev) {
1290                 pr_debug("xHCI %s called with invalid args\n", func);
1291                 return -EINVAL;
1292         }
1293         if (!udev->parent) {
1294                 pr_debug("xHCI %s called for root hub\n", func);
1295                 return 0;
1296         }
1297
1298         xhci = hcd_to_xhci(hcd);
1299         if (check_virt_dev) {
1300                 if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1301                         xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1302                                         func);
1303                         return -EINVAL;
1304                 }
1305
1306                 virt_dev = xhci->devs[udev->slot_id];
1307                 if (virt_dev->udev != udev) {
1308                         xhci_dbg(xhci, "xHCI %s called with udev and "
1309                                           "virt_dev does not match\n", func);
1310                         return -EINVAL;
1311                 }
1312         }
1313
1314         if (xhci->xhc_state & XHCI_STATE_HALTED)
1315                 return -ENODEV;
1316
1317         return 1;
1318 }
1319
1320 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1321                 struct usb_device *udev, struct xhci_command *command,
1322                 bool ctx_change, bool must_succeed);
1323
1324 /*
1325  * Full speed devices may have a max packet size greater than 8 bytes, but the
1326  * USB core doesn't know that until it reads the first 8 bytes of the
1327  * descriptor.  If the usb_device's max packet size changes after that point,
1328  * we need to issue an evaluate context command and wait on it.
1329  */
1330 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
1331                 unsigned int ep_index, struct urb *urb, gfp_t mem_flags)
1332 {
1333         struct xhci_container_ctx *out_ctx;
1334         struct xhci_input_control_ctx *ctrl_ctx;
1335         struct xhci_ep_ctx *ep_ctx;
1336         struct xhci_command *command;
1337         int max_packet_size;
1338         int hw_max_packet_size;
1339         int ret = 0;
1340
1341         out_ctx = xhci->devs[slot_id]->out_ctx;
1342         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1343         hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1344         max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1345         if (hw_max_packet_size != max_packet_size) {
1346                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1347                                 "Max Packet Size for ep 0 changed.");
1348                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1349                                 "Max packet size in usb_device = %d",
1350                                 max_packet_size);
1351                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1352                                 "Max packet size in xHCI HW = %d",
1353                                 hw_max_packet_size);
1354                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1355                                 "Issuing evaluate context command.");
1356
1357                 /* Set up the input context flags for the command */
1358                 /* FIXME: This won't work if a non-default control endpoint
1359                  * changes max packet sizes.
1360                  */
1361
1362                 command = xhci_alloc_command(xhci, false, true, mem_flags);
1363                 if (!command)
1364                         return -ENOMEM;
1365
1366                 command->in_ctx = xhci->devs[slot_id]->in_ctx;
1367                 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1368                 if (!ctrl_ctx) {
1369                         xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1370                                         __func__);
1371                         ret = -ENOMEM;
1372                         goto command_cleanup;
1373                 }
1374                 /* Set up the modified control endpoint 0 */
1375                 xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1376                                 xhci->devs[slot_id]->out_ctx, ep_index);
1377
1378                 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
1379                 ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */
1380                 ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1381                 ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1382
1383                 ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1384                 ctrl_ctx->drop_flags = 0;
1385
1386                 ret = xhci_configure_endpoint(xhci, urb->dev, command,
1387                                 true, false);
1388
1389                 /* Clean up the input context for later use by bandwidth
1390                  * functions.
1391                  */
1392                 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1393 command_cleanup:
1394                 kfree(command->completion);
1395                 kfree(command);
1396         }
1397         return ret;
1398 }
1399
1400 /*
1401  * non-error returns are a promise to giveback() the urb later
1402  * we drop ownership so next owner (or urb unlink) can get it
1403  */
1404 static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1405 {
1406         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1407         unsigned long flags;
1408         int ret = 0;
1409         unsigned int slot_id, ep_index, ep_state;
1410         struct urb_priv *urb_priv;
1411         int num_tds;
1412
1413         if (!urb)
1414                 return -EINVAL;
1415         ret = xhci_check_args(hcd, urb->dev, urb->ep,
1416                                         true, true, __func__);
1417         if (ret <= 0)
1418                 return ret ? ret : -EINVAL;
1419
1420         slot_id = urb->dev->slot_id;
1421         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1422
1423         if (!HCD_HW_ACCESSIBLE(hcd)) {
1424                 if (!in_interrupt())
1425                         xhci_dbg(xhci, "urb submitted during PCI suspend\n");
1426                 return -ESHUTDOWN;
1427         }
1428
1429         if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1430                 num_tds = urb->number_of_packets;
1431         else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
1432             urb->transfer_buffer_length > 0 &&
1433             urb->transfer_flags & URB_ZERO_PACKET &&
1434             !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
1435                 num_tds = 2;
1436         else
1437                 num_tds = 1;
1438
1439         urb_priv = kzalloc(sizeof(struct urb_priv) +
1440                            num_tds * sizeof(struct xhci_td), mem_flags);
1441         if (!urb_priv)
1442                 return -ENOMEM;
1443
1444         urb_priv->num_tds = num_tds;
1445         urb_priv->num_tds_done = 0;
1446         urb->hcpriv = urb_priv;
1447
1448         trace_xhci_urb_enqueue(urb);
1449
1450         if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1451                 /* Check to see if the max packet size for the default control
1452                  * endpoint changed during FS device enumeration
1453                  */
1454                 if (urb->dev->speed == USB_SPEED_FULL) {
1455                         ret = xhci_check_maxpacket(xhci, slot_id,
1456                                         ep_index, urb, mem_flags);
1457                         if (ret < 0) {
1458                                 xhci_urb_free_priv(urb_priv);
1459                                 urb->hcpriv = NULL;
1460                                 return ret;
1461                         }
1462                 }
1463         }
1464
1465         spin_lock_irqsave(&xhci->lock, flags);
1466
1467         if (xhci->xhc_state & XHCI_STATE_DYING) {
1468                 xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
1469                          urb->ep->desc.bEndpointAddress, urb);
1470                 ret = -ESHUTDOWN;
1471                 goto free_priv;
1472         }
1473
1474         switch (usb_endpoint_type(&urb->ep->desc)) {
1475
1476         case USB_ENDPOINT_XFER_CONTROL:
1477                 ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1478                                          slot_id, ep_index);
1479                 break;
1480         case USB_ENDPOINT_XFER_BULK:
1481                 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
1482                 if (ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
1483                         xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
1484                                   ep_state);
1485                         ret = -EINVAL;
1486                         break;
1487                 }
1488                 ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1489                                          slot_id, ep_index);
1490                 break;
1491
1492
1493         case USB_ENDPOINT_XFER_INT:
1494                 ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1495                                 slot_id, ep_index);
1496                 break;
1497
1498         case USB_ENDPOINT_XFER_ISOC:
1499                 ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1500                                 slot_id, ep_index);
1501         }
1502
1503         if (ret) {
1504 free_priv:
1505                 xhci_urb_free_priv(urb_priv);
1506                 urb->hcpriv = NULL;
1507         }
1508         spin_unlock_irqrestore(&xhci->lock, flags);
1509         return ret;
1510 }
1511
1512 /*
1513  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1514  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1515  * should pick up where it left off in the TD, unless a Set Transfer Ring
1516  * Dequeue Pointer is issued.
1517  *
1518  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1519  * the ring.  Since the ring is a contiguous structure, they can't be physically
1520  * removed.  Instead, there are two options:
1521  *
1522  *  1) If the HC is in the middle of processing the URB to be canceled, we
1523  *     simply move the ring's dequeue pointer past those TRBs using the Set
1524  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1525  *     when drivers timeout on the last submitted URB and attempt to cancel.
1526  *
1527  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1528  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1529  *     HC will need to invalidate the any TRBs it has cached after the stop
1530  *     endpoint command, as noted in the xHCI 0.95 errata.
1531  *
1532  *  3) The TD may have completed by the time the Stop Endpoint Command
1533  *     completes, so software needs to handle that case too.
1534  *
1535  * This function should protect against the TD enqueueing code ringing the
1536  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1537  * It also needs to account for multiple cancellations on happening at the same
1538  * time for the same endpoint.
1539  *
1540  * Note that this function can be called in any context, or so says
1541  * usb_hcd_unlink_urb()
1542  */
1543 static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1544 {
1545         unsigned long flags;
1546         int ret, i;
1547         u32 temp;
1548         struct xhci_hcd *xhci;
1549         struct urb_priv *urb_priv;
1550         struct xhci_td *td;
1551         unsigned int ep_index;
1552         struct xhci_ring *ep_ring;
1553         struct xhci_virt_ep *ep;
1554         struct xhci_command *command;
1555         struct xhci_virt_device *vdev;
1556
1557         xhci = hcd_to_xhci(hcd);
1558         spin_lock_irqsave(&xhci->lock, flags);
1559
1560         trace_xhci_urb_dequeue(urb);
1561
1562         /* Make sure the URB hasn't completed or been unlinked already */
1563         ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1564         if (ret)
1565                 goto done;
1566
1567         /* give back URB now if we can't queue it for cancel */
1568         vdev = xhci->devs[urb->dev->slot_id];
1569         urb_priv = urb->hcpriv;
1570         if (!vdev || !urb_priv)
1571                 goto err_giveback;
1572
1573         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1574         ep = &vdev->eps[ep_index];
1575         ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1576         if (!ep || !ep_ring)
1577                 goto err_giveback;
1578
1579         /* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */
1580         temp = readl(&xhci->op_regs->status);
1581         if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
1582                 xhci_hc_died(xhci);
1583                 goto done;
1584         }
1585
1586         /*
1587          * check ring is not re-allocated since URB was enqueued. If it is, then
1588          * make sure none of the ring related pointers in this URB private data
1589          * are touched, such as td_list, otherwise we overwrite freed data
1590          */
1591         if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
1592                 xhci_err(xhci, "Canceled URB td not found on endpoint ring");
1593                 for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
1594                         td = &urb_priv->td[i];
1595                         if (!list_empty(&td->cancelled_td_list))
1596                                 list_del_init(&td->cancelled_td_list);
1597                 }
1598                 goto err_giveback;
1599         }
1600
1601         if (xhci->xhc_state & XHCI_STATE_HALTED) {
1602                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1603                                 "HC halted, freeing TD manually.");
1604                 for (i = urb_priv->num_tds_done;
1605                      i < urb_priv->num_tds;
1606                      i++) {
1607                         td = &urb_priv->td[i];
1608                         if (!list_empty(&td->td_list))
1609                                 list_del_init(&td->td_list);
1610                         if (!list_empty(&td->cancelled_td_list))
1611                                 list_del_init(&td->cancelled_td_list);
1612                 }
1613                 goto err_giveback;
1614         }
1615
1616         i = urb_priv->num_tds_done;
1617         if (i < urb_priv->num_tds)
1618                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1619                                 "Cancel URB %p, dev %s, ep 0x%x, "
1620                                 "starting at offset 0x%llx",
1621                                 urb, urb->dev->devpath,
1622                                 urb->ep->desc.bEndpointAddress,
1623                                 (unsigned long long) xhci_trb_virt_to_dma(
1624                                         urb_priv->td[i].start_seg,
1625                                         urb_priv->td[i].first_trb));
1626
1627         for (; i < urb_priv->num_tds; i++) {
1628                 td = &urb_priv->td[i];
1629                 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
1630         }
1631
1632         /* Queue a stop endpoint command, but only if this is
1633          * the first cancellation to be handled.
1634          */
1635         if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {
1636                 command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1637                 if (!command) {
1638                         ret = -ENOMEM;
1639                         goto done;
1640                 }
1641                 ep->ep_state |= EP_STOP_CMD_PENDING;
1642                 ep->stop_cmd_timer.expires = jiffies +
1643                         XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1644                 add_timer(&ep->stop_cmd_timer);
1645                 xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1646                                          ep_index, 0);
1647                 xhci_ring_cmd_db(xhci);
1648         }
1649 done:
1650         spin_unlock_irqrestore(&xhci->lock, flags);
1651         return ret;
1652
1653 err_giveback:
1654         if (urb_priv)
1655                 xhci_urb_free_priv(urb_priv);
1656         usb_hcd_unlink_urb_from_ep(hcd, urb);
1657         spin_unlock_irqrestore(&xhci->lock, flags);
1658         usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1659         return ret;
1660 }
1661
1662 /* Drop an endpoint from a new bandwidth configuration for this device.
1663  * Only one call to this function is allowed per endpoint before
1664  * check_bandwidth() or reset_bandwidth() must be called.
1665  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1666  * add the endpoint to the schedule with possibly new parameters denoted by a
1667  * different endpoint descriptor in usb_host_endpoint.
1668  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1669  * not allowed.
1670  *
1671  * The USB core will not allow URBs to be queued to an endpoint that is being
1672  * disabled, so there's no need for mutual exclusion to protect
1673  * the xhci->devs[slot_id] structure.
1674  */
1675 static int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1676                 struct usb_host_endpoint *ep)
1677 {
1678         struct xhci_hcd *xhci;
1679         struct xhci_container_ctx *in_ctx, *out_ctx;
1680         struct xhci_input_control_ctx *ctrl_ctx;
1681         unsigned int ep_index;
1682         struct xhci_ep_ctx *ep_ctx;
1683         u32 drop_flag;
1684         u32 new_add_flags, new_drop_flags;
1685         int ret;
1686
1687         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1688         if (ret <= 0)
1689                 return ret;
1690         xhci = hcd_to_xhci(hcd);
1691         if (xhci->xhc_state & XHCI_STATE_DYING)
1692                 return -ENODEV;
1693
1694         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1695         drop_flag = xhci_get_endpoint_flag(&ep->desc);
1696         if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1697                 xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1698                                 __func__, drop_flag);
1699                 return 0;
1700         }
1701
1702         in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1703         out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1704         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1705         if (!ctrl_ctx) {
1706                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1707                                 __func__);
1708                 return 0;
1709         }
1710
1711         ep_index = xhci_get_endpoint_index(&ep->desc);
1712         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1713         /* If the HC already knows the endpoint is disabled,
1714          * or the HCD has noted it is disabled, ignore this request
1715          */
1716         if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
1717             le32_to_cpu(ctrl_ctx->drop_flags) &
1718             xhci_get_endpoint_flag(&ep->desc)) {
1719                 /* Do not warn when called after a usb_device_reset */
1720                 if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1721                         xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1722                                   __func__, ep);
1723                 return 0;
1724         }
1725
1726         ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1727         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1728
1729         ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1730         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1731
1732         xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1733
1734         if (xhci->quirks & XHCI_MTK_HOST)
1735                 xhci_mtk_drop_ep_quirk(hcd, udev, ep);
1736
1737         xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1738                         (unsigned int) ep->desc.bEndpointAddress,
1739                         udev->slot_id,
1740                         (unsigned int) new_drop_flags,
1741                         (unsigned int) new_add_flags);
1742         return 0;
1743 }
1744
1745 /* Add an endpoint to a new possible bandwidth configuration for this device.
1746  * Only one call to this function is allowed per endpoint before
1747  * check_bandwidth() or reset_bandwidth() must be called.
1748  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1749  * add the endpoint to the schedule with possibly new parameters denoted by a
1750  * different endpoint descriptor in usb_host_endpoint.
1751  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1752  * not allowed.
1753  *
1754  * The USB core will not allow URBs to be queued to an endpoint until the
1755  * configuration or alt setting is installed in the device, so there's no need
1756  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1757  */
1758 static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1759                 struct usb_host_endpoint *ep)
1760 {
1761         struct xhci_hcd *xhci;
1762         struct xhci_container_ctx *in_ctx;
1763         unsigned int ep_index;
1764         struct xhci_input_control_ctx *ctrl_ctx;
1765         u32 added_ctxs;
1766         u32 new_add_flags, new_drop_flags;
1767         struct xhci_virt_device *virt_dev;
1768         int ret = 0;
1769
1770         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1771         if (ret <= 0) {
1772                 /* So we won't queue a reset ep command for a root hub */
1773                 ep->hcpriv = NULL;
1774                 return ret;
1775         }
1776         xhci = hcd_to_xhci(hcd);
1777         if (xhci->xhc_state & XHCI_STATE_DYING)
1778                 return -ENODEV;
1779
1780         added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1781         if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1782                 /* FIXME when we have to issue an evaluate endpoint command to
1783                  * deal with ep0 max packet size changing once we get the
1784                  * descriptors
1785                  */
1786                 xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1787                                 __func__, added_ctxs);
1788                 return 0;
1789         }
1790
1791         virt_dev = xhci->devs[udev->slot_id];
1792         in_ctx = virt_dev->in_ctx;
1793         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1794         if (!ctrl_ctx) {
1795                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1796                                 __func__);
1797                 return 0;
1798         }
1799
1800         ep_index = xhci_get_endpoint_index(&ep->desc);
1801         /* If this endpoint is already in use, and the upper layers are trying
1802          * to add it again without dropping it, reject the addition.
1803          */
1804         if (virt_dev->eps[ep_index].ring &&
1805                         !(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1806                 xhci_warn(xhci, "Trying to add endpoint 0x%x "
1807                                 "without dropping it.\n",
1808                                 (unsigned int) ep->desc.bEndpointAddress);
1809                 return -EINVAL;
1810         }
1811
1812         /* If the HCD has already noted the endpoint is enabled,
1813          * ignore this request.
1814          */
1815         if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
1816                 xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1817                                 __func__, ep);
1818                 return 0;
1819         }
1820
1821         /*
1822          * Configuration and alternate setting changes must be done in
1823          * process context, not interrupt context (or so documenation
1824          * for usb_set_interface() and usb_set_configuration() claim).
1825          */
1826         if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1827                 dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1828                                 __func__, ep->desc.bEndpointAddress);
1829                 return -ENOMEM;
1830         }
1831
1832         if (xhci->quirks & XHCI_MTK_HOST) {
1833                 ret = xhci_mtk_add_ep_quirk(hcd, udev, ep);
1834                 if (ret < 0) {
1835                         xhci_ring_free(xhci, virt_dev->eps[ep_index].new_ring);
1836                         virt_dev->eps[ep_index].new_ring = NULL;
1837                         return ret;
1838                 }
1839         }
1840
1841         ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
1842         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1843
1844         /* If xhci_endpoint_disable() was called for this endpoint, but the
1845          * xHC hasn't been notified yet through the check_bandwidth() call,
1846          * this re-adds a new state for the endpoint from the new endpoint
1847          * descriptors.  We must drop and re-add this endpoint, so we leave the
1848          * drop flags alone.
1849          */
1850         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1851
1852         /* Store the usb_device pointer for later use */
1853         ep->hcpriv = udev;
1854
1855         xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1856                         (unsigned int) ep->desc.bEndpointAddress,
1857                         udev->slot_id,
1858                         (unsigned int) new_drop_flags,
1859                         (unsigned int) new_add_flags);
1860         return 0;
1861 }
1862
1863 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1864 {
1865         struct xhci_input_control_ctx *ctrl_ctx;
1866         struct xhci_ep_ctx *ep_ctx;
1867         struct xhci_slot_ctx *slot_ctx;
1868         int i;
1869
1870         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1871         if (!ctrl_ctx) {
1872                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1873                                 __func__);
1874                 return;
1875         }
1876
1877         /* When a device's add flag and drop flag are zero, any subsequent
1878          * configure endpoint command will leave that endpoint's state
1879          * untouched.  Make sure we don't leave any old state in the input
1880          * endpoint contexts.
1881          */
1882         ctrl_ctx->drop_flags = 0;
1883         ctrl_ctx->add_flags = 0;
1884         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
1885         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1886         /* Endpoint 0 is always valid */
1887         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
1888         for (i = 1; i < 31; i++) {
1889                 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
1890                 ep_ctx->ep_info = 0;
1891                 ep_ctx->ep_info2 = 0;
1892                 ep_ctx->deq = 0;
1893                 ep_ctx->tx_info = 0;
1894         }
1895 }
1896
1897 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
1898                 struct usb_device *udev, u32 *cmd_status)
1899 {
1900         int ret;
1901
1902         switch (*cmd_status) {
1903         case COMP_COMMAND_ABORTED:
1904         case COMP_COMMAND_RING_STOPPED:
1905                 xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
1906                 ret = -ETIME;
1907                 break;
1908         case COMP_RESOURCE_ERROR:
1909                 dev_warn(&udev->dev,
1910                          "Not enough host controller resources for new device state.\n");
1911                 ret = -ENOMEM;
1912                 /* FIXME: can we allocate more resources for the HC? */
1913                 break;
1914         case COMP_BANDWIDTH_ERROR:
1915         case COMP_SECONDARY_BANDWIDTH_ERROR:
1916                 dev_warn(&udev->dev,
1917                          "Not enough bandwidth for new device state.\n");
1918                 ret = -ENOSPC;
1919                 /* FIXME: can we go back to the old state? */
1920                 break;
1921         case COMP_TRB_ERROR:
1922                 /* the HCD set up something wrong */
1923                 dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
1924                                 "add flag = 1, "
1925                                 "and endpoint is not disabled.\n");
1926                 ret = -EINVAL;
1927                 break;
1928         case COMP_INCOMPATIBLE_DEVICE_ERROR:
1929                 dev_warn(&udev->dev,
1930                          "ERROR: Incompatible device for endpoint configure command.\n");
1931                 ret = -ENODEV;
1932                 break;
1933         case COMP_SUCCESS:
1934                 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1935                                 "Successful Endpoint Configure command");
1936                 ret = 0;
1937                 break;
1938         default:
1939                 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
1940                                 *cmd_status);
1941                 ret = -EINVAL;
1942                 break;
1943         }
1944         return ret;
1945 }
1946
1947 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
1948                 struct usb_device *udev, u32 *cmd_status)
1949 {
1950         int ret;
1951
1952         switch (*cmd_status) {
1953         case COMP_COMMAND_ABORTED:
1954         case COMP_COMMAND_RING_STOPPED:
1955                 xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
1956                 ret = -ETIME;
1957                 break;
1958         case COMP_PARAMETER_ERROR:
1959                 dev_warn(&udev->dev,
1960                          "WARN: xHCI driver setup invalid evaluate context command.\n");
1961                 ret = -EINVAL;
1962                 break;
1963         case COMP_SLOT_NOT_ENABLED_ERROR:
1964                 dev_warn(&udev->dev,
1965                         "WARN: slot not enabled for evaluate context command.\n");
1966                 ret = -EINVAL;
1967                 break;
1968         case COMP_CONTEXT_STATE_ERROR:
1969                 dev_warn(&udev->dev,
1970                         "WARN: invalid context state for evaluate context command.\n");
1971                 ret = -EINVAL;
1972                 break;
1973         case COMP_INCOMPATIBLE_DEVICE_ERROR:
1974                 dev_warn(&udev->dev,
1975                         "ERROR: Incompatible device for evaluate context command.\n");
1976                 ret = -ENODEV;
1977                 break;
1978         case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
1979                 /* Max Exit Latency too large error */
1980                 dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
1981                 ret = -EINVAL;
1982                 break;
1983         case COMP_SUCCESS:
1984                 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1985                                 "Successful evaluate context command");
1986                 ret = 0;
1987                 break;
1988         default:
1989                 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
1990                         *cmd_status);
1991                 ret = -EINVAL;
1992                 break;
1993         }
1994         return ret;
1995 }
1996
1997 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
1998                 struct xhci_input_control_ctx *ctrl_ctx)
1999 {
2000         u32 valid_add_flags;
2001         u32 valid_drop_flags;
2002
2003         /* Ignore the slot flag (bit 0), and the default control endpoint flag
2004          * (bit 1).  The default control endpoint is added during the Address
2005          * Device command and is never removed until the slot is disabled.
2006          */
2007         valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2008         valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2009
2010         /* Use hweight32 to count the number of ones in the add flags, or
2011          * number of endpoints added.  Don't count endpoints that are changed
2012          * (both added and dropped).
2013          */
2014         return hweight32(valid_add_flags) -
2015                 hweight32(valid_add_flags & valid_drop_flags);
2016 }
2017
2018 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
2019                 struct xhci_input_control_ctx *ctrl_ctx)
2020 {
2021         u32 valid_add_flags;
2022         u32 valid_drop_flags;
2023
2024         valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2025         valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2026
2027         return hweight32(valid_drop_flags) -
2028                 hweight32(valid_add_flags & valid_drop_flags);
2029 }
2030
2031 /*
2032  * We need to reserve the new number of endpoints before the configure endpoint
2033  * command completes.  We can't subtract the dropped endpoints from the number
2034  * of active endpoints until the command completes because we can oversubscribe
2035  * the host in this case:
2036  *
2037  *  - the first configure endpoint command drops more endpoints than it adds
2038  *  - a second configure endpoint command that adds more endpoints is queued
2039  *  - the first configure endpoint command fails, so the config is unchanged
2040  *  - the second command may succeed, even though there isn't enough resources
2041  *
2042  * Must be called with xhci->lock held.
2043  */
2044 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
2045                 struct xhci_input_control_ctx *ctrl_ctx)
2046 {
2047         u32 added_eps;
2048
2049         added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2050         if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
2051                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2052                                 "Not enough ep ctxs: "
2053                                 "%u active, need to add %u, limit is %u.",
2054                                 xhci->num_active_eps, added_eps,
2055                                 xhci->limit_active_eps);
2056                 return -ENOMEM;
2057         }
2058         xhci->num_active_eps += added_eps;
2059         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2060                         "Adding %u ep ctxs, %u now active.", added_eps,
2061                         xhci->num_active_eps);
2062         return 0;
2063 }
2064
2065 /*
2066  * The configure endpoint was failed by the xHC for some other reason, so we
2067  * need to revert the resources that failed configuration would have used.
2068  *
2069  * Must be called with xhci->lock held.
2070  */
2071 static void xhci_free_host_resources(struct xhci_hcd *xhci,
2072                 struct xhci_input_control_ctx *ctrl_ctx)
2073 {
2074         u32 num_failed_eps;
2075
2076         num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2077         xhci->num_active_eps -= num_failed_eps;
2078         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2079                         "Removing %u failed ep ctxs, %u now active.",
2080                         num_failed_eps,
2081                         xhci->num_active_eps);
2082 }
2083
2084 /*
2085  * Now that the command has completed, clean up the active endpoint count by
2086  * subtracting out the endpoints that were dropped (but not changed).
2087  *
2088  * Must be called with xhci->lock held.
2089  */
2090 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2091                 struct xhci_input_control_ctx *ctrl_ctx)
2092 {
2093         u32 num_dropped_eps;
2094
2095         num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2096         xhci->num_active_eps -= num_dropped_eps;
2097         if (num_dropped_eps)
2098                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2099                                 "Removing %u dropped ep ctxs, %u now active.",
2100                                 num_dropped_eps,
2101                                 xhci->num_active_eps);
2102 }
2103
2104 static unsigned int xhci_get_block_size(struct usb_device *udev)
2105 {
2106         switch (udev->speed) {
2107         case USB_SPEED_LOW:
2108         case USB_SPEED_FULL:
2109                 return FS_BLOCK;
2110         case USB_SPEED_HIGH:
2111                 return HS_BLOCK;
2112         case USB_SPEED_SUPER:
2113         case USB_SPEED_SUPER_PLUS:
2114                 return SS_BLOCK;
2115         case USB_SPEED_UNKNOWN:
2116         case USB_SPEED_WIRELESS:
2117         default:
2118                 /* Should never happen */
2119                 return 1;
2120         }
2121 }
2122
2123 static unsigned int
2124 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2125 {
2126         if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2127                 return LS_OVERHEAD;
2128         if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2129                 return FS_OVERHEAD;
2130         return HS_OVERHEAD;
2131 }
2132
2133 /* If we are changing a LS/FS device under a HS hub,
2134  * make sure (if we are activating a new TT) that the HS bus has enough
2135  * bandwidth for this new TT.
2136  */
2137 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2138                 struct xhci_virt_device *virt_dev,
2139                 int old_active_eps)
2140 {
2141         struct xhci_interval_bw_table *bw_table;
2142         struct xhci_tt_bw_info *tt_info;
2143
2144         /* Find the bandwidth table for the root port this TT is attached to. */
2145         bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
2146         tt_info = virt_dev->tt_info;
2147         /* If this TT already had active endpoints, the bandwidth for this TT
2148          * has already been added.  Removing all periodic endpoints (and thus
2149          * making the TT enactive) will only decrease the bandwidth used.
2150          */
2151         if (old_active_eps)
2152                 return 0;
2153         if (old_active_eps == 0 && tt_info->active_eps != 0) {
2154                 if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2155                         return -ENOMEM;
2156                 return 0;
2157         }
2158         /* Not sure why we would have no new active endpoints...
2159          *
2160          * Maybe because of an Evaluate Context change for a hub update or a
2161          * control endpoint 0 max packet size change?
2162          * FIXME: skip the bandwidth calculation in that case.
2163          */
2164         return 0;
2165 }
2166
2167 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2168                 struct xhci_virt_device *virt_dev)
2169 {
2170         unsigned int bw_reserved;
2171
2172         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2173         if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2174                 return -ENOMEM;
2175
2176         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2177         if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2178                 return -ENOMEM;
2179
2180         return 0;
2181 }
2182
2183 /*
2184  * This algorithm is a very conservative estimate of the worst-case scheduling
2185  * scenario for any one interval.  The hardware dynamically schedules the
2186  * packets, so we can't tell which microframe could be the limiting factor in
2187  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2188  *
2189  * Obviously, we can't solve an NP complete problem to find the minimum worst
2190  * case scenario.  Instead, we come up with an estimate that is no less than
2191  * the worst case bandwidth used for any one microframe, but may be an
2192  * over-estimate.
2193  *
2194  * We walk the requirements for each endpoint by interval, starting with the
2195  * smallest interval, and place packets in the schedule where there is only one
2196  * possible way to schedule packets for that interval.  In order to simplify
2197  * this algorithm, we record the largest max packet size for each interval, and
2198  * assume all packets will be that size.
2199  *
2200  * For interval 0, we obviously must schedule all packets for each interval.
2201  * The bandwidth for interval 0 is just the amount of data to be transmitted
2202  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2203  * the number of packets).
2204  *
2205  * For interval 1, we have two possible microframes to schedule those packets
2206  * in.  For this algorithm, if we can schedule the same number of packets for
2207  * each possible scheduling opportunity (each microframe), we will do so.  The
2208  * remaining number of packets will be saved to be transmitted in the gaps in
2209  * the next interval's scheduling sequence.
2210  *
2211  * As we move those remaining packets to be scheduled with interval 2 packets,
2212  * we have to double the number of remaining packets to transmit.  This is
2213  * because the intervals are actually powers of 2, and we would be transmitting
2214  * the previous interval's packets twice in this interval.  We also have to be
2215  * sure that when we look at the largest max packet size for this interval, we
2216  * also look at the largest max packet size for the remaining packets and take
2217  * the greater of the two.
2218  *
2219  * The algorithm continues to evenly distribute packets in each scheduling
2220  * opportunity, and push the remaining packets out, until we get to the last
2221  * interval.  Then those packets and their associated overhead are just added
2222  * to the bandwidth used.
2223  */
2224 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2225                 struct xhci_virt_device *virt_dev,
2226                 int old_active_eps)
2227 {
2228         unsigned int bw_reserved;
2229         unsigned int max_bandwidth;
2230         unsigned int bw_used;
2231         unsigned int block_size;
2232         struct xhci_interval_bw_table *bw_table;
2233         unsigned int packet_size = 0;
2234         unsigned int overhead = 0;
2235         unsigned int packets_transmitted = 0;
2236         unsigned int packets_remaining = 0;
2237         unsigned int i;
2238
2239         if (virt_dev->udev->speed >= USB_SPEED_SUPER)
2240                 return xhci_check_ss_bw(xhci, virt_dev);
2241
2242         if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2243                 max_bandwidth = HS_BW_LIMIT;
2244                 /* Convert percent of bus BW reserved to blocks reserved */
2245                 bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2246         } else {
2247                 max_bandwidth = FS_BW_LIMIT;
2248                 bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2249         }
2250
2251         bw_table = virt_dev->bw_table;
2252         /* We need to translate the max packet size and max ESIT payloads into
2253          * the units the hardware uses.
2254          */
2255         block_size = xhci_get_block_size(virt_dev->udev);
2256
2257         /* If we are manipulating a LS/FS device under a HS hub, double check
2258          * that the HS bus has enough bandwidth if we are activing a new TT.
2259          */
2260         if (virt_dev->tt_info) {
2261                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2262                                 "Recalculating BW for rootport %u",
2263                                 virt_dev->real_port);
2264                 if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2265                         xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2266                                         "newly activated TT.\n");
2267                         return -ENOMEM;
2268                 }
2269                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2270                                 "Recalculating BW for TT slot %u port %u",
2271                                 virt_dev->tt_info->slot_id,
2272                                 virt_dev->tt_info->ttport);
2273         } else {
2274                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2275                                 "Recalculating BW for rootport %u",
2276                                 virt_dev->real_port);
2277         }
2278
2279         /* Add in how much bandwidth will be used for interval zero, or the
2280          * rounded max ESIT payload + number of packets * largest overhead.
2281          */
2282         bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2283                 bw_table->interval_bw[0].num_packets *
2284                 xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2285
2286         for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2287                 unsigned int bw_added;
2288                 unsigned int largest_mps;
2289                 unsigned int interval_overhead;
2290
2291                 /*
2292                  * How many packets could we transmit in this interval?
2293                  * If packets didn't fit in the previous interval, we will need
2294                  * to transmit that many packets twice within this interval.
2295                  */
2296                 packets_remaining = 2 * packets_remaining +
2297                         bw_table->interval_bw[i].num_packets;
2298
2299                 /* Find the largest max packet size of this or the previous
2300                  * interval.
2301                  */
2302                 if (list_empty(&bw_table->interval_bw[i].endpoints))
2303                         largest_mps = 0;
2304                 else {
2305                         struct xhci_virt_ep *virt_ep;
2306                         struct list_head *ep_entry;
2307
2308                         ep_entry = bw_table->interval_bw[i].endpoints.next;
2309                         virt_ep = list_entry(ep_entry,
2310                                         struct xhci_virt_ep, bw_endpoint_list);
2311                         /* Convert to blocks, rounding up */
2312                         largest_mps = DIV_ROUND_UP(
2313                                         virt_ep->bw_info.max_packet_size,
2314                                         block_size);
2315                 }
2316                 if (largest_mps > packet_size)
2317                         packet_size = largest_mps;
2318
2319                 /* Use the larger overhead of this or the previous interval. */
2320                 interval_overhead = xhci_get_largest_overhead(
2321                                 &bw_table->interval_bw[i]);
2322                 if (interval_overhead > overhead)
2323                         overhead = interval_overhead;
2324
2325                 /* How many packets can we evenly distribute across
2326                  * (1 << (i + 1)) possible scheduling opportunities?
2327                  */
2328                 packets_transmitted = packets_remaining >> (i + 1);
2329
2330                 /* Add in the bandwidth used for those scheduled packets */
2331                 bw_added = packets_transmitted * (overhead + packet_size);
2332
2333                 /* How many packets do we have remaining to transmit? */
2334                 packets_remaining = packets_remaining % (1 << (i + 1));
2335
2336                 /* What largest max packet size should those packets have? */
2337                 /* If we've transmitted all packets, don't carry over the
2338                  * largest packet size.
2339                  */
2340                 if (packets_remaining == 0) {
2341                         packet_size = 0;
2342                         overhead = 0;
2343                 } else if (packets_transmitted > 0) {
2344                         /* Otherwise if we do have remaining packets, and we've
2345                          * scheduled some packets in this interval, take the
2346                          * largest max packet size from endpoints with this
2347                          * interval.
2348                          */
2349                         packet_size = largest_mps;
2350                         overhead = interval_overhead;
2351                 }
2352                 /* Otherwise carry over packet_size and overhead from the last
2353                  * time we had a remainder.
2354                  */
2355                 bw_used += bw_added;
2356                 if (bw_used > max_bandwidth) {
2357                         xhci_warn(xhci, "Not enough bandwidth. "
2358                                         "Proposed: %u, Max: %u\n",
2359                                 bw_used, max_bandwidth);
2360                         return -ENOMEM;
2361                 }
2362         }
2363         /*
2364          * Ok, we know we have some packets left over after even-handedly
2365          * scheduling interval 15.  We don't know which microframes they will
2366          * fit into, so we over-schedule and say they will be scheduled every
2367          * microframe.
2368          */
2369         if (packets_remaining > 0)
2370                 bw_used += overhead + packet_size;
2371
2372         if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2373                 unsigned int port_index = virt_dev->real_port - 1;
2374
2375                 /* OK, we're manipulating a HS device attached to a
2376                  * root port bandwidth domain.  Include the number of active TTs
2377                  * in the bandwidth used.
2378                  */
2379                 bw_used += TT_HS_OVERHEAD *
2380                         xhci->rh_bw[port_index].num_active_tts;
2381         }
2382
2383         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2384                 "Final bandwidth: %u, Limit: %u, Reserved: %u, "
2385                 "Available: %u " "percent",
2386                 bw_used, max_bandwidth, bw_reserved,
2387                 (max_bandwidth - bw_used - bw_reserved) * 100 /
2388                 max_bandwidth);
2389
2390         bw_used += bw_reserved;
2391         if (bw_used > max_bandwidth) {
2392                 xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2393                                 bw_used, max_bandwidth);
2394                 return -ENOMEM;
2395         }
2396
2397         bw_table->bw_used = bw_used;
2398         return 0;
2399 }
2400
2401 static bool xhci_is_async_ep(unsigned int ep_type)
2402 {
2403         return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2404                                         ep_type != ISOC_IN_EP &&
2405                                         ep_type != INT_IN_EP);
2406 }
2407
2408 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2409 {
2410         return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2411 }
2412
2413 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2414 {
2415         unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2416
2417         if (ep_bw->ep_interval == 0)
2418                 return SS_OVERHEAD_BURST +
2419                         (ep_bw->mult * ep_bw->num_packets *
2420                                         (SS_OVERHEAD + mps));
2421         return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2422                                 (SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2423                                 1 << ep_bw->ep_interval);
2424
2425 }
2426
2427 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2428                 struct xhci_bw_info *ep_bw,
2429                 struct xhci_interval_bw_table *bw_table,
2430                 struct usb_device *udev,
2431                 struct xhci_virt_ep *virt_ep,
2432                 struct xhci_tt_bw_info *tt_info)
2433 {
2434         struct xhci_interval_bw *interval_bw;
2435         int normalized_interval;
2436
2437         if (xhci_is_async_ep(ep_bw->type))
2438                 return;
2439
2440         if (udev->speed >= USB_SPEED_SUPER) {
2441                 if (xhci_is_sync_in_ep(ep_bw->type))
2442                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2443                                 xhci_get_ss_bw_consumed(ep_bw);
2444                 else
2445                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2446                                 xhci_get_ss_bw_consumed(ep_bw);
2447                 return;
2448         }
2449
2450         /* SuperSpeed endpoints never get added to intervals in the table, so
2451          * this check is only valid for HS/FS/LS devices.
2452          */
2453         if (list_empty(&virt_ep->bw_endpoint_list))
2454                 return;
2455         /* For LS/FS devices, we need to translate the interval expressed in
2456          * microframes to frames.
2457          */
2458         if (udev->speed == USB_SPEED_HIGH)
2459                 normalized_interval = ep_bw->ep_interval;
2460         else
2461                 normalized_interval = ep_bw->ep_interval - 3;
2462
2463         if (normalized_interval == 0)
2464                 bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2465         interval_bw = &bw_table->interval_bw[normalized_interval];
2466         interval_bw->num_packets -= ep_bw->num_packets;
2467         switch (udev->speed) {
2468         case USB_SPEED_LOW:
2469                 interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2470                 break;
2471         case USB_SPEED_FULL:
2472                 interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2473                 break;
2474         case USB_SPEED_HIGH:
2475                 interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2476                 break;
2477         case USB_SPEED_SUPER:
2478         case USB_SPEED_SUPER_PLUS:
2479         case USB_SPEED_UNKNOWN:
2480         case USB_SPEED_WIRELESS:
2481                 /* Should never happen because only LS/FS/HS endpoints will get
2482                  * added to the endpoint list.
2483                  */
2484                 return;
2485         }
2486         if (tt_info)
2487                 tt_info->active_eps -= 1;
2488         list_del_init(&virt_ep->bw_endpoint_list);
2489 }
2490
2491 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2492                 struct xhci_bw_info *ep_bw,
2493                 struct xhci_interval_bw_table *bw_table,
2494                 struct usb_device *udev,
2495                 struct xhci_virt_ep *virt_ep,
2496                 struct xhci_tt_bw_info *tt_info)
2497 {
2498         struct xhci_interval_bw *interval_bw;
2499         struct xhci_virt_ep *smaller_ep;
2500         int normalized_interval;
2501
2502         if (xhci_is_async_ep(ep_bw->type))
2503                 return;
2504
2505         if (udev->speed == USB_SPEED_SUPER) {
2506                 if (xhci_is_sync_in_ep(ep_bw->type))
2507                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2508                                 xhci_get_ss_bw_consumed(ep_bw);
2509                 else
2510                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2511                                 xhci_get_ss_bw_consumed(ep_bw);
2512                 return;
2513         }
2514
2515         /* For LS/FS devices, we need to translate the interval expressed in
2516          * microframes to frames.
2517          */
2518         if (udev->speed == USB_SPEED_HIGH)
2519                 normalized_interval = ep_bw->ep_interval;
2520         else
2521                 normalized_interval = ep_bw->ep_interval - 3;
2522
2523         if (normalized_interval == 0)
2524                 bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2525         interval_bw = &bw_table->interval_bw[normalized_interval];
2526         interval_bw->num_packets += ep_bw->num_packets;
2527         switch (udev->speed) {
2528         case USB_SPEED_LOW:
2529                 interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2530                 break;
2531         case USB_SPEED_FULL:
2532                 interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2533                 break;
2534         case USB_SPEED_HIGH:
2535                 interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2536                 break;
2537         case USB_SPEED_SUPER:
2538         case USB_SPEED_SUPER_PLUS:
2539         case USB_SPEED_UNKNOWN:
2540         case USB_SPEED_WIRELESS:
2541                 /* Should never happen because only LS/FS/HS endpoints will get
2542                  * added to the endpoint list.
2543                  */
2544                 return;
2545         }
2546
2547         if (tt_info)
2548                 tt_info->active_eps += 1;
2549         /* Insert the endpoint into the list, largest max packet size first. */
2550         list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2551                         bw_endpoint_list) {
2552                 if (ep_bw->max_packet_size >=
2553                                 smaller_ep->bw_info.max_packet_size) {
2554                         /* Add the new ep before the smaller endpoint */
2555                         list_add_tail(&virt_ep->bw_endpoint_list,
2556                                         &smaller_ep->bw_endpoint_list);
2557                         return;
2558                 }
2559         }
2560         /* Add the new endpoint at the end of the list. */
2561         list_add_tail(&virt_ep->bw_endpoint_list,
2562                         &interval_bw->endpoints);
2563 }
2564
2565 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2566                 struct xhci_virt_device *virt_dev,
2567                 int old_active_eps)
2568 {
2569         struct xhci_root_port_bw_info *rh_bw_info;
2570         if (!virt_dev->tt_info)
2571                 return;
2572
2573         rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
2574         if (old_active_eps == 0 &&
2575                                 virt_dev->tt_info->active_eps != 0) {
2576                 rh_bw_info->num_active_tts += 1;
2577                 rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2578         } else if (old_active_eps != 0 &&
2579                                 virt_dev->tt_info->active_eps == 0) {
2580                 rh_bw_info->num_active_tts -= 1;
2581                 rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2582         }
2583 }
2584
2585 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2586                 struct xhci_virt_device *virt_dev,
2587                 struct xhci_container_ctx *in_ctx)
2588 {
2589         struct xhci_bw_info ep_bw_info[31];
2590         int i;
2591         struct xhci_input_control_ctx *ctrl_ctx;
2592         int old_active_eps = 0;
2593
2594         if (virt_dev->tt_info)
2595                 old_active_eps = virt_dev->tt_info->active_eps;
2596
2597         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2598         if (!ctrl_ctx) {
2599                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2600                                 __func__);
2601                 return -ENOMEM;
2602         }
2603
2604         for (i = 0; i < 31; i++) {
2605                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2606                         continue;
2607
2608                 /* Make a copy of the BW info in case we need to revert this */
2609                 memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2610                                 sizeof(ep_bw_info[i]));
2611                 /* Drop the endpoint from the interval table if the endpoint is
2612                  * being dropped or changed.
2613                  */
2614                 if (EP_IS_DROPPED(ctrl_ctx, i))
2615                         xhci_drop_ep_from_interval_table(xhci,
2616                                         &virt_dev->eps[i].bw_info,
2617                                         virt_dev->bw_table,
2618                                         virt_dev->udev,
2619                                         &virt_dev->eps[i],
2620                                         virt_dev->tt_info);
2621         }
2622         /* Overwrite the information stored in the endpoints' bw_info */
2623         xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2624         for (i = 0; i < 31; i++) {
2625                 /* Add any changed or added endpoints to the interval table */
2626                 if (EP_IS_ADDED(ctrl_ctx, i))
2627                         xhci_add_ep_to_interval_table(xhci,
2628                                         &virt_dev->eps[i].bw_info,
2629                                         virt_dev->bw_table,
2630                                         virt_dev->udev,
2631                                         &virt_dev->eps[i],
2632                                         virt_dev->tt_info);
2633         }
2634
2635         if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2636                 /* Ok, this fits in the bandwidth we have.
2637                  * Update the number of active TTs.
2638                  */
2639                 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2640                 return 0;
2641         }
2642
2643         /* We don't have enough bandwidth for this, revert the stored info. */
2644         for (i = 0; i < 31; i++) {
2645                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2646                         continue;
2647
2648                 /* Drop the new copies of any added or changed endpoints from
2649                  * the interval table.
2650                  */
2651                 if (EP_IS_ADDED(ctrl_ctx, i)) {
2652                         xhci_drop_ep_from_interval_table(xhci,
2653                                         &virt_dev->eps[i].bw_info,
2654                                         virt_dev->bw_table,
2655                                         virt_dev->udev,
2656                                         &virt_dev->eps[i],
2657                                         virt_dev->tt_info);
2658                 }
2659                 /* Revert the endpoint back to its old information */
2660                 memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2661                                 sizeof(ep_bw_info[i]));
2662                 /* Add any changed or dropped endpoints back into the table */
2663                 if (EP_IS_DROPPED(ctrl_ctx, i))
2664                         xhci_add_ep_to_interval_table(xhci,
2665                                         &virt_dev->eps[i].bw_info,
2666                                         virt_dev->bw_table,
2667                                         virt_dev->udev,
2668                                         &virt_dev->eps[i],
2669                                         virt_dev->tt_info);
2670         }
2671         return -ENOMEM;
2672 }
2673
2674
2675 /* Issue a configure endpoint command or evaluate context command
2676  * and wait for it to finish.
2677  */
2678 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2679                 struct usb_device *udev,
2680                 struct xhci_command *command,
2681                 bool ctx_change, bool must_succeed)
2682 {
2683         int ret;
2684         unsigned long flags;
2685         struct xhci_input_control_ctx *ctrl_ctx;
2686         struct xhci_virt_device *virt_dev;
2687
2688         if (!command)
2689                 return -EINVAL;
2690
2691         spin_lock_irqsave(&xhci->lock, flags);
2692
2693         if (xhci->xhc_state & XHCI_STATE_DYING) {
2694                 spin_unlock_irqrestore(&xhci->lock, flags);
2695                 return -ESHUTDOWN;
2696         }
2697
2698         virt_dev = xhci->devs[udev->slot_id];
2699
2700         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2701         if (!ctrl_ctx) {
2702                 spin_unlock_irqrestore(&xhci->lock, flags);
2703                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2704                                 __func__);
2705                 return -ENOMEM;
2706         }
2707
2708         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2709                         xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2710                 spin_unlock_irqrestore(&xhci->lock, flags);
2711                 xhci_warn(xhci, "Not enough host resources, "
2712                                 "active endpoint contexts = %u\n",
2713                                 xhci->num_active_eps);
2714                 return -ENOMEM;
2715         }
2716         if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2717             xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2718                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2719                         xhci_free_host_resources(xhci, ctrl_ctx);
2720                 spin_unlock_irqrestore(&xhci->lock, flags);
2721                 xhci_warn(xhci, "Not enough bandwidth\n");
2722                 return -ENOMEM;
2723         }
2724
2725         if (!ctx_change)
2726                 ret = xhci_queue_configure_endpoint(xhci, command,
2727                                 command->in_ctx->dma,
2728                                 udev->slot_id, must_succeed);
2729         else
2730                 ret = xhci_queue_evaluate_context(xhci, command,
2731                                 command->in_ctx->dma,
2732                                 udev->slot_id, must_succeed);
2733         if (ret < 0) {
2734                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2735                         xhci_free_host_resources(xhci, ctrl_ctx);
2736                 spin_unlock_irqrestore(&xhci->lock, flags);
2737                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
2738                                 "FIXME allocate a new ring segment");
2739                 return -ENOMEM;
2740         }
2741         xhci_ring_cmd_db(xhci);
2742         spin_unlock_irqrestore(&xhci->lock, flags);
2743
2744         /* Wait for the configure endpoint command to complete */
2745         wait_for_completion(command->completion);
2746
2747         if (!ctx_change)
2748                 ret = xhci_configure_endpoint_result(xhci, udev,
2749                                                      &command->status);
2750         else
2751                 ret = xhci_evaluate_context_result(xhci, udev,
2752                                                    &command->status);
2753
2754         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2755                 spin_lock_irqsave(&xhci->lock, flags);
2756                 /* If the command failed, remove the reserved resources.
2757                  * Otherwise, clean up the estimate to include dropped eps.
2758                  */
2759                 if (ret)
2760                         xhci_free_host_resources(xhci, ctrl_ctx);
2761                 else
2762                         xhci_finish_resource_reservation(xhci, ctrl_ctx);
2763                 spin_unlock_irqrestore(&xhci->lock, flags);
2764         }
2765         return ret;
2766 }
2767
2768 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2769         struct xhci_virt_device *vdev, int i)
2770 {
2771         struct xhci_virt_ep *ep = &vdev->eps[i];
2772
2773         if (ep->ep_state & EP_HAS_STREAMS) {
2774                 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2775                                 xhci_get_endpoint_address(i));
2776                 xhci_free_stream_info(xhci, ep->stream_info);
2777                 ep->stream_info = NULL;
2778                 ep->ep_state &= ~EP_HAS_STREAMS;
2779         }
2780 }
2781
2782 /* Called after one or more calls to xhci_add_endpoint() or
2783  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2784  * to call xhci_reset_bandwidth().
2785  *
2786  * Since we are in the middle of changing either configuration or
2787  * installing a new alt setting, the USB core won't allow URBs to be
2788  * enqueued for any endpoint on the old config or interface.  Nothing
2789  * else should be touching the xhci->devs[slot_id] structure, so we
2790  * don't need to take the xhci->lock for manipulating that.
2791  */
2792 static int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2793 {
2794         int i;
2795         int ret = 0;
2796         struct xhci_hcd *xhci;
2797         struct xhci_virt_device *virt_dev;
2798         struct xhci_input_control_ctx *ctrl_ctx;
2799         struct xhci_slot_ctx *slot_ctx;
2800         struct xhci_command *command;
2801
2802         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2803         if (ret <= 0)
2804                 return ret;
2805         xhci = hcd_to_xhci(hcd);
2806         if ((xhci->xhc_state & XHCI_STATE_DYING) ||
2807                 (xhci->xhc_state & XHCI_STATE_REMOVING))
2808                 return -ENODEV;
2809
2810         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2811         virt_dev = xhci->devs[udev->slot_id];
2812
2813         command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
2814         if (!command)
2815                 return -ENOMEM;
2816
2817         command->in_ctx = virt_dev->in_ctx;
2818
2819         /* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2820         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2821         if (!ctrl_ctx) {
2822                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2823                                 __func__);
2824                 ret = -ENOMEM;
2825                 goto command_cleanup;
2826         }
2827         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2828         ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
2829         ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
2830
2831         /* Don't issue the command if there's no endpoints to update. */
2832         if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
2833             ctrl_ctx->drop_flags == 0) {
2834                 ret = 0;
2835                 goto command_cleanup;
2836         }
2837         /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
2838         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2839         for (i = 31; i >= 1; i--) {
2840                 __le32 le32 = cpu_to_le32(BIT(i));
2841
2842                 if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
2843                     || (ctrl_ctx->add_flags & le32) || i == 1) {
2844                         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2845                         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
2846                         break;
2847                 }
2848         }
2849
2850         ret = xhci_configure_endpoint(xhci, udev, command,
2851                         false, false);
2852         if (ret)
2853                 /* Callee should call reset_bandwidth() */
2854                 goto command_cleanup;
2855
2856         /* Free any rings that were dropped, but not changed. */
2857         for (i = 1; i < 31; i++) {
2858                 if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
2859                     !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
2860                         xhci_free_endpoint_ring(xhci, virt_dev, i);
2861                         xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2862                 }
2863         }
2864         xhci_zero_in_ctx(xhci, virt_dev);
2865         /*
2866          * Install any rings for completely new endpoints or changed endpoints,
2867          * and free any old rings from changed endpoints.
2868          */
2869         for (i = 1; i < 31; i++) {
2870                 if (!virt_dev->eps[i].new_ring)
2871                         continue;
2872                 /* Only free the old ring if it exists.
2873                  * It may not if this is the first add of an endpoint.
2874                  */
2875                 if (virt_dev->eps[i].ring) {
2876                         xhci_free_endpoint_ring(xhci, virt_dev, i);
2877                 }
2878                 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2879                 virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
2880                 virt_dev->eps[i].new_ring = NULL;
2881         }
2882 command_cleanup:
2883         kfree(command->completion);
2884         kfree(command);
2885
2886         return ret;
2887 }
2888
2889 static void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2890 {
2891         struct xhci_hcd *xhci;
2892         struct xhci_virt_device *virt_dev;
2893         int i, ret;
2894
2895         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2896         if (ret <= 0)
2897                 return;
2898         xhci = hcd_to_xhci(hcd);
2899
2900         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2901         virt_dev = xhci->devs[udev->slot_id];
2902         /* Free any rings allocated for added endpoints */
2903         for (i = 0; i < 31; i++) {
2904                 if (virt_dev->eps[i].new_ring) {
2905                         xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
2906                         virt_dev->eps[i].new_ring = NULL;
2907                 }
2908         }
2909         xhci_zero_in_ctx(xhci, virt_dev);
2910 }
2911
2912 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
2913                 struct xhci_container_ctx *in_ctx,
2914                 struct xhci_container_ctx *out_ctx,
2915                 struct xhci_input_control_ctx *ctrl_ctx,
2916                 u32 add_flags, u32 drop_flags)
2917 {
2918         ctrl_ctx->add_flags = cpu_to_le32(add_flags);
2919         ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
2920         xhci_slot_copy(xhci, in_ctx, out_ctx);
2921         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2922 }
2923
2924 static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
2925                 unsigned int slot_id, unsigned int ep_index,
2926                 struct xhci_dequeue_state *deq_state)
2927 {
2928         struct xhci_input_control_ctx *ctrl_ctx;
2929         struct xhci_container_ctx *in_ctx;
2930         struct xhci_ep_ctx *ep_ctx;
2931         u32 added_ctxs;
2932         dma_addr_t addr;
2933
2934         in_ctx = xhci->devs[slot_id]->in_ctx;
2935         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2936         if (!ctrl_ctx) {
2937                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2938                                 __func__);
2939                 return;
2940         }
2941
2942         xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
2943                         xhci->devs[slot_id]->out_ctx, ep_index);
2944         ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
2945         addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
2946                         deq_state->new_deq_ptr);
2947         if (addr == 0) {
2948                 xhci_warn(xhci, "WARN Cannot submit config ep after "
2949                                 "reset ep command\n");
2950                 xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
2951                                 deq_state->new_deq_seg,
2952                                 deq_state->new_deq_ptr);
2953                 return;
2954         }
2955         ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
2956
2957         added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
2958         xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
2959                         xhci->devs[slot_id]->out_ctx, ctrl_ctx,
2960                         added_ctxs, added_ctxs);
2961 }
2962
2963 void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci, unsigned int ep_index,
2964                                unsigned int stream_id, struct xhci_td *td)
2965 {
2966         struct xhci_dequeue_state deq_state;
2967         struct xhci_virt_ep *ep;
2968         struct usb_device *udev = td->urb->dev;
2969
2970         xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
2971                         "Cleaning up stalled endpoint ring");
2972         ep = &xhci->devs[udev->slot_id]->eps[ep_index];
2973         /* We need to move the HW's dequeue pointer past this TD,
2974          * or it will attempt to resend it on the next doorbell ring.
2975          */
2976         xhci_find_new_dequeue_state(xhci, udev->slot_id,
2977                         ep_index, stream_id, td, &deq_state);
2978
2979         if (!deq_state.new_deq_ptr || !deq_state.new_deq_seg)
2980                 return;
2981
2982         /* HW with the reset endpoint quirk will use the saved dequeue state to
2983          * issue a configure endpoint command later.
2984          */
2985         if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
2986                 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
2987                                 "Queueing new dequeue state");
2988                 xhci_queue_new_dequeue_state(xhci, udev->slot_id,
2989                                 ep_index, &deq_state);
2990         } else {
2991                 /* Better hope no one uses the input context between now and the
2992                  * reset endpoint completion!
2993                  * XXX: No idea how this hardware will react when stream rings
2994                  * are enabled.
2995                  */
2996                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2997                                 "Setting up input context for "
2998                                 "configure endpoint command");
2999                 xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
3000                                 ep_index, &deq_state);
3001         }
3002 }
3003
3004 /* Called when clearing halted device. The core should have sent the control
3005  * message to clear the device halt condition. The host side of the halt should
3006  * already be cleared with a reset endpoint command issued when the STALL tx
3007  * event was received.
3008  *
3009  * Context: in_interrupt
3010  */
3011
3012 static void xhci_endpoint_reset(struct usb_hcd *hcd,
3013                 struct usb_host_endpoint *ep)
3014 {
3015         struct xhci_hcd *xhci;
3016
3017         xhci = hcd_to_xhci(hcd);
3018
3019         /*
3020          * We might need to implement the config ep cmd in xhci 4.8.1 note:
3021          * The Reset Endpoint Command may only be issued to endpoints in the
3022          * Halted state. If software wishes reset the Data Toggle or Sequence
3023          * Number of an endpoint that isn't in the Halted state, then software
3024          * may issue a Configure Endpoint Command with the Drop and Add bits set
3025          * for the target endpoint. that is in the Stopped state.
3026          */
3027
3028         /* For now just print debug to follow the situation */
3029         xhci_dbg(xhci, "Endpoint 0x%x ep reset callback called\n",
3030                  ep->desc.bEndpointAddress);
3031 }
3032
3033 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
3034                 struct usb_device *udev, struct usb_host_endpoint *ep,
3035                 unsigned int slot_id)
3036 {
3037         int ret;
3038         unsigned int ep_index;
3039         unsigned int ep_state;
3040
3041         if (!ep)
3042                 return -EINVAL;
3043         ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
3044         if (ret <= 0)
3045                 return ret ? ret : -EINVAL;
3046         if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
3047                 xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
3048                                 " descriptor for ep 0x%x does not support streams\n",
3049                                 ep->desc.bEndpointAddress);
3050                 return -EINVAL;
3051         }
3052
3053         ep_index = xhci_get_endpoint_index(&ep->desc);
3054         ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3055         if (ep_state & EP_HAS_STREAMS ||
3056                         ep_state & EP_GETTING_STREAMS) {
3057                 xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3058                                 "already has streams set up.\n",
3059                                 ep->desc.bEndpointAddress);
3060                 xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3061                                 "dynamic stream context array reallocation.\n");
3062                 return -EINVAL;
3063         }
3064         if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3065                 xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3066                                 "endpoint 0x%x; URBs are pending.\n",
3067                                 ep->desc.bEndpointAddress);
3068                 return -EINVAL;
3069         }
3070         return 0;
3071 }
3072
3073 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3074                 unsigned int *num_streams, unsigned int *num_stream_ctxs)
3075 {
3076         unsigned int max_streams;
3077
3078         /* The stream context array size must be a power of two */
3079         *num_stream_ctxs = roundup_pow_of_two(*num_streams);
3080         /*
3081          * Find out how many primary stream array entries the host controller
3082          * supports.  Later we may use secondary stream arrays (similar to 2nd
3083          * level page entries), but that's an optional feature for xHCI host
3084          * controllers. xHCs must support at least 4 stream IDs.
3085          */
3086         max_streams = HCC_MAX_PSA(xhci->hcc_params);
3087         if (*num_stream_ctxs > max_streams) {
3088                 xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3089                                 max_streams);
3090                 *num_stream_ctxs = max_streams;
3091                 *num_streams = max_streams;
3092         }
3093 }
3094
3095 /* Returns an error code if one of the endpoint already has streams.
3096  * This does not change any data structures, it only checks and gathers
3097  * information.
3098  */
3099 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3100                 struct usb_device *udev,
3101                 struct usb_host_endpoint **eps, unsigned int num_eps,
3102                 unsigned int *num_streams, u32 *changed_ep_bitmask)
3103 {
3104         unsigned int max_streams;
3105         unsigned int endpoint_flag;
3106         int i;
3107         int ret;
3108
3109         for (i = 0; i < num_eps; i++) {
3110                 ret = xhci_check_streams_endpoint(xhci, udev,
3111                                 eps[i], udev->slot_id);
3112                 if (ret < 0)
3113                         return ret;
3114
3115                 max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3116                 if (max_streams < (*num_streams - 1)) {
3117                         xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3118                                         eps[i]->desc.bEndpointAddress,
3119                                         max_streams);
3120                         *num_streams = max_streams+1;
3121                 }
3122
3123                 endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3124                 if (*changed_ep_bitmask & endpoint_flag)
3125                         return -EINVAL;
3126                 *changed_ep_bitmask |= endpoint_flag;
3127         }
3128         return 0;
3129 }
3130
3131 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3132                 struct usb_device *udev,
3133                 struct usb_host_endpoint **eps, unsigned int num_eps)
3134 {
3135         u32 changed_ep_bitmask = 0;
3136         unsigned int slot_id;
3137         unsigned int ep_index;
3138         unsigned int ep_state;
3139         int i;
3140
3141         slot_id = udev->slot_id;
3142         if (!xhci->devs[slot_id])
3143                 return 0;
3144
3145         for (i = 0; i < num_eps; i++) {
3146                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3147                 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3148                 /* Are streams already being freed for the endpoint? */
3149                 if (ep_state & EP_GETTING_NO_STREAMS) {
3150                         xhci_warn(xhci, "WARN Can't disable streams for "
3151                                         "endpoint 0x%x, "
3152                                         "streams are being disabled already\n",
3153                                         eps[i]->desc.bEndpointAddress);
3154                         return 0;
3155                 }
3156                 /* Are there actually any streams to free? */
3157                 if (!(ep_state & EP_HAS_STREAMS) &&
3158                                 !(ep_state & EP_GETTING_STREAMS)) {
3159                         xhci_warn(xhci, "WARN Can't disable streams for "
3160                                         "endpoint 0x%x, "
3161                                         "streams are already disabled!\n",
3162                                         eps[i]->desc.bEndpointAddress);
3163                         xhci_warn(xhci, "WARN xhci_free_streams() called "
3164                                         "with non-streams endpoint\n");
3165                         return 0;
3166                 }
3167                 changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3168         }
3169         return changed_ep_bitmask;
3170 }
3171
3172 /*
3173  * The USB device drivers use this function (through the HCD interface in USB
3174  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3175  * coordinate mass storage command queueing across multiple endpoints (basically
3176  * a stream ID == a task ID).
3177  *
3178  * Setting up streams involves allocating the same size stream context array
3179  * for each endpoint and issuing a configure endpoint command for all endpoints.
3180  *
3181  * Don't allow the call to succeed if one endpoint only supports one stream
3182  * (which means it doesn't support streams at all).
3183  *
3184  * Drivers may get less stream IDs than they asked for, if the host controller
3185  * hardware or endpoints claim they can't support the number of requested
3186  * stream IDs.
3187  */
3188 static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3189                 struct usb_host_endpoint **eps, unsigned int num_eps,
3190                 unsigned int num_streams, gfp_t mem_flags)
3191 {
3192         int i, ret;
3193         struct xhci_hcd *xhci;
3194         struct xhci_virt_device *vdev;
3195         struct xhci_command *config_cmd;
3196         struct xhci_input_control_ctx *ctrl_ctx;
3197         unsigned int ep_index;
3198         unsigned int num_stream_ctxs;
3199         unsigned int max_packet;
3200         unsigned long flags;
3201         u32 changed_ep_bitmask = 0;
3202
3203         if (!eps)
3204                 return -EINVAL;
3205
3206         /* Add one to the number of streams requested to account for
3207          * stream 0 that is reserved for xHCI usage.
3208          */
3209         num_streams += 1;
3210         xhci = hcd_to_xhci(hcd);
3211         xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3212                         num_streams);
3213
3214         /* MaxPSASize value 0 (2 streams) means streams are not supported */
3215         if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3216                         HCC_MAX_PSA(xhci->hcc_params) < 4) {
3217                 xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3218                 return -ENOSYS;
3219         }
3220
3221         config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
3222         if (!config_cmd)
3223                 return -ENOMEM;
3224
3225         ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3226         if (!ctrl_ctx) {
3227                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3228                                 __func__);
3229                 xhci_free_command(xhci, config_cmd);
3230                 return -ENOMEM;
3231         }
3232
3233         /* Check to make sure all endpoints are not already configured for
3234          * streams.  While we're at it, find the maximum number of streams that
3235          * all the endpoints will support and check for duplicate endpoints.
3236          */
3237         spin_lock_irqsave(&xhci->lock, flags);
3238         ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3239                         num_eps, &num_streams, &changed_ep_bitmask);
3240         if (ret < 0) {
3241                 xhci_free_command(xhci, config_cmd);
3242                 spin_unlock_irqrestore(&xhci->lock, flags);
3243                 return ret;
3244         }
3245         if (num_streams <= 1) {
3246                 xhci_warn(xhci, "WARN: endpoints can't handle "
3247                                 "more than one stream.\n");
3248                 xhci_free_command(xhci, config_cmd);
3249                 spin_unlock_irqrestore(&xhci->lock, flags);
3250                 return -EINVAL;
3251         }
3252         vdev = xhci->devs[udev->slot_id];
3253         /* Mark each endpoint as being in transition, so
3254          * xhci_urb_enqueue() will reject all URBs.
3255          */
3256         for (i = 0; i < num_eps; i++) {
3257                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3258                 vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3259         }
3260         spin_unlock_irqrestore(&xhci->lock, flags);
3261
3262         /* Setup internal data structures and allocate HW data structures for
3263          * streams (but don't install the HW structures in the input context
3264          * until we're sure all memory allocation succeeded).
3265          */
3266         xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3267         xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3268                         num_stream_ctxs, num_streams);
3269
3270         for (i = 0; i < num_eps; i++) {
3271                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3272                 max_packet = usb_endpoint_maxp(&eps[i]->desc);
3273                 vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3274                                 num_stream_ctxs,
3275                                 num_streams,
3276                                 max_packet, mem_flags);
3277                 if (!vdev->eps[ep_index].stream_info)
3278                         goto cleanup;
3279                 /* Set maxPstreams in endpoint context and update deq ptr to
3280                  * point to stream context array. FIXME
3281                  */
3282         }
3283
3284         /* Set up the input context for a configure endpoint command. */
3285         for (i = 0; i < num_eps; i++) {
3286                 struct xhci_ep_ctx *ep_ctx;
3287
3288                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3289                 ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3290
3291                 xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3292                                 vdev->out_ctx, ep_index);
3293                 xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3294                                 vdev->eps[ep_index].stream_info);
3295         }
3296         /* Tell the HW to drop its old copy of the endpoint context info
3297          * and add the updated copy from the input context.
3298          */
3299         xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3300                         vdev->out_ctx, ctrl_ctx,
3301                         changed_ep_bitmask, changed_ep_bitmask);
3302
3303         /* Issue and wait for the configure endpoint command */
3304         ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3305                         false, false);
3306
3307         /* xHC rejected the configure endpoint command for some reason, so we
3308          * leave the old ring intact and free our internal streams data
3309          * structure.
3310          */
3311         if (ret < 0)
3312                 goto cleanup;
3313
3314         spin_lock_irqsave(&xhci->lock, flags);
3315         for (i = 0; i < num_eps; i++) {
3316                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3317                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3318                 xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3319                          udev->slot_id, ep_index);
3320                 vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3321         }
3322         xhci_free_command(xhci, config_cmd);
3323         spin_unlock_irqrestore(&xhci->lock, flags);
3324
3325         /* Subtract 1 for stream 0, which drivers can't use */
3326         return num_streams - 1;
3327
3328 cleanup:
3329         /* If it didn't work, free the streams! */
3330         for (i = 0; i < num_eps; i++) {
3331                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3332                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3333                 vdev->eps[ep_index].stream_info = NULL;
3334                 /* FIXME Unset maxPstreams in endpoint context and
3335                  * update deq ptr to point to normal string ring.
3336                  */
3337                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3338                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3339                 xhci_endpoint_zero(xhci, vdev, eps[i]);
3340         }
3341         xhci_free_command(xhci, config_cmd);
3342         return -ENOMEM;
3343 }
3344
3345 /* Transition the endpoint from using streams to being a "normal" endpoint
3346  * without streams.
3347  *
3348  * Modify the endpoint context state, submit a configure endpoint command,
3349  * and free all endpoint rings for streams if that completes successfully.
3350  */
3351 static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3352                 struct usb_host_endpoint **eps, unsigned int num_eps,
3353                 gfp_t mem_flags)
3354 {
3355         int i, ret;
3356         struct xhci_hcd *xhci;
3357         struct xhci_virt_device *vdev;
3358         struct xhci_command *command;
3359         struct xhci_input_control_ctx *ctrl_ctx;
3360         unsigned int ep_index;
3361         unsigned long flags;
3362         u32 changed_ep_bitmask;
3363
3364         xhci = hcd_to_xhci(hcd);
3365         vdev = xhci->devs[udev->slot_id];
3366
3367         /* Set up a configure endpoint command to remove the streams rings */
3368         spin_lock_irqsave(&xhci->lock, flags);
3369         changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3370                         udev, eps, num_eps);
3371         if (changed_ep_bitmask == 0) {
3372                 spin_unlock_irqrestore(&xhci->lock, flags);
3373                 return -EINVAL;
3374         }
3375
3376         /* Use the xhci_command structure from the first endpoint.  We may have
3377          * allocated too many, but the driver may call xhci_free_streams() for
3378          * each endpoint it grouped into one call to xhci_alloc_streams().
3379          */
3380         ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3381         command = vdev->eps[ep_index].stream_info->free_streams_command;
3382         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3383         if (!ctrl_ctx) {
3384                 spin_unlock_irqrestore(&xhci->lock, flags);
3385                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3386                                 __func__);
3387                 return -EINVAL;
3388         }
3389
3390         for (i = 0; i < num_eps; i++) {
3391                 struct xhci_ep_ctx *ep_ctx;
3392
3393                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3394                 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3395                 xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3396                         EP_GETTING_NO_STREAMS;
3397
3398                 xhci_endpoint_copy(xhci, command->in_ctx,
3399                                 vdev->out_ctx, ep_index);
3400                 xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3401                                 &vdev->eps[ep_index]);
3402         }
3403         xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3404                         vdev->out_ctx, ctrl_ctx,
3405                         changed_ep_bitmask, changed_ep_bitmask);
3406         spin_unlock_irqrestore(&xhci->lock, flags);
3407
3408         /* Issue and wait for the configure endpoint command,
3409          * which must succeed.
3410          */
3411         ret = xhci_configure_endpoint(xhci, udev, command,
3412                         false, true);
3413
3414         /* xHC rejected the configure endpoint command for some reason, so we
3415          * leave the streams rings intact.
3416          */
3417         if (ret < 0)
3418                 return ret;
3419
3420         spin_lock_irqsave(&xhci->lock, flags);
3421         for (i = 0; i < num_eps; i++) {
3422                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3423                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3424                 vdev->eps[ep_index].stream_info = NULL;
3425                 /* FIXME Unset maxPstreams in endpoint context and
3426                  * update deq ptr to point to normal string ring.
3427                  */
3428                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3429                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3430         }
3431         spin_unlock_irqrestore(&xhci->lock, flags);
3432
3433         return 0;
3434 }
3435
3436 /*
3437  * Deletes endpoint resources for endpoints that were active before a Reset
3438  * Device command, or a Disable Slot command.  The Reset Device command leaves
3439  * the control endpoint intact, whereas the Disable Slot command deletes it.
3440  *
3441  * Must be called with xhci->lock held.
3442  */
3443 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3444         struct xhci_virt_device *virt_dev, bool drop_control_ep)
3445 {
3446         int i;
3447         unsigned int num_dropped_eps = 0;
3448         unsigned int drop_flags = 0;
3449
3450         for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3451                 if (virt_dev->eps[i].ring) {
3452                         drop_flags |= 1 << i;
3453                         num_dropped_eps++;
3454                 }
3455         }
3456         xhci->num_active_eps -= num_dropped_eps;
3457         if (num_dropped_eps)
3458                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3459                                 "Dropped %u ep ctxs, flags = 0x%x, "
3460                                 "%u now active.",
3461                                 num_dropped_eps, drop_flags,
3462                                 xhci->num_active_eps);
3463 }
3464
3465 /*
3466  * This submits a Reset Device Command, which will set the device state to 0,
3467  * set the device address to 0, and disable all the endpoints except the default
3468  * control endpoint.  The USB core should come back and call
3469  * xhci_address_device(), and then re-set up the configuration.  If this is
3470  * called because of a usb_reset_and_verify_device(), then the old alternate
3471  * settings will be re-installed through the normal bandwidth allocation
3472  * functions.
3473  *
3474  * Wait for the Reset Device command to finish.  Remove all structures
3475  * associated with the endpoints that were disabled.  Clear the input device
3476  * structure? Reset the control endpoint 0 max packet size?
3477  *
3478  * If the virt_dev to be reset does not exist or does not match the udev,
3479  * it means the device is lost, possibly due to the xHC restore error and
3480  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3481  * re-allocate the device.
3482  */
3483 static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
3484                 struct usb_device *udev)
3485 {
3486         int ret, i;
3487         unsigned long flags;
3488         struct xhci_hcd *xhci;
3489         unsigned int slot_id;
3490         struct xhci_virt_device *virt_dev;
3491         struct xhci_command *reset_device_cmd;
3492         int last_freed_endpoint;
3493         struct xhci_slot_ctx *slot_ctx;
3494         int old_active_eps = 0;
3495
3496         ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3497         if (ret <= 0)
3498                 return ret;
3499         xhci = hcd_to_xhci(hcd);
3500         slot_id = udev->slot_id;
3501         virt_dev = xhci->devs[slot_id];
3502         if (!virt_dev) {
3503                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3504                                 "not exist. Re-allocate the device\n", slot_id);
3505                 ret = xhci_alloc_dev(hcd, udev);
3506                 if (ret == 1)
3507                         return 0;
3508                 else
3509                         return -EINVAL;
3510         }
3511
3512         if (virt_dev->tt_info)
3513                 old_active_eps = virt_dev->tt_info->active_eps;
3514
3515         if (virt_dev->udev != udev) {
3516                 /* If the virt_dev and the udev does not match, this virt_dev
3517                  * may belong to another udev.
3518                  * Re-allocate the device.
3519                  */
3520                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3521                                 "not match the udev. Re-allocate the device\n",
3522                                 slot_id);
3523                 ret = xhci_alloc_dev(hcd, udev);
3524                 if (ret == 1)
3525                         return 0;
3526                 else
3527                         return -EINVAL;
3528         }
3529
3530         /* If device is not setup, there is no point in resetting it */
3531         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3532         if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3533                                                 SLOT_STATE_DISABLED)
3534                 return 0;
3535
3536         trace_xhci_discover_or_reset_device(slot_ctx);
3537
3538         xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3539         /* Allocate the command structure that holds the struct completion.
3540          * Assume we're in process context, since the normal device reset
3541          * process has to wait for the device anyway.  Storage devices are
3542          * reset as part of error handling, so use GFP_NOIO instead of
3543          * GFP_KERNEL.
3544          */
3545         reset_device_cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
3546         if (!reset_device_cmd) {
3547                 xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3548                 return -ENOMEM;
3549         }
3550
3551         /* Attempt to submit the Reset Device command to the command ring */
3552         spin_lock_irqsave(&xhci->lock, flags);
3553
3554         ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3555         if (ret) {
3556                 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3557                 spin_unlock_irqrestore(&xhci->lock, flags);
3558                 goto command_cleanup;
3559         }
3560         xhci_ring_cmd_db(xhci);
3561         spin_unlock_irqrestore(&xhci->lock, flags);
3562
3563         /* Wait for the Reset Device command to finish */
3564         wait_for_completion(reset_device_cmd->completion);
3565
3566         /* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3567          * unless we tried to reset a slot ID that wasn't enabled,
3568          * or the device wasn't in the addressed or configured state.
3569          */
3570         ret = reset_device_cmd->status;
3571         switch (ret) {
3572         case COMP_COMMAND_ABORTED:
3573         case COMP_COMMAND_RING_STOPPED:
3574                 xhci_warn(xhci, "Timeout waiting for reset device command\n");
3575                 ret = -ETIME;
3576                 goto command_cleanup;
3577         case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */
3578         case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */
3579                 xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3580                                 slot_id,
3581                                 xhci_get_slot_state(xhci, virt_dev->out_ctx));
3582                 xhci_dbg(xhci, "Not freeing device rings.\n");
3583                 /* Don't treat this as an error.  May change my mind later. */
3584                 ret = 0;
3585                 goto command_cleanup;
3586         case COMP_SUCCESS:
3587                 xhci_dbg(xhci, "Successful reset device command.\n");
3588                 break;
3589         default:
3590                 if (xhci_is_vendor_info_code(xhci, ret))
3591                         break;
3592                 xhci_warn(xhci, "Unknown completion code %u for "
3593                                 "reset device command.\n", ret);
3594                 ret = -EINVAL;
3595                 goto command_cleanup;
3596         }
3597
3598         /* Free up host controller endpoint resources */
3599         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3600                 spin_lock_irqsave(&xhci->lock, flags);
3601                 /* Don't delete the default control endpoint resources */
3602                 xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3603                 spin_unlock_irqrestore(&xhci->lock, flags);
3604         }
3605
3606         /* Everything but endpoint 0 is disabled, so free the rings. */
3607         last_freed_endpoint = 1;
3608         for (i = 1; i < 31; i++) {
3609                 struct xhci_virt_ep *ep = &virt_dev->eps[i];
3610
3611                 if (ep->ep_state & EP_HAS_STREAMS) {
3612                         xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3613                                         xhci_get_endpoint_address(i));
3614                         xhci_free_stream_info(xhci, ep->stream_info);
3615                         ep->stream_info = NULL;
3616                         ep->ep_state &= ~EP_HAS_STREAMS;
3617                 }
3618
3619                 if (ep->ring) {
3620                         xhci_free_endpoint_ring(xhci, virt_dev, i);
3621                         last_freed_endpoint = i;
3622                 }
3623                 if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3624                         xhci_drop_ep_from_interval_table(xhci,
3625                                         &virt_dev->eps[i].bw_info,
3626                                         virt_dev->bw_table,
3627                                         udev,
3628                                         &virt_dev->eps[i],
3629                                         virt_dev->tt_info);
3630                 xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3631         }
3632         /* If necessary, update the number of active TTs on this root port */
3633         xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3634         ret = 0;
3635
3636 command_cleanup:
3637         xhci_free_command(xhci, reset_device_cmd);
3638         return ret;
3639 }
3640
3641 /*
3642  * At this point, the struct usb_device is about to go away, the device has
3643  * disconnected, and all traffic has been stopped and the endpoints have been
3644  * disabled.  Free any HC data structures associated with that device.
3645  */
3646 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3647 {
3648         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3649         struct xhci_virt_device *virt_dev;
3650         struct xhci_slot_ctx *slot_ctx;
3651         int i, ret;
3652
3653         /*
3654          * We called pm_runtime_get_noresume when the device was attached.
3655          * Decrement the counter here to allow controller to runtime suspend
3656          * if no devices remain.
3657          */
3658         if (xhci->quirks & XHCI_RESET_ON_RESUME)
3659                 pm_runtime_put_noidle(hcd->self.controller);
3660
3661         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3662         /* If the host is halted due to driver unload, we still need to free the
3663          * device.
3664          */
3665         if (ret <= 0 && ret != -ENODEV)
3666                 return;
3667
3668         virt_dev = xhci->devs[udev->slot_id];
3669         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3670         trace_xhci_free_dev(slot_ctx);
3671
3672         /* Stop any wayward timer functions (which may grab the lock) */
3673         for (i = 0; i < 31; i++) {
3674                 virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
3675                 del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
3676         }
3677
3678         virt_dev->udev = NULL;
3679         xhci_disable_slot(xhci, udev->slot_id);
3680         /*
3681          * Event command completion handler will free any data structures
3682          * associated with the slot.  XXX Can free sleep?
3683          */
3684 }
3685
3686 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
3687 {
3688         struct xhci_command *command;
3689         unsigned long flags;
3690         u32 state;
3691         int ret = 0;
3692
3693         command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
3694         if (!command)
3695                 return -ENOMEM;
3696
3697         spin_lock_irqsave(&xhci->lock, flags);
3698         /* Don't disable the slot if the host controller is dead. */
3699         state = readl(&xhci->op_regs->status);
3700         if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3701                         (xhci->xhc_state & XHCI_STATE_HALTED)) {
3702                 spin_unlock_irqrestore(&xhci->lock, flags);
3703                 kfree(command);
3704                 return -ENODEV;
3705         }
3706
3707         ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3708                                 slot_id);
3709         if (ret) {
3710                 spin_unlock_irqrestore(&xhci->lock, flags);
3711                 kfree(command);
3712                 return ret;
3713         }
3714         xhci_ring_cmd_db(xhci);
3715         spin_unlock_irqrestore(&xhci->lock, flags);
3716         return ret;
3717 }
3718
3719 /*
3720  * Checks if we have enough host controller resources for the default control
3721  * endpoint.
3722  *
3723  * Must be called with xhci->lock held.
3724  */
3725 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
3726 {
3727         if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
3728                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3729                                 "Not enough ep ctxs: "
3730                                 "%u active, need to add 1, limit is %u.",
3731                                 xhci->num_active_eps, xhci->limit_active_eps);
3732                 return -ENOMEM;
3733         }
3734         xhci->num_active_eps += 1;
3735         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3736                         "Adding 1 ep ctx, %u now active.",
3737                         xhci->num_active_eps);
3738         return 0;
3739 }
3740
3741
3742 /*
3743  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
3744  * timed out, or allocating memory failed.  Returns 1 on success.
3745  */
3746 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
3747 {
3748         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3749         struct xhci_virt_device *vdev;
3750         struct xhci_slot_ctx *slot_ctx;
3751         unsigned long flags;
3752         int ret, slot_id;
3753         struct xhci_command *command;
3754
3755         command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
3756         if (!command)
3757                 return 0;
3758
3759         /* xhci->slot_id and xhci->addr_dev are not thread-safe */
3760         mutex_lock(&xhci->mutex);
3761         spin_lock_irqsave(&xhci->lock, flags);
3762         ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
3763         if (ret) {
3764                 spin_unlock_irqrestore(&xhci->lock, flags);
3765                 mutex_unlock(&xhci->mutex);
3766                 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3767                 xhci_free_command(xhci, command);
3768                 return 0;
3769         }
3770         xhci_ring_cmd_db(xhci);
3771         spin_unlock_irqrestore(&xhci->lock, flags);
3772
3773         wait_for_completion(command->completion);
3774         slot_id = command->slot_id;
3775         mutex_unlock(&xhci->mutex);
3776
3777         if (!slot_id || command->status != COMP_SUCCESS) {
3778                 xhci_err(xhci, "Error while assigning device slot ID\n");
3779                 xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
3780                                 HCS_MAX_SLOTS(
3781                                         readl(&xhci->cap_regs->hcs_params1)));
3782                 xhci_free_command(xhci, command);
3783                 return 0;
3784         }
3785
3786         xhci_free_command(xhci, command);
3787
3788         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3789                 spin_lock_irqsave(&xhci->lock, flags);
3790                 ret = xhci_reserve_host_control_ep_resources(xhci);
3791                 if (ret) {
3792                         spin_unlock_irqrestore(&xhci->lock, flags);
3793                         xhci_warn(xhci, "Not enough host resources, "
3794                                         "active endpoint contexts = %u\n",
3795                                         xhci->num_active_eps);
3796                         goto disable_slot;
3797                 }
3798                 spin_unlock_irqrestore(&xhci->lock, flags);
3799         }
3800         /* Use GFP_NOIO, since this function can be called from
3801          * xhci_discover_or_reset_device(), which may be called as part of
3802          * mass storage driver error handling.
3803          */
3804         if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
3805                 xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
3806                 goto disable_slot;
3807         }
3808         vdev = xhci->devs[slot_id];
3809         slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
3810         trace_xhci_alloc_dev(slot_ctx);
3811
3812         udev->slot_id = slot_id;
3813
3814         /*
3815          * If resetting upon resume, we can't put the controller into runtime
3816          * suspend if there is a device attached.
3817          */
3818         if (xhci->quirks & XHCI_RESET_ON_RESUME)
3819                 pm_runtime_get_noresume(hcd->self.controller);
3820
3821         /* Is this a LS or FS device under a HS hub? */
3822         /* Hub or peripherial? */
3823         return 1;
3824
3825 disable_slot:
3826         return xhci_disable_slot(xhci, udev->slot_id);
3827 }
3828
3829 /*
3830  * Issue an Address Device command and optionally send a corresponding
3831  * SetAddress request to the device.
3832  */
3833 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
3834                              enum xhci_setup_dev setup)
3835 {
3836         const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
3837         unsigned long flags;
3838         struct xhci_virt_device *virt_dev;
3839         int ret = 0;
3840         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3841         struct xhci_slot_ctx *slot_ctx;
3842         struct xhci_input_control_ctx *ctrl_ctx;
3843         u64 temp_64;
3844         struct xhci_command *command = NULL;
3845
3846         mutex_lock(&xhci->mutex);
3847
3848         if (xhci->xhc_state) {  /* dying, removing or halted */
3849                 ret = -ESHUTDOWN;
3850                 goto out;
3851         }
3852
3853         if (!udev->slot_id) {
3854                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3855                                 "Bad Slot ID %d", udev->slot_id);
3856                 ret = -EINVAL;
3857                 goto out;
3858         }
3859
3860         virt_dev = xhci->devs[udev->slot_id];
3861
3862         if (WARN_ON(!virt_dev)) {
3863                 /*
3864                  * In plug/unplug torture test with an NEC controller,
3865                  * a zero-dereference was observed once due to virt_dev = 0.
3866                  * Print useful debug rather than crash if it is observed again!
3867                  */
3868                 xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
3869                         udev->slot_id);
3870                 ret = -EINVAL;
3871                 goto out;
3872         }
3873         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3874         trace_xhci_setup_device_slot(slot_ctx);
3875
3876         if (setup == SETUP_CONTEXT_ONLY) {
3877                 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3878                     SLOT_STATE_DEFAULT) {
3879                         xhci_dbg(xhci, "Slot already in default state\n");
3880                         goto out;
3881                 }
3882         }
3883
3884         command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
3885         if (!command) {
3886                 ret = -ENOMEM;
3887                 goto out;
3888         }
3889
3890         command->in_ctx = virt_dev->in_ctx;
3891
3892         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3893         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
3894         if (!ctrl_ctx) {
3895                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3896                                 __func__);
3897                 ret = -EINVAL;
3898                 goto out;
3899         }
3900         /*
3901          * If this is the first Set Address since device plug-in or
3902          * virt_device realloaction after a resume with an xHCI power loss,
3903          * then set up the slot context.
3904          */
3905         if (!slot_ctx->dev_info)
3906                 xhci_setup_addressable_virt_dev(xhci, udev);
3907         /* Otherwise, update the control endpoint ring enqueue pointer. */
3908         else
3909                 xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
3910         ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
3911         ctrl_ctx->drop_flags = 0;
3912
3913         trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3914                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
3915
3916         spin_lock_irqsave(&xhci->lock, flags);
3917         trace_xhci_setup_device(virt_dev);
3918         ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
3919                                         udev->slot_id, setup);
3920         if (ret) {
3921                 spin_unlock_irqrestore(&xhci->lock, flags);
3922                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3923                                 "FIXME: allocate a command ring segment");
3924                 goto out;
3925         }
3926         xhci_ring_cmd_db(xhci);
3927         spin_unlock_irqrestore(&xhci->lock, flags);
3928
3929         /* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
3930         wait_for_completion(command->completion);
3931
3932         /* FIXME: From section 4.3.4: "Software shall be responsible for timing
3933          * the SetAddress() "recovery interval" required by USB and aborting the
3934          * command on a timeout.
3935          */
3936         switch (command->status) {
3937         case COMP_COMMAND_ABORTED:
3938         case COMP_COMMAND_RING_STOPPED:
3939                 xhci_warn(xhci, "Timeout while waiting for setup device command\n");
3940                 ret = -ETIME;
3941                 break;
3942         case COMP_CONTEXT_STATE_ERROR:
3943         case COMP_SLOT_NOT_ENABLED_ERROR:
3944                 xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
3945                          act, udev->slot_id);
3946                 ret = -EINVAL;
3947                 break;
3948         case COMP_USB_TRANSACTION_ERROR:
3949                 dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
3950                 ret = -EPROTO;
3951                 break;
3952         case COMP_INCOMPATIBLE_DEVICE_ERROR:
3953                 dev_warn(&udev->dev,
3954                          "ERROR: Incompatible device for setup %s command\n", act);
3955                 ret = -ENODEV;
3956                 break;
3957         case COMP_SUCCESS:
3958                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3959                                "Successful setup %s command", act);
3960                 break;
3961         default:
3962                 xhci_err(xhci,
3963                          "ERROR: unexpected setup %s command completion code 0x%x.\n",
3964                          act, command->status);
3965                 trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
3966                 ret = -EINVAL;
3967                 break;
3968         }
3969         if (ret)
3970                 goto out;
3971         temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
3972         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3973                         "Op regs DCBAA ptr = %#016llx", temp_64);
3974         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3975                 "Slot ID %d dcbaa entry @%p = %#016llx",
3976                 udev->slot_id,
3977                 &xhci->dcbaa->dev_context_ptrs[udev->slot_id],
3978                 (unsigned long long)
3979                 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
3980         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3981                         "Output Context DMA address = %#08llx",
3982                         (unsigned long long)virt_dev->out_ctx->dma);
3983         trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3984                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
3985         /*
3986          * USB core uses address 1 for the roothubs, so we add one to the
3987          * address given back to us by the HC.
3988          */
3989         trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
3990                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
3991         /* Zero the input context control for later use */
3992         ctrl_ctx->add_flags = 0;
3993         ctrl_ctx->drop_flags = 0;
3994
3995         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3996                        "Internal device address = %d",
3997                        le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
3998 out:
3999         mutex_unlock(&xhci->mutex);
4000         if (command) {
4001                 kfree(command->completion);
4002                 kfree(command);
4003         }
4004         return ret;
4005 }
4006
4007 static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
4008 {
4009         return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
4010 }
4011
4012 static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
4013 {
4014         return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
4015 }
4016
4017 /*
4018  * Transfer the port index into real index in the HW port status
4019  * registers. Caculate offset between the port's PORTSC register
4020  * and port status base. Divide the number of per port register
4021  * to get the real index. The raw port number bases 1.
4022  */
4023 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
4024 {
4025         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4026         __le32 __iomem *base_addr = &xhci->op_regs->port_status_base;
4027         __le32 __iomem *addr;
4028         int raw_port;
4029
4030         if (hcd->speed < HCD_USB3)
4031                 addr = xhci->usb2_ports[port1 - 1];
4032         else
4033                 addr = xhci->usb3_ports[port1 - 1];
4034
4035         raw_port = (addr - base_addr)/NUM_PORT_REGS + 1;
4036         return raw_port;
4037 }
4038
4039 /*
4040  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
4041  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
4042  */
4043 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
4044                         struct usb_device *udev, u16 max_exit_latency)
4045 {
4046         struct xhci_virt_device *virt_dev;
4047         struct xhci_command *command;
4048         struct xhci_input_control_ctx *ctrl_ctx;
4049         struct xhci_slot_ctx *slot_ctx;
4050         unsigned long flags;
4051         int ret;
4052
4053         spin_lock_irqsave(&xhci->lock, flags);
4054
4055         virt_dev = xhci->devs[udev->slot_id];
4056
4057         /*
4058          * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
4059          * xHC was re-initialized. Exit latency will be set later after
4060          * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
4061          */
4062
4063         if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4064                 spin_unlock_irqrestore(&xhci->lock, flags);
4065                 return 0;
4066         }
4067
4068         /* Attempt to issue an Evaluate Context command to change the MEL. */
4069         command = xhci->lpm_command;
4070         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4071         if (!ctrl_ctx) {
4072                 spin_unlock_irqrestore(&xhci->lock, flags);
4073                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4074                                 __func__);
4075                 return -ENOMEM;
4076         }
4077
4078         xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4079         spin_unlock_irqrestore(&xhci->lock, flags);
4080
4081         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4082         slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4083         slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4084         slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4085         slot_ctx->dev_state = 0;
4086
4087         xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4088                         "Set up evaluate context for LPM MEL change.");
4089
4090         /* Issue and wait for the evaluate context command. */
4091         ret = xhci_configure_endpoint(xhci, udev, command,
4092                         true, true);
4093
4094         if (!ret) {
4095                 spin_lock_irqsave(&xhci->lock, flags);
4096                 virt_dev->current_mel = max_exit_latency;
4097                 spin_unlock_irqrestore(&xhci->lock, flags);
4098         }
4099         return ret;
4100 }
4101
4102 #ifdef CONFIG_PM
4103
4104 /* BESL to HIRD Encoding array for USB2 LPM */
4105 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4106         3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4107
4108 /* Calculate HIRD/BESL for USB2 PORTPMSC*/
4109 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4110                                         struct usb_device *udev)
4111 {
4112         int u2del, besl, besl_host;
4113         int besl_device = 0;
4114         u32 field;
4115
4116         u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4117         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4118
4119         if (field & USB_BESL_SUPPORT) {
4120                 for (besl_host = 0; besl_host < 16; besl_host++) {
4121                         if (xhci_besl_encoding[besl_host] >= u2del)
4122                                 break;
4123                 }
4124                 /* Use baseline BESL value as default */
4125                 if (field & USB_BESL_BASELINE_VALID)
4126                         besl_device = USB_GET_BESL_BASELINE(field);
4127                 else if (field & USB_BESL_DEEP_VALID)
4128                         besl_device = USB_GET_BESL_DEEP(field);
4129         } else {
4130                 if (u2del <= 50)
4131                         besl_host = 0;
4132                 else
4133                         besl_host = (u2del - 51) / 75 + 1;
4134         }
4135
4136         besl = besl_host + besl_device;
4137         if (besl > 15)
4138                 besl = 15;
4139
4140         return besl;
4141 }
4142
4143 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
4144 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4145 {
4146         u32 field;
4147         int l1;
4148         int besld = 0;
4149         int hirdm = 0;
4150
4151         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4152
4153         /* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4154         l1 = udev->l1_params.timeout / 256;
4155
4156         /* device has preferred BESLD */
4157         if (field & USB_BESL_DEEP_VALID) {
4158                 besld = USB_GET_BESL_DEEP(field);
4159                 hirdm = 1;
4160         }
4161
4162         return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4163 }
4164
4165 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4166                         struct usb_device *udev, int enable)
4167 {
4168         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4169         __le32 __iomem  **port_array;
4170         __le32 __iomem  *pm_addr, *hlpm_addr;
4171         u32             pm_val, hlpm_val, field;
4172         unsigned int    port_num;
4173         unsigned long   flags;
4174         int             hird, exit_latency;
4175         int             ret;
4176
4177         if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
4178                         !udev->lpm_capable)
4179                 return -EPERM;
4180
4181         if (!udev->parent || udev->parent->parent ||
4182                         udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4183                 return -EPERM;
4184
4185         if (udev->usb2_hw_lpm_capable != 1)
4186                 return -EPERM;
4187
4188         spin_lock_irqsave(&xhci->lock, flags);
4189
4190         port_array = xhci->usb2_ports;
4191         port_num = udev->portnum - 1;
4192         pm_addr = port_array[port_num] + PORTPMSC;
4193         pm_val = readl(pm_addr);
4194         hlpm_addr = port_array[port_num] + PORTHLPMC;
4195
4196         xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4197                         enable ? "enable" : "disable", port_num + 1);
4198
4199         if (enable) {
4200                 /* Host supports BESL timeout instead of HIRD */
4201                 if (udev->usb2_hw_lpm_besl_capable) {
4202                         /* if device doesn't have a preferred BESL value use a
4203                          * default one which works with mixed HIRD and BESL
4204                          * systems. See XHCI_DEFAULT_BESL definition in xhci.h
4205                          */
4206                         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4207                         if ((field & USB_BESL_SUPPORT) &&
4208                             (field & USB_BESL_BASELINE_VALID))
4209                                 hird = USB_GET_BESL_BASELINE(field);
4210                         else
4211                                 hird = udev->l1_params.besl;
4212
4213                         exit_latency = xhci_besl_encoding[hird];
4214                         spin_unlock_irqrestore(&xhci->lock, flags);
4215
4216                         /* USB 3.0 code dedicate one xhci->lpm_command->in_ctx
4217                          * input context for link powermanagement evaluate
4218                          * context commands. It is protected by hcd->bandwidth
4219                          * mutex and is shared by all devices. We need to set
4220                          * the max ext latency in USB 2 BESL LPM as well, so
4221                          * use the same mutex and xhci_change_max_exit_latency()
4222                          */
4223                         mutex_lock(hcd->bandwidth_mutex);
4224                         ret = xhci_change_max_exit_latency(xhci, udev,
4225                                                            exit_latency);
4226                         mutex_unlock(hcd->bandwidth_mutex);
4227
4228                         if (ret < 0)
4229                                 return ret;
4230                         spin_lock_irqsave(&xhci->lock, flags);
4231
4232                         hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4233                         writel(hlpm_val, hlpm_addr);
4234                         /* flush write */
4235                         readl(hlpm_addr);
4236                 } else {
4237                         hird = xhci_calculate_hird_besl(xhci, udev);
4238                 }
4239
4240                 pm_val &= ~PORT_HIRD_MASK;
4241                 pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4242                 writel(pm_val, pm_addr);
4243                 pm_val = readl(pm_addr);
4244                 pm_val |= PORT_HLE;
4245                 writel(pm_val, pm_addr);
4246                 /* flush write */
4247                 readl(pm_addr);
4248         } else {
4249                 pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4250                 writel(pm_val, pm_addr);
4251                 /* flush write */
4252                 readl(pm_addr);
4253                 if (udev->usb2_hw_lpm_besl_capable) {
4254                         spin_unlock_irqrestore(&xhci->lock, flags);
4255                         mutex_lock(hcd->bandwidth_mutex);
4256                         xhci_change_max_exit_latency(xhci, udev, 0);
4257                         mutex_unlock(hcd->bandwidth_mutex);
4258                         readl_poll_timeout(port_array[port_num], pm_val,
4259                                            (pm_val & PORT_PLS_MASK) == XDEV_U0,
4260                                            100, 10000);
4261                         return 0;
4262                 }
4263         }
4264
4265         spin_unlock_irqrestore(&xhci->lock, flags);
4266         return 0;
4267 }
4268
4269 /* check if a usb2 port supports a given extened capability protocol
4270  * only USB2 ports extended protocol capability values are cached.
4271  * Return 1 if capability is supported
4272  */
4273 static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
4274                                            unsigned capability)
4275 {
4276         u32 port_offset, port_count;
4277         int i;
4278
4279         for (i = 0; i < xhci->num_ext_caps; i++) {
4280                 if (xhci->ext_caps[i] & capability) {
4281                         /* port offsets starts at 1 */
4282                         port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
4283                         port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
4284                         if (port >= port_offset &&
4285                             port < port_offset + port_count)
4286                                 return 1;
4287                 }
4288         }
4289         return 0;
4290 }
4291
4292 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4293 {
4294         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4295         int             portnum = udev->portnum - 1;
4296
4297         if (hcd->speed >= HCD_USB3 || !xhci->sw_lpm_support ||
4298                         !udev->lpm_capable)
4299                 return 0;
4300
4301         /* we only support lpm for non-hub device connected to root hub yet */
4302         if (!udev->parent || udev->parent->parent ||
4303                         udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4304                 return 0;
4305
4306         if (xhci->hw_lpm_support == 1 &&
4307                         xhci_check_usb2_port_capability(
4308                                 xhci, portnum, XHCI_HLC)) {
4309                 udev->usb2_hw_lpm_capable = 1;
4310                 udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4311                 udev->l1_params.besl = XHCI_DEFAULT_BESL;
4312                 if (xhci_check_usb2_port_capability(xhci, portnum,
4313                                         XHCI_BLC))
4314                         udev->usb2_hw_lpm_besl_capable = 1;
4315         }
4316
4317         return 0;
4318 }
4319
4320 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4321
4322 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4323 static unsigned long long xhci_service_interval_to_ns(
4324                 struct usb_endpoint_descriptor *desc)
4325 {
4326         return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4327 }
4328
4329 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4330                 enum usb3_link_state state)
4331 {
4332         unsigned long long sel;
4333         unsigned long long pel;
4334         unsigned int max_sel_pel;
4335         char *state_name;
4336
4337         switch (state) {
4338         case USB3_LPM_U1:
4339                 /* Convert SEL and PEL stored in nanoseconds to microseconds */
4340                 sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4341                 pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4342                 max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4343                 state_name = "U1";
4344                 break;
4345         case USB3_LPM_U2:
4346                 sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4347                 pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4348                 max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4349                 state_name = "U2";
4350                 break;
4351         default:
4352                 dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4353                                 __func__);
4354                 return USB3_LPM_DISABLED;
4355         }
4356
4357         if (sel <= max_sel_pel && pel <= max_sel_pel)
4358                 return USB3_LPM_DEVICE_INITIATED;
4359
4360         if (sel > max_sel_pel)
4361                 dev_dbg(&udev->dev, "Device-initiated %s disabled "
4362                                 "due to long SEL %llu ms\n",
4363                                 state_name, sel);
4364         else
4365                 dev_dbg(&udev->dev, "Device-initiated %s disabled "
4366                                 "due to long PEL %llu ms\n",
4367                                 state_name, pel);
4368         return USB3_LPM_DISABLED;
4369 }
4370
4371 /* The U1 timeout should be the maximum of the following values:
4372  *  - For control endpoints, U1 system exit latency (SEL) * 3
4373  *  - For bulk endpoints, U1 SEL * 5
4374  *  - For interrupt endpoints:
4375  *    - Notification EPs, U1 SEL * 3
4376  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4377  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4378  */
4379 static unsigned long long xhci_calculate_intel_u1_timeout(
4380                 struct usb_device *udev,
4381                 struct usb_endpoint_descriptor *desc)
4382 {
4383         unsigned long long timeout_ns;
4384         int ep_type;
4385         int intr_type;
4386
4387         ep_type = usb_endpoint_type(desc);
4388         switch (ep_type) {
4389         case USB_ENDPOINT_XFER_CONTROL:
4390                 timeout_ns = udev->u1_params.sel * 3;
4391                 break;
4392         case USB_ENDPOINT_XFER_BULK:
4393                 timeout_ns = udev->u1_params.sel * 5;
4394                 break;
4395         case USB_ENDPOINT_XFER_INT:
4396                 intr_type = usb_endpoint_interrupt_type(desc);
4397                 if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4398                         timeout_ns = udev->u1_params.sel * 3;
4399                         break;
4400                 }
4401                 /* Otherwise the calculation is the same as isoc eps */
4402         case USB_ENDPOINT_XFER_ISOC:
4403                 timeout_ns = xhci_service_interval_to_ns(desc);
4404                 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4405                 if (timeout_ns < udev->u1_params.sel * 2)
4406                         timeout_ns = udev->u1_params.sel * 2;
4407                 break;
4408         default:
4409                 return 0;
4410         }
4411
4412         return timeout_ns;
4413 }
4414
4415 /* Returns the hub-encoded U1 timeout value. */
4416 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4417                 struct usb_device *udev,
4418                 struct usb_endpoint_descriptor *desc)
4419 {
4420         unsigned long long timeout_ns;
4421
4422         /* Prevent U1 if service interval is shorter than U1 exit latency */
4423         if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4424                 if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
4425                         dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
4426                         return USB3_LPM_DISABLED;
4427                 }
4428         }
4429
4430         if (xhci->quirks & XHCI_INTEL_HOST)
4431                 timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4432         else
4433                 timeout_ns = udev->u1_params.sel;
4434
4435         /* The U1 timeout is encoded in 1us intervals.
4436          * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
4437          */
4438         if (timeout_ns == USB3_LPM_DISABLED)
4439                 timeout_ns = 1;
4440         else
4441                 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4442
4443         /* If the necessary timeout value is bigger than what we can set in the
4444          * USB 3.0 hub, we have to disable hub-initiated U1.
4445          */
4446         if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4447                 return timeout_ns;
4448         dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4449                         "due to long timeout %llu ms\n", timeout_ns);
4450         return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4451 }
4452
4453 /* The U2 timeout should be the maximum of:
4454  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4455  *  - largest bInterval of any active periodic endpoint (to avoid going
4456  *    into lower power link states between intervals).
4457  *  - the U2 Exit Latency of the device
4458  */
4459 static unsigned long long xhci_calculate_intel_u2_timeout(
4460                 struct usb_device *udev,
4461                 struct usb_endpoint_descriptor *desc)
4462 {
4463         unsigned long long timeout_ns;
4464         unsigned long long u2_del_ns;
4465
4466         timeout_ns = 10 * 1000 * 1000;
4467
4468         if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4469                         (xhci_service_interval_to_ns(desc) > timeout_ns))
4470                 timeout_ns = xhci_service_interval_to_ns(desc);
4471
4472         u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4473         if (u2_del_ns > timeout_ns)
4474                 timeout_ns = u2_del_ns;
4475
4476         return timeout_ns;
4477 }
4478
4479 /* Returns the hub-encoded U2 timeout value. */
4480 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4481                 struct usb_device *udev,
4482                 struct usb_endpoint_descriptor *desc)
4483 {
4484         unsigned long long timeout_ns;
4485
4486         /* Prevent U2 if service interval is shorter than U2 exit latency */
4487         if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4488                 if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
4489                         dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
4490                         return USB3_LPM_DISABLED;
4491                 }
4492         }
4493
4494         if (xhci->quirks & XHCI_INTEL_HOST)
4495                 timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4496         else
4497                 timeout_ns = udev->u2_params.sel;
4498
4499         /* The U2 timeout is encoded in 256us intervals */
4500         timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4501         /* If the necessary timeout value is bigger than what we can set in the
4502          * USB 3.0 hub, we have to disable hub-initiated U2.
4503          */
4504         if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4505                 return timeout_ns;
4506         dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4507                         "due to long timeout %llu ms\n", timeout_ns);
4508         return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4509 }
4510
4511 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4512                 struct usb_device *udev,
4513                 struct usb_endpoint_descriptor *desc,
4514                 enum usb3_link_state state,
4515                 u16 *timeout)
4516 {
4517         if (state == USB3_LPM_U1)
4518                 return xhci_calculate_u1_timeout(xhci, udev, desc);
4519         else if (state == USB3_LPM_U2)
4520                 return xhci_calculate_u2_timeout(xhci, udev, desc);
4521
4522         return USB3_LPM_DISABLED;
4523 }
4524
4525 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4526                 struct usb_device *udev,
4527                 struct usb_endpoint_descriptor *desc,
4528                 enum usb3_link_state state,
4529                 u16 *timeout)
4530 {
4531         u16 alt_timeout;
4532
4533         alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4534                 desc, state, timeout);
4535
4536         /* If we found we can't enable hub-initiated LPM, and
4537          * the U1 or U2 exit latency was too high to allow
4538          * device-initiated LPM as well, then we will disable LPM
4539          * for this device, so stop searching any further.
4540          */
4541         if (alt_timeout == USB3_LPM_DISABLED) {
4542                 *timeout = alt_timeout;
4543                 return -E2BIG;
4544         }
4545         if (alt_timeout > *timeout)
4546                 *timeout = alt_timeout;
4547         return 0;
4548 }
4549
4550 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4551                 struct usb_device *udev,
4552                 struct usb_host_interface *alt,
4553                 enum usb3_link_state state,
4554                 u16 *timeout)
4555 {
4556         int j;
4557
4558         for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4559                 if (xhci_update_timeout_for_endpoint(xhci, udev,
4560                                         &alt->endpoint[j].desc, state, timeout))
4561                         return -E2BIG;
4562                 continue;
4563         }
4564         return 0;
4565 }
4566
4567 static int xhci_check_intel_tier_policy(struct usb_device *udev,
4568                 enum usb3_link_state state)
4569 {
4570         struct usb_device *parent;
4571         unsigned int num_hubs;
4572
4573         if (state == USB3_LPM_U2)
4574                 return 0;
4575
4576         /* Don't enable U1 if the device is on a 2nd tier hub or lower. */
4577         for (parent = udev->parent, num_hubs = 0; parent->parent;
4578                         parent = parent->parent)
4579                 num_hubs++;
4580
4581         if (num_hubs < 2)
4582                 return 0;
4583
4584         dev_dbg(&udev->dev, "Disabling U1 link state for device"
4585                         " below second-tier hub.\n");
4586         dev_dbg(&udev->dev, "Plug device into first-tier hub "
4587                         "to decrease power consumption.\n");
4588         return -E2BIG;
4589 }
4590
4591 static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4592                 struct usb_device *udev,
4593                 enum usb3_link_state state)
4594 {
4595         if (xhci->quirks & XHCI_INTEL_HOST)
4596                 return xhci_check_intel_tier_policy(udev, state);
4597         else
4598                 return 0;
4599 }
4600
4601 /* Returns the U1 or U2 timeout that should be enabled.
4602  * If the tier check or timeout setting functions return with a non-zero exit
4603  * code, that means the timeout value has been finalized and we shouldn't look
4604  * at any more endpoints.
4605  */
4606 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4607                         struct usb_device *udev, enum usb3_link_state state)
4608 {
4609         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4610         struct usb_host_config *config;
4611         char *state_name;
4612         int i;
4613         u16 timeout = USB3_LPM_DISABLED;
4614
4615         if (state == USB3_LPM_U1)
4616                 state_name = "U1";
4617         else if (state == USB3_LPM_U2)
4618                 state_name = "U2";
4619         else {
4620                 dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4621                                 state);
4622                 return timeout;
4623         }
4624
4625         if (xhci_check_tier_policy(xhci, udev, state) < 0)
4626                 return timeout;
4627
4628         /* Gather some information about the currently installed configuration
4629          * and alternate interface settings.
4630          */
4631         if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4632                         state, &timeout))
4633                 return timeout;
4634
4635         config = udev->actconfig;
4636         if (!config)
4637                 return timeout;
4638
4639         for (i = 0; i < config->desc.bNumInterfaces; i++) {
4640                 struct usb_driver *driver;
4641                 struct usb_interface *intf = config->interface[i];
4642
4643                 if (!intf)
4644                         continue;
4645
4646                 /* Check if any currently bound drivers want hub-initiated LPM
4647                  * disabled.
4648                  */
4649                 if (intf->dev.driver) {
4650                         driver = to_usb_driver(intf->dev.driver);
4651                         if (driver && driver->disable_hub_initiated_lpm) {
4652                                 dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
4653                                         state_name, driver->name);
4654                                 timeout = xhci_get_timeout_no_hub_lpm(udev,
4655                                                                       state);
4656                                 if (timeout == USB3_LPM_DISABLED)
4657                                         return timeout;
4658                         }
4659                 }
4660
4661                 /* Not sure how this could happen... */
4662                 if (!intf->cur_altsetting)
4663                         continue;
4664
4665                 if (xhci_update_timeout_for_interface(xhci, udev,
4666                                         intf->cur_altsetting,
4667                                         state, &timeout))
4668                         return timeout;
4669         }
4670         return timeout;
4671 }
4672
4673 static int calculate_max_exit_latency(struct usb_device *udev,
4674                 enum usb3_link_state state_changed,
4675                 u16 hub_encoded_timeout)
4676 {
4677         unsigned long long u1_mel_us = 0;
4678         unsigned long long u2_mel_us = 0;
4679         unsigned long long mel_us = 0;
4680         bool disabling_u1;
4681         bool disabling_u2;
4682         bool enabling_u1;
4683         bool enabling_u2;
4684
4685         disabling_u1 = (state_changed == USB3_LPM_U1 &&
4686                         hub_encoded_timeout == USB3_LPM_DISABLED);
4687         disabling_u2 = (state_changed == USB3_LPM_U2 &&
4688                         hub_encoded_timeout == USB3_LPM_DISABLED);
4689
4690         enabling_u1 = (state_changed == USB3_LPM_U1 &&
4691                         hub_encoded_timeout != USB3_LPM_DISABLED);
4692         enabling_u2 = (state_changed == USB3_LPM_U2 &&
4693                         hub_encoded_timeout != USB3_LPM_DISABLED);
4694
4695         /* If U1 was already enabled and we're not disabling it,
4696          * or we're going to enable U1, account for the U1 max exit latency.
4697          */
4698         if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4699                         enabling_u1)
4700                 u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4701         if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4702                         enabling_u2)
4703                 u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
4704
4705         if (u1_mel_us > u2_mel_us)
4706                 mel_us = u1_mel_us;
4707         else
4708                 mel_us = u2_mel_us;
4709         /* xHCI host controller max exit latency field is only 16 bits wide. */
4710         if (mel_us > MAX_EXIT) {
4711                 dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
4712                                 "is too big.\n", mel_us);
4713                 return -E2BIG;
4714         }
4715         return mel_us;
4716 }
4717
4718 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
4719 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
4720                         struct usb_device *udev, enum usb3_link_state state)
4721 {
4722         struct xhci_hcd *xhci;
4723         u16 hub_encoded_timeout;
4724         int mel;
4725         int ret;
4726
4727         xhci = hcd_to_xhci(hcd);
4728         /* The LPM timeout values are pretty host-controller specific, so don't
4729          * enable hub-initiated timeouts unless the vendor has provided
4730          * information about their timeout algorithm.
4731          */
4732         if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4733                         !xhci->devs[udev->slot_id])
4734                 return USB3_LPM_DISABLED;
4735
4736         hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
4737         mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
4738         if (mel < 0) {
4739                 /* Max Exit Latency is too big, disable LPM. */
4740                 hub_encoded_timeout = USB3_LPM_DISABLED;
4741                 mel = 0;
4742         }
4743
4744         ret = xhci_change_max_exit_latency(xhci, udev, mel);
4745         if (ret)
4746                 return ret;
4747         return hub_encoded_timeout;
4748 }
4749
4750 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
4751                         struct usb_device *udev, enum usb3_link_state state)
4752 {
4753         struct xhci_hcd *xhci;
4754         u16 mel;
4755
4756         xhci = hcd_to_xhci(hcd);
4757         if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4758                         !xhci->devs[udev->slot_id])
4759                 return 0;
4760
4761         mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
4762         return xhci_change_max_exit_latency(xhci, udev, mel);
4763 }
4764 #else /* CONFIG_PM */
4765
4766 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4767                                 struct usb_device *udev, int enable)
4768 {
4769         return 0;
4770 }
4771
4772 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4773 {
4774         return 0;
4775 }
4776
4777 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
4778                         struct usb_device *udev, enum usb3_link_state state)
4779 {
4780         return USB3_LPM_DISABLED;
4781 }
4782
4783 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
4784                         struct usb_device *udev, enum usb3_link_state state)
4785 {
4786         return 0;
4787 }
4788 #endif  /* CONFIG_PM */
4789
4790 /*-------------------------------------------------------------------------*/
4791
4792 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
4793  * internal data structures for the device.
4794  */
4795 static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
4796                         struct usb_tt *tt, gfp_t mem_flags)
4797 {
4798         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4799         struct xhci_virt_device *vdev;
4800         struct xhci_command *config_cmd;
4801         struct xhci_input_control_ctx *ctrl_ctx;
4802         struct xhci_slot_ctx *slot_ctx;
4803         unsigned long flags;
4804         unsigned think_time;
4805         int ret;
4806
4807         /* Ignore root hubs */
4808         if (!hdev->parent)
4809                 return 0;
4810
4811         vdev = xhci->devs[hdev->slot_id];
4812         if (!vdev) {
4813                 xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
4814                 return -EINVAL;
4815         }
4816
4817         config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
4818         if (!config_cmd)
4819                 return -ENOMEM;
4820
4821         ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
4822         if (!ctrl_ctx) {
4823                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4824                                 __func__);
4825                 xhci_free_command(xhci, config_cmd);
4826                 return -ENOMEM;
4827         }
4828
4829         spin_lock_irqsave(&xhci->lock, flags);
4830         if (hdev->speed == USB_SPEED_HIGH &&
4831                         xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
4832                 xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
4833                 xhci_free_command(xhci, config_cmd);
4834                 spin_unlock_irqrestore(&xhci->lock, flags);
4835                 return -ENOMEM;
4836         }
4837
4838         xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
4839         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4840         slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
4841         slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
4842         /*
4843          * refer to section 6.2.2: MTT should be 0 for full speed hub,
4844          * but it may be already set to 1 when setup an xHCI virtual
4845          * device, so clear it anyway.
4846          */
4847         if (tt->multi)
4848                 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
4849         else if (hdev->speed == USB_SPEED_FULL)
4850                 slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
4851
4852         if (xhci->hci_version > 0x95) {
4853                 xhci_dbg(xhci, "xHCI version %x needs hub "
4854                                 "TT think time and number of ports\n",
4855                                 (unsigned int) xhci->hci_version);
4856                 slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
4857                 /* Set TT think time - convert from ns to FS bit times.
4858                  * 0 = 8 FS bit times, 1 = 16 FS bit times,
4859                  * 2 = 24 FS bit times, 3 = 32 FS bit times.
4860                  *
4861                  * xHCI 1.0: this field shall be 0 if the device is not a
4862                  * High-spped hub.
4863                  */
4864                 think_time = tt->think_time;
4865                 if (think_time != 0)
4866                         think_time = (think_time / 666) - 1;
4867                 if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
4868                         slot_ctx->tt_info |=
4869                                 cpu_to_le32(TT_THINK_TIME(think_time));
4870         } else {
4871                 xhci_dbg(xhci, "xHCI version %x doesn't need hub "
4872                                 "TT think time or number of ports\n",
4873                                 (unsigned int) xhci->hci_version);
4874         }
4875         slot_ctx->dev_state = 0;
4876         spin_unlock_irqrestore(&xhci->lock, flags);
4877
4878         xhci_dbg(xhci, "Set up %s for hub device.\n",
4879                         (xhci->hci_version > 0x95) ?
4880                         "configure endpoint" : "evaluate context");
4881
4882         /* Issue and wait for the configure endpoint or
4883          * evaluate context command.
4884          */
4885         if (xhci->hci_version > 0x95)
4886                 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
4887                                 false, false);
4888         else
4889                 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
4890                                 true, false);
4891
4892         xhci_free_command(xhci, config_cmd);
4893         return ret;
4894 }
4895
4896 static int xhci_get_frame(struct usb_hcd *hcd)
4897 {
4898         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4899         /* EHCI mods by the periodic size.  Why? */
4900         return readl(&xhci->run_regs->microframe_index) >> 3;
4901 }
4902
4903 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
4904 {
4905         struct xhci_hcd         *xhci;
4906         /*
4907          * TODO: Check with DWC3 clients for sysdev according to
4908          * quirks
4909          */
4910         struct device           *dev = hcd->self.sysdev;
4911         unsigned int            minor_rev;
4912         int                     retval;
4913
4914         /* Accept arbitrarily long scatter-gather lists */
4915         hcd->self.sg_tablesize = ~0;
4916
4917         /* support to build packet from discontinuous buffers */
4918         hcd->self.no_sg_constraint = 1;
4919
4920         /* XHCI controllers don't stop the ep queue on short packets :| */
4921         hcd->self.no_stop_on_short = 1;
4922
4923         xhci = hcd_to_xhci(hcd);
4924
4925         if (usb_hcd_is_primary_hcd(hcd)) {
4926                 xhci->main_hcd = hcd;
4927                 /* Mark the first roothub as being USB 2.0.
4928                  * The xHCI driver will register the USB 3.0 roothub.
4929                  */
4930                 hcd->speed = HCD_USB2;
4931                 hcd->self.root_hub->speed = USB_SPEED_HIGH;
4932                 /*
4933                  * USB 2.0 roothub under xHCI has an integrated TT,
4934                  * (rate matching hub) as opposed to having an OHCI/UHCI
4935                  * companion controller.
4936                  */
4937                 hcd->has_tt = 1;
4938         } else {
4939                 /*
4940                  * Some 3.1 hosts return sbrn 0x30, use xhci supported protocol
4941                  * minor revision instead of sbrn
4942                  */
4943                 minor_rev = xhci->usb3_rhub.min_rev;
4944                 if (minor_rev) {
4945                         hcd->speed = HCD_USB31;
4946                         hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
4947                 }
4948                 xhci_info(xhci, "Host supports USB 3.%x %s SuperSpeed\n",
4949                           minor_rev,
4950                           minor_rev ? "Enhanced" : "");
4951
4952                 /* xHCI private pointer was set in xhci_pci_probe for the second
4953                  * registered roothub.
4954                  */
4955                 return 0;
4956         }
4957
4958         mutex_init(&xhci->mutex);
4959         xhci->cap_regs = hcd->regs;
4960         xhci->op_regs = hcd->regs +
4961                 HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
4962         xhci->run_regs = hcd->regs +
4963                 (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
4964         /* Cache read-only capability registers */
4965         xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
4966         xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
4967         xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
4968         xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
4969         xhci->hci_version = HC_VERSION(xhci->hcc_params);
4970         xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
4971         if (xhci->hci_version > 0x100)
4972                 xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
4973         xhci_print_registers(xhci);
4974
4975         xhci->quirks |= quirks;
4976
4977         get_quirks(dev, xhci);
4978
4979         /* In xhci controllers which follow xhci 1.0 spec gives a spurious
4980          * success event after a short transfer. This quirk will ignore such
4981          * spurious event.
4982          */
4983         if (xhci->hci_version > 0x96)
4984                 xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
4985
4986         /* Make sure the HC is halted. */
4987         retval = xhci_halt(xhci);
4988         if (retval)
4989                 return retval;
4990
4991         xhci_dbg(xhci, "Resetting HCD\n");
4992         /* Reset the internal HC memory state and registers. */
4993         retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);
4994         if (retval)
4995                 return retval;
4996         xhci_dbg(xhci, "Reset complete\n");
4997
4998         /*
4999          * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
5000          * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
5001          * address memory pointers actually. So, this driver clears the AC64
5002          * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
5003          * DMA_BIT_MASK(32)) in this xhci_gen_setup().
5004          */
5005         if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
5006                 xhci->hcc_params &= ~BIT(0);
5007
5008         /* Set dma_mask and coherent_dma_mask to 64-bits,
5009          * if xHC supports 64-bit addressing */
5010         if (HCC_64BIT_ADDR(xhci->hcc_params) &&
5011                         !dma_set_mask(dev, DMA_BIT_MASK(64))) {
5012                 xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
5013                 dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
5014         } else {
5015                 /*
5016                  * This is to avoid error in cases where a 32-bit USB
5017                  * controller is used on a 64-bit capable system.
5018                  */
5019                 retval = dma_set_mask(dev, DMA_BIT_MASK(32));
5020                 if (retval)
5021                         return retval;
5022                 xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
5023                 dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
5024         }
5025
5026         xhci_dbg(xhci, "Calling HCD init\n");
5027         /* Initialize HCD and host controller data structures. */
5028         retval = xhci_init(hcd);
5029         if (retval)
5030                 return retval;
5031         xhci_dbg(xhci, "Called HCD init\n");
5032
5033         xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
5034                   xhci->hcc_params, xhci->hci_version, xhci->quirks);
5035
5036         return 0;
5037 }
5038 EXPORT_SYMBOL_GPL(xhci_gen_setup);
5039
5040 static const struct hc_driver xhci_hc_driver = {
5041         .description =          "xhci-hcd",
5042         .product_desc =         "xHCI Host Controller",
5043         .hcd_priv_size =        sizeof(struct xhci_hcd),
5044
5045         /*
5046          * generic hardware linkage
5047          */
5048         .irq =                  xhci_irq,
5049         .flags =                HCD_MEMORY | HCD_USB3 | HCD_SHARED,
5050
5051         /*
5052          * basic lifecycle operations
5053          */
5054         .reset =                NULL, /* set in xhci_init_driver() */
5055         .start =                xhci_run,
5056         .stop =                 xhci_stop,
5057         .shutdown =             xhci_shutdown,
5058
5059         /*
5060          * managing i/o requests and associated device resources
5061          */
5062         .urb_enqueue =          xhci_urb_enqueue,
5063         .urb_dequeue =          xhci_urb_dequeue,
5064         .alloc_dev =            xhci_alloc_dev,
5065         .free_dev =             xhci_free_dev,
5066         .alloc_streams =        xhci_alloc_streams,
5067         .free_streams =         xhci_free_streams,
5068         .add_endpoint =         xhci_add_endpoint,
5069         .drop_endpoint =        xhci_drop_endpoint,
5070         .endpoint_reset =       xhci_endpoint_reset,
5071         .check_bandwidth =      xhci_check_bandwidth,
5072         .reset_bandwidth =      xhci_reset_bandwidth,
5073         .address_device =       xhci_address_device,
5074         .enable_device =        xhci_enable_device,
5075         .update_hub_device =    xhci_update_hub_device,
5076         .reset_device =         xhci_discover_or_reset_device,
5077
5078         /*
5079          * scheduling support
5080          */
5081         .get_frame_number =     xhci_get_frame,
5082
5083         /*
5084          * root hub support
5085          */
5086         .hub_control =          xhci_hub_control,
5087         .hub_status_data =      xhci_hub_status_data,
5088         .bus_suspend =          xhci_bus_suspend,
5089         .bus_resume =           xhci_bus_resume,
5090
5091         /*
5092          * call back when device connected and addressed
5093          */
5094         .update_device =        xhci_update_device,
5095         .set_usb2_hw_lpm =      xhci_set_usb2_hardware_lpm,
5096         .enable_usb3_lpm_timeout =      xhci_enable_usb3_lpm_timeout,
5097         .disable_usb3_lpm_timeout =     xhci_disable_usb3_lpm_timeout,
5098         .find_raw_port_number = xhci_find_raw_port_number,
5099 };
5100
5101 void xhci_init_driver(struct hc_driver *drv,
5102                       const struct xhci_driver_overrides *over)
5103 {
5104         BUG_ON(!over);
5105
5106         /* Copy the generic table to drv then apply the overrides */
5107         *drv = xhci_hc_driver;
5108
5109         if (over) {
5110                 drv->hcd_priv_size += over->extra_priv_size;
5111                 if (over->reset)
5112                         drv->reset = over->reset;
5113                 if (over->start)
5114                         drv->start = over->start;
5115         }
5116 }
5117 EXPORT_SYMBOL_GPL(xhci_init_driver);
5118
5119 MODULE_DESCRIPTION(DRIVER_DESC);
5120 MODULE_AUTHOR(DRIVER_AUTHOR);
5121 MODULE_LICENSE("GPL");
5122
5123 static int __init xhci_hcd_init(void)
5124 {
5125         /*
5126          * Check the compiler generated sizes of structures that must be laid
5127          * out in specific ways for hardware access.
5128          */
5129         BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5130         BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5131         BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5132         /* xhci_device_control has eight fields, and also
5133          * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
5134          */
5135         BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5136         BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5137         BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5138         BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5139         BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5140         /* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
5141         BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5142
5143         if (usb_disabled())
5144                 return -ENODEV;
5145
5146         return 0;
5147 }
5148
5149 /*
5150  * If an init function is provided, an exit function must also be provided
5151  * to allow module unload.
5152  */
5153 static void __exit xhci_hcd_fini(void) { }
5154
5155 module_init(xhci_hcd_init);
5156 module_exit(xhci_hcd_fini);