drivers/staging/octeon-usb/octeon-hcd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file is subject to the terms and conditions of the GNU General Public
   4  * License.  See the file "COPYING" in the main directory of this archive
   5  * for more details.
   6  *
   7  * Copyright (C) 2008 Cavium Networks
   8  *
   9  * Some parts of the code were originally released under BSD license:
  10  *
  11  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  12  * reserved.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met:
  17  *
  18  *   * Redistributions of source code must retain the above copyright
  19  *     notice, this list of conditions and the following disclaimer.
  20  *
  21  *   * Redistributions in binary form must reproduce the above
  22  *     copyright notice, this list of conditions and the following
  23  *     disclaimer in the documentation and/or other materials provided
  24  *     with the distribution.
  25  *
  26  *   * Neither the name of Cavium Networks nor the names of
  27  *     its contributors may be used to endorse or promote products
  28  *     derived from this software without specific prior written
  29  *     permission.
  30  *
  31  * This Software, including technical data, may be subject to U.S. export
  32  * control laws, including the U.S. Export Administration Act and its associated
  33  * regulations, and may be subject to export or import regulations in other
  34  * countries.
  35  *
  36  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  37  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  38  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  39  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  40  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  41  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  42  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  43  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  44  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  45  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  46  */
  47
  48 #include <linux/usb.h>
  49 #include <linux/slab.h>
  50 #include <linux/module.h>
  51 #include <linux/usb/hcd.h>
  52 #include <linux/prefetch.h>
  53 #include <linux/irqdomain.h>
  54 #include <linux/dma-mapping.h>
  55 #include <linux/platform_device.h>
  56 #include <linux/of.h>
  57
  58 #include <asm/octeon/octeon.h>
  59
  60 #include "octeon-hcd.h"
  61
  62 /**
  63  * enum cvmx_usb_speed - the possible USB device speeds
  64  *
  65  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  66  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  67  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  68  */
  69 enum cvmx_usb_speed {
  70         CVMX_USB_SPEED_HIGH = 0,
  71         CVMX_USB_SPEED_FULL = 1,
  72         CVMX_USB_SPEED_LOW = 2,
  73 };
  74
  75 /**
  76  * enum cvmx_usb_transfer - the possible USB transfer types
  77  *
  78  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  79  *                                 transfers
  80  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  81  *                                 priority periodic transfers
  82  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  83  *                                 transfers
  84  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  85  *                                 periodic transfers
  86  */
  87 enum cvmx_usb_transfer {
  88         CVMX_USB_TRANSFER_CONTROL = 0,
  89         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
  90         CVMX_USB_TRANSFER_BULK = 2,
  91         CVMX_USB_TRANSFER_INTERRUPT = 3,
  92 };
  93
  94 /**
  95  * enum cvmx_usb_direction - the transfer directions
  96  *
  97  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
  98  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
  99  */
 100 enum cvmx_usb_direction {
 101         CVMX_USB_DIRECTION_OUT,
 102         CVMX_USB_DIRECTION_IN,
 103 };
 104
 105 /**
 106  * enum cvmx_usb_status - possible callback function status codes
 107  *
 108  * @CVMX_USB_STATUS_OK:           The transaction / operation finished without
 109  *                                any errors
 110  * @CVMX_USB_STATUS_SHORT:        FIXME: This is currently not implemented
 111  * @CVMX_USB_STATUS_CANCEL:       The transaction was canceled while in flight
 112  *                                by a user call to cvmx_usb_cancel
 113  * @CVMX_USB_STATUS_ERROR:        The transaction aborted with an unexpected
 114  *                                error status
 115  * @CVMX_USB_STATUS_STALL:        The transaction received a USB STALL response
 116  *                                from the device
 117  * @CVMX_USB_STATUS_XACTERR:      The transaction failed with an error from the
 118  *                                device even after a number of retries
 119  * @CVMX_USB_STATUS_DATATGLERR:   The transaction failed with a data toggle
 120  *                                error even after a number of retries
 121  * @CVMX_USB_STATUS_BABBLEERR:    The transaction failed with a babble error
 122  * @CVMX_USB_STATUS_FRAMEERR:     The transaction failed with a frame error
 123  *                                even after a number of retries
 124  */
 125 enum cvmx_usb_status {
 126         CVMX_USB_STATUS_OK,
 127         CVMX_USB_STATUS_SHORT,
 128         CVMX_USB_STATUS_CANCEL,
 129         CVMX_USB_STATUS_ERROR,
 130         CVMX_USB_STATUS_STALL,
 131         CVMX_USB_STATUS_XACTERR,
 132         CVMX_USB_STATUS_DATATGLERR,
 133         CVMX_USB_STATUS_BABBLEERR,
 134         CVMX_USB_STATUS_FRAMEERR,
 135 };
 136
 137 /**
 138  * struct cvmx_usb_port_status - the USB port status information
 139  *
 140  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 141  * @port_over_current:  1 = Over current detected, 0 = Over current not
 142  *                      detected. Octeon doesn't support over current detection.
 143  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 144  *                      power is off. Octeon doesn't support turning port power
 145  *                      off.
 146  * @port_speed:         Current port speed.
 147  * @connected:          1 = A device is connected to the port, 0 = No device is
 148  *                      connected.
 149  * @connect_change:     1 = Device connected state changed since the last set
 150  *                      status call.
 151  */
 152 struct cvmx_usb_port_status {
 153         u32 reserved                    : 25;
 154         u32 port_enabled                : 1;
 155         u32 port_over_current           : 1;
 156         u32 port_powered                : 1;
 157         enum cvmx_usb_speed port_speed  : 2;
 158         u32 connected                   : 1;
 159         u32 connect_change              : 1;
 160 };
 161
 162 /**
 163  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 164  *
 165  * @offset:     This is the offset in bytes into the main buffer where this data
 166  *              is stored.
 167  * @length:     This is the length in bytes of the data.
 168  * @status:     This is the status of this individual packet transfer.
 169  */
 170 struct cvmx_usb_iso_packet {
 171         int offset;
 172         int length;
 173         enum cvmx_usb_status status;
 174 };
 175
 176 /**
 177  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 178  *
 179  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 180  *                                            as clock source at USB_XO and
 181  *                                            USB_XI.
 182  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 183  *                                            board clock source at USB_XO.
 184  *                                            USB_XI should be tied to GND.
 185  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 186  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 187  *                                            crystal
 188  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 190  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 191  *                                            data transfer use for the USB
 192  */
 193 enum cvmx_usb_initialize_flags {
 194         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 195         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 196         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 197         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 198         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 199         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 200         /* Bits 3-4 used to encode the clock frequency */
 201         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 202 };
 203
 204 /**
 205  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 206  *
 207  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 208  *                                 actively using hardware.
 209  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
 210  *                                 pipe is in the ping state.
 211  */
 212 enum cvmx_usb_pipe_flags {
 213         CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
 214         CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
 215 };
 216
 217 /* Maximum number of times to retry failed transactions */
 218 #define MAX_RETRIES             3
 219
 220 /* Maximum number of hardware channels supported by the USB block */
 221 #define MAX_CHANNELS            8
 222
 223 /*
 224  * The low level hardware can transfer a maximum of this number of bytes in each
 225  * transfer. The field is 19 bits wide
 226  */
 227 #define MAX_TRANSFER_BYTES      ((1 << 19) - 1)
 228
 229 /*
 230  * The low level hardware can transfer a maximum of this number of packets in
 231  * each transfer. The field is 10 bits wide
 232  */
 233 #define MAX_TRANSFER_PACKETS    ((1 << 10) - 1)
 234
 235 /**
 236  * Logical transactions may take numerous low level
 237  * transactions, especially when splits are concerned. This
 238  * enum represents all of the possible stages a transaction can
 239  * be in. Note that split completes are always even. This is so
 240  * the NAK handler can backup to the previous low level
 241  * transaction with a simple clearing of bit 0.
 242  */
 243 enum cvmx_usb_stage {
 244         CVMX_USB_STAGE_NON_CONTROL,
 245         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 246         CVMX_USB_STAGE_SETUP,
 247         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 248         CVMX_USB_STAGE_DATA,
 249         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 250         CVMX_USB_STAGE_STATUS,
 251         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 252 };
 253
 254 /**
 255  * struct cvmx_usb_transaction - describes each pending USB transaction
 256  *                               regardless of type. These are linked together
 257  *                               to form a list of pending requests for a pipe.
 258  *
 259  * @node:               List node for transactions in the pipe.
 260  * @type:               Type of transaction, duplicated of the pipe.
 261  * @flags:              State flags for this transaction.
 262  * @buffer:             User's physical buffer address to read/write.
 263  * @buffer_length:      Size of the user's buffer in bytes.
 264  * @control_header:     For control transactions, physical address of the 8
 265  *                      byte standard header.
 266  * @iso_start_frame:    For ISO transactions, the starting frame number.
 267  * @iso_number_packets: For ISO transactions, the number of packets in the
 268  *                      request.
 269  * @iso_packets:        For ISO transactions, the sub packets in the request.
 270  * @actual_bytes:       Actual bytes transfer for this transaction.
 271  * @stage:              For control transactions, the current stage.
 272  * @urb:                URB.
 273  */
 274 struct cvmx_usb_transaction {
 275         struct list_head node;
 276         enum cvmx_usb_transfer type;
 277         u64 buffer;
 278         int buffer_length;
 279         u64 control_header;
 280         int iso_start_frame;
 281         int iso_number_packets;
 282         struct cvmx_usb_iso_packet *iso_packets;
 283         int xfersize;
 284         int pktcnt;
 285         int retries;
 286         int actual_bytes;
 287         enum cvmx_usb_stage stage;
 288         struct urb *urb;
 289 };
 290
 291 /**
 292  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 293  *                        and some USB device. It contains a list of pending
 294  *                        request to the device.
 295  *
 296  * @node:               List node for pipe list
 297  * @next:               Pipe after this one in the list
 298  * @transactions:       List of pending transactions
 299  * @interval:           For periodic pipes, the interval between packets in
 300  *                      frames
 301  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 302  * @flags:              State flags for this pipe
 303  * @device_speed:       Speed of device connected to this pipe
 304  * @transfer_type:      Type of transaction supported by this pipe
 305  * @transfer_dir:       IN or OUT. Ignored for Control
 306  * @multi_count:        Max packet in a row for the device
 307  * @max_packet:         The device's maximum packet size in bytes
 308  * @device_addr:        USB device address at other end of pipe
 309  * @endpoint_num:       USB endpoint number at other end of pipe
 310  * @hub_device_addr:    Hub address this device is connected to
 311  * @hub_port:           Hub port this device is connected to
 312  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 313  *                      the data pid needed
 314  * @channel:            Hardware DMA channel for this pipe
 315  * @split_sc_frame:     The low order bits of the frame number the split
 316  *                      complete should be sent on
 317  */
 318 struct cvmx_usb_pipe {
 319         struct list_head node;
 320         struct list_head transactions;
 321         u64 interval;
 322         u64 next_tx_frame;
 323         enum cvmx_usb_pipe_flags flags;
 324         enum cvmx_usb_speed device_speed;
 325         enum cvmx_usb_transfer transfer_type;
 326         enum cvmx_usb_direction transfer_dir;
 327         int multi_count;
 328         u16 max_packet;
 329         u8 device_addr;
 330         u8 endpoint_num;
 331         u8 hub_device_addr;
 332         u8 hub_port;
 333         u8 pid_toggle;
 334         u8 channel;
 335         s8 split_sc_frame;
 336 };
 337
 338 struct cvmx_usb_tx_fifo {
 339         struct {
 340                 int channel;
 341                 int size;
 342                 u64 address;
 343         } entry[MAX_CHANNELS + 1];
 344         int head;
 345         int tail;
 346 };
 347
 348 /**
 349  * struct octeon_hcd - the state of the USB block
 350  *
 351  * lock:                   Serialization lock.
 352  * init_flags:             Flags passed to initialize.
 353  * index:                  Which USB block this is for.
 354  * idle_hardware_channels: Bit set for every idle hardware channel.
 355  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 356  *                         determine splits.
 357  * pipe_for_channel:       Map channels to pipes.
 358  * pipe:                   Storage for pipes.
 359  * indent:                 Used by debug output to indent functions.
 360  * port_status:            Last port status used for change notification.
 361  * idle_pipes:             List of open pipes that have no transactions.
 362  * active_pipes:           Active pipes indexed by transfer type.
 363  * frame_number:           Increments every SOF interrupt for time keeping.
 364  * active_split:           Points to the current active split, or NULL.
 365  */
 366 struct octeon_hcd {
 367         spinlock_t lock; /* serialization lock */
 368         int init_flags;
 369         int index;
 370         int idle_hardware_channels;
 371         union cvmx_usbcx_hprt usbcx_hprt;
 372         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 373         int indent;
 374         struct cvmx_usb_port_status port_status;
 375         struct list_head idle_pipes;
 376         struct list_head active_pipes[4];
 377         u64 frame_number;
 378         struct cvmx_usb_transaction *active_split;
 379         struct cvmx_usb_tx_fifo periodic;
 380         struct cvmx_usb_tx_fifo nonperiodic;
 381 };
 382
 383 /*
 384  * This macro logically sets a single field in a CSR. It does the sequence
 385  * read, modify, and write
 386  */
 387 #define USB_SET_FIELD32(address, _union, field, value)          \
 388         do {                                                    \
 389                 union _union c;                                 \
 390                                                                 \
 391                 c.u32 = cvmx_usb_read_csr32(usb, address);      \
 392                 c.s.field = value;                              \
 393                 cvmx_usb_write_csr32(usb, address, c.u32);      \
 394         } while (0)
 395
 396 /* Returns the IO address to push/pop stuff data from the FIFOs */
 397 #define USB_FIFO_ADDRESS(channel, usb_index) \
 398         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel) + 1) * 0x1000)
 399
 400 /**
 401  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 402  * @orig_buffer: the original buffer passed by the USB stack
 403  * @data:        the newly allocated temporary buffer (excluding meta-data)
 404  *
 405  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 406  * the buffer is too short, we need to allocate a temporary one, and this struct
 407  * represents it.
 408  */
 409 struct octeon_temp_buffer {
 410         void *orig_buffer;
 411         u8 data[];
 412 };
 413
 414 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 415 {
 416         return container_of((void *)p, struct usb_hcd, hcd_priv);
 417 }
 418
 419 /**
 420  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 421  *                            (if needed)
 422  * @urb:        URB.
 423  * @mem_flags:  Memory allocation flags.
 424  *
 425  * This function allocates a temporary bounce buffer whenever it's needed
 426  * due to HW limitations.
 427  */
 428 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 429 {
 430         struct octeon_temp_buffer *temp;
 431
 432         if (urb->num_sgs || urb->sg ||
 433             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 434             !(urb->transfer_buffer_length % sizeof(u32)))
 435                 return 0;
 436
 437         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 438                        sizeof(*temp), mem_flags);
 439         if (!temp)
 440                 return -ENOMEM;
 441
 442         temp->orig_buffer = urb->transfer_buffer;
 443         if (usb_urb_dir_out(urb))
 444                 memcpy(temp->data, urb->transfer_buffer,
 445                        urb->transfer_buffer_length);
 446         urb->transfer_buffer = temp->data;
 447         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 448
 449         return 0;
 450 }
 451
 452 /**
 453  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 454  * @urb: URB.
 455  *
 456  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 457  */
 458 static void octeon_free_temp_buffer(struct urb *urb)
 459 {
 460         struct octeon_temp_buffer *temp;
 461         size_t length;
 462
 463         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 464                 return;
 465
 466         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 467                             data);
 468         if (usb_urb_dir_in(urb)) {
 469                 if (usb_pipeisoc(urb->pipe))
 470                         length = urb->transfer_buffer_length;
 471                 else
 472                         length = urb->actual_length;
 473
 474                 memcpy(temp->orig_buffer, urb->transfer_buffer, length);
 475         }
 476         urb->transfer_buffer = temp->orig_buffer;
 477         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 478         kfree(temp);
 479 }
 480
 481 /**
 482  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 483  * @hcd:        USB HCD structure.
 484  * @urb:        URB.
 485  * @mem_flags:  Memory allocation flags.
 486  */
 487 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 488                                   gfp_t mem_flags)
 489 {
 490         int ret;
 491
 492         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 493         if (ret)
 494                 return ret;
 495
 496         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 497         if (ret)
 498                 octeon_free_temp_buffer(urb);
 499
 500         return ret;
 501 }
 502
 503 /**
 504  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 505  * @hcd:        USB HCD structure.
 506  * @urb:        URB.
 507  */
 508 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 509 {
 510         usb_hcd_unmap_urb_for_dma(hcd, urb);
 511         octeon_free_temp_buffer(urb);
 512 }
 513
 514 /**
 515  * Read a USB 32bit CSR. It performs the necessary address swizzle
 516  * for 32bit CSRs and logs the value in a readable format if
 517  * debugging is on.
 518  *
 519  * @usb:     USB block this access is for
 520  * @address: 64bit address to read
 521  *
 522  * Returns: Result of the read
 523  */
 524 static inline u32 cvmx_usb_read_csr32(struct octeon_hcd *usb, u64 address)
 525 {
 526         return cvmx_read64_uint32(address ^ 4);
 527 }
 528
 529 /**
 530  * Write a USB 32bit CSR. It performs the necessary address
 531  * swizzle for 32bit CSRs and logs the value in a readable format
 532  * if debugging is on.
 533  *
 534  * @usb:     USB block this access is for
 535  * @address: 64bit address to write
 536  * @value:   Value to write
 537  */
 538 static inline void cvmx_usb_write_csr32(struct octeon_hcd *usb,
 539                                         u64 address, u32 value)
 540 {
 541         cvmx_write64_uint32(address ^ 4, value);
 542         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 543 }
 544
 545 /**
 546  * Return non zero if this pipe connects to a non HIGH speed
 547  * device through a high speed hub.
 548  *
 549  * @usb:    USB block this access is for
 550  * @pipe:   Pipe to check
 551  *
 552  * Returns: Non zero if we need to do split transactions
 553  */
 554 static inline int cvmx_usb_pipe_needs_split(struct octeon_hcd *usb,
 555                                             struct cvmx_usb_pipe *pipe)
 556 {
 557         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 558                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 559 }
 560
 561 /**
 562  * Trivial utility function to return the correct PID for a pipe
 563  *
 564  * @pipe:   pipe to check
 565  *
 566  * Returns: PID for pipe
 567  */
 568 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 569 {
 570         if (pipe->pid_toggle)
 571                 return 2; /* Data1 */
 572         return 0; /* Data0 */
 573 }
 574
 575 /* Loops through register until txfflsh or rxfflsh become zero.*/
 576 static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type)
 577 {
 578         int result;
 579         u64 address = CVMX_USBCX_GRSTCTL(usb->index);
 580         u64 done = cvmx_get_cycle() + 100 *
 581                    (u64)octeon_get_clock_rate / 1000000;
 582         union cvmx_usbcx_grstctl c;
 583
 584         while (1) {
 585                 c.u32 = cvmx_usb_read_csr32(usb, address);
 586                 if (fflsh_type == 0 && c.s.txfflsh == 0) {
 587                         result = 0;
 588                         break;
 589                 } else if (fflsh_type == 1 && c.s.rxfflsh == 0) {
 590                         result = 0;
 591                         break;
 592                 } else if (cvmx_get_cycle() > done) {
 593                         result = -1;
 594                         break;
 595                 }
 596
 597                 __delay(100);
 598         }
 599         return result;
 600 }
 601
 602 static void cvmx_fifo_setup(struct octeon_hcd *usb)
 603 {
 604         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 605         union cvmx_usbcx_gnptxfsiz npsiz;
 606         union cvmx_usbcx_hptxfsiz psiz;
 607
 608         usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
 609                                                 CVMX_USBCX_GHWCFG3(usb->index));
 610
 611         /*
 612          * Program the USBC_GRXFSIZ register to select the size of the receive
 613          * FIFO (25%).
 614          */
 615         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
 616                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 617
 618         /*
 619          * Program the USBC_GNPTXFSIZ register to select the size and the start
 620          * address of the non-periodic transmit FIFO for nonperiodic
 621          * transactions (50%).
 622          */
 623         npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 624         npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 625         npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 626         cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
 627
 628         /*
 629          * Program the USBC_HPTXFSIZ register to select the size and start
 630          * address of the periodic transmit FIFO for periodic transactions
 631          * (25%).
 632          */
 633         psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 634         psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 635         psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 636         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
 637
 638         /* Flush all FIFOs */
 639         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 640                         cvmx_usbcx_grstctl, txfnum, 0x10);
 641         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 642                         cvmx_usbcx_grstctl, txfflsh, 1);
 643         cvmx_wait_tx_rx(usb, 0);
 644         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 645                         cvmx_usbcx_grstctl, rxfflsh, 1);
 646         cvmx_wait_tx_rx(usb, 1);
 647 }
 648
 649 /**
 650  * Shutdown a USB port after a call to cvmx_usb_initialize().
 651  * The port should be disabled with all pipes closed when this
 652  * function is called.
 653  *
 654  * @usb: USB device state populated by cvmx_usb_initialize().
 655  *
 656  * Returns: 0 or a negative error code.
 657  */
 658 static int cvmx_usb_shutdown(struct octeon_hcd *usb)
 659 {
 660         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 661
 662         /* Make sure all pipes are closed */
 663         if (!list_empty(&usb->idle_pipes) ||
 664             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 665             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 666             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 667             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 668                 return -EBUSY;
 669
 670         /* Disable the clocks and put them in power on reset */
 671         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 672         usbn_clk_ctl.s.enable = 1;
 673         usbn_clk_ctl.s.por = 1;
 674         usbn_clk_ctl.s.hclk_rst = 1;
 675         usbn_clk_ctl.s.prst = 0;
 676         usbn_clk_ctl.s.hrst = 0;
 677         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 678         return 0;
 679 }
 680
 681 /**
 682  * Initialize a USB port for use. This must be called before any
 683  * other access to the Octeon USB port is made. The port starts
 684  * off in the disabled state.
 685  *
 686  * @dev:         Pointer to struct device for logging purposes.
 687  * @usb:         Pointer to struct octeon_hcd.
 688  *
 689  * Returns: 0 or a negative error code.
 690  */
 691 static int cvmx_usb_initialize(struct device *dev,
 692                                struct octeon_hcd *usb)
 693 {
 694         int channel;
 695         int divisor;
 696         int retries = 0;
 697         union cvmx_usbcx_hcfg usbcx_hcfg;
 698         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 699         union cvmx_usbcx_gintsts usbc_gintsts;
 700         union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 701         union cvmx_usbcx_gintmsk usbcx_gintmsk;
 702         union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 703         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 704
 705 retry:
 706         /*
 707          * Power On Reset and PHY Initialization
 708          *
 709          * 1. Wait for DCOK to assert (nothing to do)
 710          *
 711          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 712          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 713          */
 714         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 715         usbn_clk_ctl.s.por = 1;
 716         usbn_clk_ctl.s.hrst = 0;
 717         usbn_clk_ctl.s.prst = 0;
 718         usbn_clk_ctl.s.hclk_rst = 0;
 719         usbn_clk_ctl.s.enable = 0;
 720         /*
 721          * 2b. Select the USB reference clock/crystal parameters by writing
 722          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 723          */
 724         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 725                 /*
 726                  * The USB port uses 12/24/48MHz 2.5V board clock
 727                  * source at USB_XO. USB_XI should be tied to GND.
 728                  * Most Octeon evaluation boards require this setting
 729                  */
 730                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 731                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 732                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 733                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 734                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 735                 else
 736                         /* From CN52XX manual */
 737                         usbn_clk_ctl.s.p_rtype = 1;
 738
 739                 switch (usb->init_flags &
 740                         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 741                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 742                         usbn_clk_ctl.s.p_c_sel = 0;
 743                         break;
 744                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 745                         usbn_clk_ctl.s.p_c_sel = 1;
 746                         break;
 747                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 748                         usbn_clk_ctl.s.p_c_sel = 2;
 749                         break;
 750                 }
 751         } else {
 752                 /*
 753                  * The USB port uses a 12MHz crystal as clock source
 754                  * at USB_XO and USB_XI
 755                  */
 756                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 757                         /* From CN31XX,CN30XX manual */
 758                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 759                 else
 760                         /* From CN56XX,CN52XX,CN50XX manuals. */
 761                         usbn_clk_ctl.s.p_rtype = 0;
 762
 763                 usbn_clk_ctl.s.p_c_sel = 0;
 764         }
 765         /*
 766          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 767          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 768          *     such that USB is as close as possible to 125Mhz
 769          */
 770         divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 771         /* Lower than 4 doesn't seem to work properly */
 772         if (divisor < 4)
 773                 divisor = 4;
 774         usbn_clk_ctl.s.divide = divisor;
 775         usbn_clk_ctl.s.divide2 = 0;
 776         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 777
 778         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 779         usbn_clk_ctl.s.hclk_rst = 1;
 780         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 781         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 782         __delay(64);
 783         /*
 784          * 3. Program the power-on reset field in the USBN clock-control
 785          *    register:
 786          *    USBN_CLK_CTL[POR] = 0
 787          */
 788         usbn_clk_ctl.s.por = 0;
 789         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 790         /* 4. Wait 1 ms for PHY clock to start */
 791         mdelay(1);
 792         /*
 793          * 5. Program the Reset input from automatic test equipment field in the
 794          *    USBP control and status register:
 795          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 796          */
 797         usbn_usbp_ctl_status.u64 =
 798                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 799         usbn_usbp_ctl_status.s.ate_reset = 1;
 800         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 801                             usbn_usbp_ctl_status.u64);
 802         /* 6. Wait 10 cycles */
 803         __delay(10);
 804         /*
 805          * 7. Clear ATE_RESET field in the USBN clock-control register:
 806          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 807          */
 808         usbn_usbp_ctl_status.s.ate_reset = 0;
 809         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 810                             usbn_usbp_ctl_status.u64);
 811         /*
 812          * 8. Program the PHY reset field in the USBN clock-control register:
 813          *    USBN_CLK_CTL[PRST] = 1
 814          */
 815         usbn_clk_ctl.s.prst = 1;
 816         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 817         /*
 818          * 9. Program the USBP control and status register to select host or
 819          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 820          *    device
 821          */
 822         usbn_usbp_ctl_status.s.hst_mode = 0;
 823         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 824                             usbn_usbp_ctl_status.u64);
 825         /* 10. Wait 1 us */
 826         udelay(1);
 827         /*
 828          * 11. Program the hreset_n field in the USBN clock-control register:
 829          *     USBN_CLK_CTL[HRST] = 1
 830          */
 831         usbn_clk_ctl.s.hrst = 1;
 832         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 833         /* 12. Proceed to USB core initialization */
 834         usbn_clk_ctl.s.enable = 1;
 835         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 836         udelay(1);
 837
 838         /*
 839          * USB Core Initialization
 840          *
 841          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 842          *    determine USB core configuration parameters.
 843          *
 844          *    Nothing needed
 845          *
 846          * 2. Program the following fields in the global AHB configuration
 847          *    register (USBC_GAHBCFG)
 848          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 849          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 850          *    Nonperiodic TxFIFO empty level (slave mode only),
 851          *    USBC_GAHBCFG[NPTXFEMPLVL]
 852          *    Periodic TxFIFO empty level (slave mode only),
 853          *    USBC_GAHBCFG[PTXFEMPLVL]
 854          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 855          */
 856         usbcx_gahbcfg.u32 = 0;
 857         usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 858                                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 859         usbcx_gahbcfg.s.hbstlen = 0;
 860         usbcx_gahbcfg.s.nptxfemplvl = 1;
 861         usbcx_gahbcfg.s.ptxfemplvl = 1;
 862         usbcx_gahbcfg.s.glblintrmsk = 1;
 863         cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 864                              usbcx_gahbcfg.u32);
 865
 866         /*
 867          * 3. Program the following fields in USBC_GUSBCFG register.
 868          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 869          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 870          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 871          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 872          */
 873         usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
 874                                                 CVMX_USBCX_GUSBCFG(usb->index));
 875         usbcx_gusbcfg.s.toutcal = 0;
 876         usbcx_gusbcfg.s.ddrsel = 0;
 877         usbcx_gusbcfg.s.usbtrdtim = 0x5;
 878         usbcx_gusbcfg.s.phylpwrclksel = 0;
 879         cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 880                              usbcx_gusbcfg.u32);
 881
 882         /*
 883          * 4. The software must unmask the following bits in the USBC_GINTMSK
 884          *    register.
 885          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 886          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 887          */
 888         usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
 889                                                 CVMX_USBCX_GINTMSK(usb->index));
 890         usbcx_gintmsk.s.otgintmsk = 1;
 891         usbcx_gintmsk.s.modemismsk = 1;
 892         usbcx_gintmsk.s.hchintmsk = 1;
 893         usbcx_gintmsk.s.sofmsk = 0;
 894         /* We need RX FIFO interrupts if we don't have DMA */
 895         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 896                 usbcx_gintmsk.s.rxflvlmsk = 1;
 897         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 898                              usbcx_gintmsk.u32);
 899
 900         /*
 901          * Disable all channel interrupts. We'll enable them per channel later.
 902          */
 903         for (channel = 0; channel < 8; channel++)
 904                 cvmx_usb_write_csr32(usb,
 905                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
 906                                      0);
 907
 908         /*
 909          * Host Port Initialization
 910          *
 911          * 1. Program the host-port interrupt-mask field to unmask,
 912          *    USBC_GINTMSK[PRTINT] = 1
 913          */
 914         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 915                         cvmx_usbcx_gintmsk, prtintmsk, 1);
 916         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 917                         cvmx_usbcx_gintmsk, disconnintmsk, 1);
 918
 919         /*
 920          * 2. Program the USBC_HCFG register to select full-speed host
 921          *    or high-speed host.
 922          */
 923         usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 924         usbcx_hcfg.s.fslssupp = 0;
 925         usbcx_hcfg.s.fslspclksel = 0;
 926         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 927
 928         cvmx_fifo_setup(usb);
 929
 930         /*
 931          * If the controller is getting port events right after the reset, it
 932          * means the initialization failed. Try resetting the controller again
 933          * in such case. This is seen to happen after cold boot on DSR-1000N.
 934          */
 935         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
 936                                                CVMX_USBCX_GINTSTS(usb->index));
 937         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
 938                              usbc_gintsts.u32);
 939         dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
 940         if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
 941                 return 0;
 942         if (retries++ >= 5)
 943                 return -EAGAIN;
 944         dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
 945                  (int)usbc_gintsts.u32);
 946         msleep(50);
 947         cvmx_usb_shutdown(usb);
 948         msleep(50);
 949         goto retry;
 950 }
 951
 952 /**
 953  * Reset a USB port. After this call succeeds, the USB port is
 954  * online and servicing requests.
 955  *
 956  * @usb: USB device state populated by cvmx_usb_initialize().
 957  */
 958 static void cvmx_usb_reset_port(struct octeon_hcd *usb)
 959 {
 960         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 961                                                   CVMX_USBCX_HPRT(usb->index));
 962
 963         /* Program the port reset bit to start the reset process */
 964         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 965                         prtrst, 1);
 966
 967         /*
 968          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 969          * process to complete.
 970          */
 971         mdelay(50);
 972
 973         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 974         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 975                         prtrst, 0);
 976
 977         /*
 978          * Read the port speed field to get the enumerated speed,
 979          * USBC_HPRT[PRTSPD].
 980          */
 981         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 982                                                   CVMX_USBCX_HPRT(usb->index));
 983 }
 984
 985 /**
 986  * Disable a USB port. After this call the USB port will not
 987  * generate data transfers and will not generate events.
 988  * Transactions in process will fail and call their
 989  * associated callbacks.
 990  *
 991  * @usb: USB device state populated by cvmx_usb_initialize().
 992  *
 993  * Returns: 0 or a negative error code.
 994  */
 995 static int cvmx_usb_disable(struct octeon_hcd *usb)
 996 {
 997         /* Disable the port */
 998         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 999                         prtena, 1);
1000         return 0;
1001 }
1002
1003 /**
1004  * Get the current state of the USB port. Use this call to
1005  * determine if the usb port has anything connected, is enabled,
1006  * or has some sort of error condition. The return value of this
1007  * call has "changed" bits to signal of the value of some fields
1008  * have changed between calls.
1009  *
1010  * @usb: USB device state populated by cvmx_usb_initialize().
1011  *
1012  * Returns: Port status information
1013  */
1014 static struct cvmx_usb_port_status cvmx_usb_get_status(struct octeon_hcd *usb)
1015 {
1016         union cvmx_usbcx_hprt usbc_hprt;
1017         struct cvmx_usb_port_status result;
1018
1019         memset(&result, 0, sizeof(result));
1020
1021         usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1022         result.port_enabled = usbc_hprt.s.prtena;
1023         result.port_over_current = usbc_hprt.s.prtovrcurract;
1024         result.port_powered = usbc_hprt.s.prtpwr;
1025         result.port_speed = usbc_hprt.s.prtspd;
1026         result.connected = usbc_hprt.s.prtconnsts;
1027         result.connect_change =
1028                 result.connected != usb->port_status.connected;
1029
1030         return result;
1031 }
1032
1033 /**
1034  * Open a virtual pipe between the host and a USB device. A pipe
1035  * must be opened before data can be transferred between a device
1036  * and Octeon.
1037  *
1038  * @usb:             USB device state populated by cvmx_usb_initialize().
1039  * @device_addr:
1040  *                   USB device address to open the pipe to
1041  *                   (0-127).
1042  * @endpoint_num:
1043  *                   USB endpoint number to open the pipe to
1044  *                   (0-15).
1045  * @device_speed:
1046  *                   The speed of the device the pipe is going
1047  *                   to. This must match the device's speed,
1048  *                   which may be different than the port speed.
1049  * @max_packet:      The maximum packet length the device can
1050  *                   transmit/receive (low speed=0-8, full
1051  *                   speed=0-1023, high speed=0-1024). This value
1052  *                   comes from the standard endpoint descriptor
1053  *                   field wMaxPacketSize bits <10:0>.
1054  * @transfer_type:
1055  *                   The type of transfer this pipe is for.
1056  * @transfer_dir:
1057  *                   The direction the pipe is in. This is not
1058  *                   used for control pipes.
1059  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1060  *                   this is how often the transfer is scheduled
1061  *                   for. All other transfers should specify
1062  *                   zero. The units are in frames (8000/sec at
1063  *                   high speed, 1000/sec for full speed).
1064  * @multi_count:
1065  *                   For high speed devices, this is the maximum
1066  *                   allowed number of packet per microframe.
1067  *                   Specify zero for non high speed devices. This
1068  *                   value comes from the standard endpoint descriptor
1069  *                   field wMaxPacketSize bits <12:11>.
1070  * @hub_device_addr:
1071  *                   Hub device address this device is connected
1072  *                   to. Devices connected directly to Octeon
1073  *                   use zero. This is only used when the device
1074  *                   is full/low speed behind a high speed hub.
1075  *                   The address will be of the high speed hub,
1076  *                   not and full speed hubs after it.
1077  * @hub_port:        Which port on the hub the device is
1078  *                   connected. Use zero for devices connected
1079  *                   directly to Octeon. Like hub_device_addr,
1080  *                   this is only used for full/low speed
1081  *                   devices behind a high speed hub.
1082  *
1083  * Returns: A non-NULL value is a pipe. NULL means an error.
1084  */
1085 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct octeon_hcd *usb,
1086                                                 int device_addr,
1087                                                 int endpoint_num,
1088                                                 enum cvmx_usb_speed
1089                                                         device_speed,
1090                                                 int max_packet,
1091                                                 enum cvmx_usb_transfer
1092                                                         transfer_type,
1093                                                 enum cvmx_usb_direction
1094                                                         transfer_dir,
1095                                                 int interval, int multi_count,
1096                                                 int hub_device_addr,
1097                                                 int hub_port)
1098 {
1099         struct cvmx_usb_pipe *pipe;
1100
1101         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1102         if (!pipe)
1103                 return NULL;
1104         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1105             (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1106             (transfer_type == CVMX_USB_TRANSFER_BULK))
1107                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
1108         pipe->device_addr = device_addr;
1109         pipe->endpoint_num = endpoint_num;
1110         pipe->device_speed = device_speed;
1111         pipe->max_packet = max_packet;
1112         pipe->transfer_type = transfer_type;
1113         pipe->transfer_dir = transfer_dir;
1114         INIT_LIST_HEAD(&pipe->transactions);
1115
1116         /*
1117          * All pipes use interval to rate limit NAK processing. Force an
1118          * interval if one wasn't supplied
1119          */
1120         if (!interval)
1121                 interval = 1;
1122         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1123                 pipe->interval = interval * 8;
1124                 /* Force start splits to be schedule on uFrame 0 */
1125                 pipe->next_tx_frame = ((usb->frame_number + 7) & ~7) +
1126                                         pipe->interval;
1127         } else {
1128                 pipe->interval = interval;
1129                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1130         }
1131         pipe->multi_count = multi_count;
1132         pipe->hub_device_addr = hub_device_addr;
1133         pipe->hub_port = hub_port;
1134         pipe->pid_toggle = 0;
1135         pipe->split_sc_frame = -1;
1136         list_add_tail(&pipe->node, &usb->idle_pipes);
1137
1138         /*
1139          * We don't need to tell the hardware about this pipe yet since
1140          * it doesn't have any submitted requests
1141          */
1142
1143         return pipe;
1144 }
1145
1146 /**
1147  * Poll the RX FIFOs and remove data as needed. This function is only used
1148  * in non DMA mode. It is very important that this function be called quickly
1149  * enough to prevent FIFO overflow.
1150  *
1151  * @usb:        USB device state populated by cvmx_usb_initialize().
1152  */
1153 static void cvmx_usb_poll_rx_fifo(struct octeon_hcd *usb)
1154 {
1155         union cvmx_usbcx_grxstsph rx_status;
1156         int channel;
1157         int bytes;
1158         u64 address;
1159         u32 *ptr;
1160
1161         rx_status.u32 = cvmx_usb_read_csr32(usb,
1162                                             CVMX_USBCX_GRXSTSPH(usb->index));
1163         /* Only read data if IN data is there */
1164         if (rx_status.s.pktsts != 2)
1165                 return;
1166         /* Check if no data is available */
1167         if (!rx_status.s.bcnt)
1168                 return;
1169
1170         channel = rx_status.s.chnum;
1171         bytes = rx_status.s.bcnt;
1172         if (!bytes)
1173                 return;
1174
1175         /* Get where the DMA engine would have written this data */
1176         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1177                                      channel * 8);
1178
1179         ptr = cvmx_phys_to_ptr(address);
1180         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1181                             address + bytes);
1182
1183         /* Loop writing the FIFO data for this packet into memory */
1184         while (bytes > 0) {
1185                 *ptr++ = cvmx_usb_read_csr32(usb,
1186                                         USB_FIFO_ADDRESS(channel, usb->index));
1187                 bytes -= 4;
1188         }
1189         CVMX_SYNCW;
1190 }
1191
1192 /**
1193  * Fill the TX hardware fifo with data out of the software
1194  * fifos
1195  *
1196  * @usb:            USB device state populated by cvmx_usb_initialize().
1197  * @fifo:           Software fifo to use
1198  * @available:      Amount of space in the hardware fifo
1199  *
1200  * Returns: Non zero if the hardware fifo was too small and needs
1201  *          to be serviced again.
1202  */
1203 static int cvmx_usb_fill_tx_hw(struct octeon_hcd *usb,
1204                                struct cvmx_usb_tx_fifo *fifo, int available)
1205 {
1206         /*
1207          * We're done either when there isn't anymore space or the software FIFO
1208          * is empty
1209          */
1210         while (available && (fifo->head != fifo->tail)) {
1211                 int i = fifo->tail;
1212                 const u32 *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1213                 u64 csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1214                                                    usb->index) ^ 4;
1215                 int words = available;
1216
1217                 /* Limit the amount of data to what the SW fifo has */
1218                 if (fifo->entry[i].size <= available) {
1219                         words = fifo->entry[i].size;
1220                         fifo->tail++;
1221                         if (fifo->tail > MAX_CHANNELS)
1222                                 fifo->tail = 0;
1223                 }
1224
1225                 /* Update the next locations and counts */
1226                 available -= words;
1227                 fifo->entry[i].address += words * 4;
1228                 fifo->entry[i].size -= words;
1229
1230                 /*
1231                  * Write the HW fifo data. The read every three writes is due
1232                  * to an errata on CN3XXX chips
1233                  */
1234                 while (words > 3) {
1235                         cvmx_write64_uint32(csr_address, *ptr++);
1236                         cvmx_write64_uint32(csr_address, *ptr++);
1237                         cvmx_write64_uint32(csr_address, *ptr++);
1238                         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1239                         words -= 3;
1240                 }
1241                 cvmx_write64_uint32(csr_address, *ptr++);
1242                 if (--words) {
1243                         cvmx_write64_uint32(csr_address, *ptr++);
1244                         if (--words)
1245                                 cvmx_write64_uint32(csr_address, *ptr++);
1246                 }
1247                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1248         }
1249         return fifo->head != fifo->tail;
1250 }
1251
1252 /**
1253  * Check the hardware FIFOs and fill them as needed
1254  *
1255  * @usb:        USB device state populated by cvmx_usb_initialize().
1256  */
1257 static void cvmx_usb_poll_tx_fifo(struct octeon_hcd *usb)
1258 {
1259         if (usb->periodic.head != usb->periodic.tail) {
1260                 union cvmx_usbcx_hptxsts tx_status;
1261
1262                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1263                                                     CVMX_USBCX_HPTXSTS(usb->index));
1264                 if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1265                                         tx_status.s.ptxfspcavail))
1266                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1267                                         cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1268                 else
1269                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1270                                         cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1271         }
1272
1273         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1274                 union cvmx_usbcx_gnptxsts tx_status;
1275
1276                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1277                                                     CVMX_USBCX_GNPTXSTS(usb->index));
1278                 if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1279                                         tx_status.s.nptxfspcavail))
1280                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1281                                         cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1282                 else
1283                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1284                                         cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1285         }
1286 }
1287
1288 /**
1289  * Fill the TX FIFO with an outgoing packet
1290  *
1291  * @usb:          USB device state populated by cvmx_usb_initialize().
1292  * @channel:      Channel number to get packet from
1293  */
1294 static void cvmx_usb_fill_tx_fifo(struct octeon_hcd *usb, int channel)
1295 {
1296         union cvmx_usbcx_hccharx hcchar;
1297         union cvmx_usbcx_hcspltx usbc_hcsplt;
1298         union cvmx_usbcx_hctsizx usbc_hctsiz;
1299         struct cvmx_usb_tx_fifo *fifo;
1300
1301         /* We only need to fill data on outbound channels */
1302         hcchar.u32 = cvmx_usb_read_csr32(usb,
1303                                          CVMX_USBCX_HCCHARX(channel, usb->index));
1304         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1305                 return;
1306
1307         /* OUT Splits only have data on the start and not the complete */
1308         usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
1309                                               CVMX_USBCX_HCSPLTX(channel, usb->index));
1310         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1311                 return;
1312
1313         /*
1314          * Find out how many bytes we need to fill and convert it into 32bit
1315          * words.
1316          */
1317         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1318                                               CVMX_USBCX_HCTSIZX(channel, usb->index));
1319         if (!usbc_hctsiz.s.xfersize)
1320                 return;
1321
1322         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1323             (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1324                 fifo = &usb->periodic;
1325         else
1326                 fifo = &usb->nonperiodic;
1327
1328         fifo->entry[fifo->head].channel = channel;
1329         fifo->entry[fifo->head].address =
1330                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1331                                    channel * 8);
1332         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize + 3) >> 2;
1333         fifo->head++;
1334         if (fifo->head > MAX_CHANNELS)
1335                 fifo->head = 0;
1336
1337         cvmx_usb_poll_tx_fifo(usb);
1338 }
1339
1340 /**
1341  * Perform channel specific setup for Control transactions. All
1342  * the generic stuff will already have been done in cvmx_usb_start_channel().
1343  *
1344  * @usb:          USB device state populated by cvmx_usb_initialize().
1345  * @channel:      Channel to setup
1346  * @pipe:         Pipe for control transaction
1347  */
1348 static void cvmx_usb_start_channel_control(struct octeon_hcd *usb,
1349                                            int channel,
1350                                            struct cvmx_usb_pipe *pipe)
1351 {
1352         struct usb_hcd *hcd = octeon_to_hcd(usb);
1353         struct device *dev = hcd->self.controller;
1354         struct cvmx_usb_transaction *transaction =
1355                 list_first_entry(&pipe->transactions, typeof(*transaction),
1356                                  node);
1357         struct usb_ctrlrequest *header =
1358                 cvmx_phys_to_ptr(transaction->control_header);
1359         int bytes_to_transfer = transaction->buffer_length -
1360                 transaction->actual_bytes;
1361         int packets_to_transfer;
1362         union cvmx_usbcx_hctsizx usbc_hctsiz;
1363
1364         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1365                                               CVMX_USBCX_HCTSIZX(channel, usb->index));
1366
1367         switch (transaction->stage) {
1368         case CVMX_USB_STAGE_NON_CONTROL:
1369         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1370                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1371                 break;
1372         case CVMX_USB_STAGE_SETUP:
1373                 usbc_hctsiz.s.pid = 3; /* Setup */
1374                 bytes_to_transfer = sizeof(*header);
1375                 /* All Control operations start with a setup going OUT */
1376                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1377                                 cvmx_usbcx_hccharx, epdir,
1378                                 CVMX_USB_DIRECTION_OUT);
1379                 /*
1380                  * Setup send the control header instead of the buffer data. The
1381                  * buffer data will be used in the next stage
1382                  */
1383                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1384                                         channel * 8,
1385                                     transaction->control_header);
1386                 break;
1387         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1388                 usbc_hctsiz.s.pid = 3; /* Setup */
1389                 bytes_to_transfer = 0;
1390                 /* All Control operations start with a setup going OUT */
1391                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1392                                 cvmx_usbcx_hccharx, epdir,
1393                                 CVMX_USB_DIRECTION_OUT);
1394
1395                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1396                                 cvmx_usbcx_hcspltx, compsplt, 1);
1397                 break;
1398         case CVMX_USB_STAGE_DATA:
1399                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1400                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1401                         if (header->bRequestType & USB_DIR_IN)
1402                                 bytes_to_transfer = 0;
1403                         else if (bytes_to_transfer > pipe->max_packet)
1404                                 bytes_to_transfer = pipe->max_packet;
1405                 }
1406                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1407                                 cvmx_usbcx_hccharx, epdir,
1408                                 ((header->bRequestType & USB_DIR_IN) ?
1409                                         CVMX_USB_DIRECTION_IN :
1410                                         CVMX_USB_DIRECTION_OUT));
1411                 break;
1412         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1413                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1414                 if (!(header->bRequestType & USB_DIR_IN))
1415                         bytes_to_transfer = 0;
1416                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1417                                 cvmx_usbcx_hccharx, epdir,
1418                                 ((header->bRequestType & USB_DIR_IN) ?
1419                                         CVMX_USB_DIRECTION_IN :
1420                                         CVMX_USB_DIRECTION_OUT));
1421                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1422                                 cvmx_usbcx_hcspltx, compsplt, 1);
1423                 break;
1424         case CVMX_USB_STAGE_STATUS:
1425                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1426                 bytes_to_transfer = 0;
1427                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1428                                 cvmx_usbcx_hccharx, epdir,
1429                                 ((header->bRequestType & USB_DIR_IN) ?
1430                                         CVMX_USB_DIRECTION_OUT :
1431                                         CVMX_USB_DIRECTION_IN));
1432                 break;
1433         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1434                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1435                 bytes_to_transfer = 0;
1436                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1437                                 cvmx_usbcx_hccharx, epdir,
1438                                 ((header->bRequestType & USB_DIR_IN) ?
1439                                         CVMX_USB_DIRECTION_OUT :
1440                                         CVMX_USB_DIRECTION_IN));
1441                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1442                                 cvmx_usbcx_hcspltx, compsplt, 1);
1443                 break;
1444         }
1445
1446         /*
1447          * Make sure the transfer never exceeds the byte limit of the hardware.
1448          * Further bytes will be sent as continued transactions
1449          */
1450         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1451                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1452                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1453                 bytes_to_transfer *= pipe->max_packet;
1454         }
1455
1456         /*
1457          * Calculate the number of packets to transfer. If the length is zero
1458          * we still need to transfer one packet
1459          */
1460         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1461                                            pipe->max_packet);
1462         if (packets_to_transfer == 0) {
1463                 packets_to_transfer = 1;
1464         } else if ((packets_to_transfer > 1) &&
1465                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1466                 /*
1467                  * Limit to one packet when not using DMA. Channels must be
1468                  * restarted between every packet for IN transactions, so there
1469                  * is no reason to do multiple packets in a row
1470                  */
1471                 packets_to_transfer = 1;
1472                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1473         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1474                 /*
1475                  * Limit the number of packet and data transferred to what the
1476                  * hardware can handle
1477                  */
1478                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1479                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1480         }
1481
1482         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1483         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1484
1485         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1486                              usbc_hctsiz.u32);
1487 }
1488
1489 /**
1490  * Start a channel to perform the pipe's head transaction
1491  *
1492  * @usb:          USB device state populated by cvmx_usb_initialize().
1493  * @channel:      Channel to setup
1494  * @pipe:         Pipe to start
1495  */
1496 static void cvmx_usb_start_channel(struct octeon_hcd *usb, int channel,
1497                                    struct cvmx_usb_pipe *pipe)
1498 {
1499         struct cvmx_usb_transaction *transaction =
1500                 list_first_entry(&pipe->transactions, typeof(*transaction),
1501                                  node);
1502
1503         /* Make sure all writes to the DMA region get flushed */
1504         CVMX_SYNCW;
1505
1506         /* Attach the channel to the pipe */
1507         usb->pipe_for_channel[channel] = pipe;
1508         pipe->channel = channel;
1509         pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
1510
1511         /* Mark this channel as in use */
1512         usb->idle_hardware_channels &= ~(1 << channel);
1513
1514         /* Enable the channel interrupt bits */
1515         {
1516                 union cvmx_usbcx_hcintx usbc_hcint;
1517                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1518                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1519
1520                 /* Clear all channel status bits */
1521                 usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
1522                                                      CVMX_USBCX_HCINTX(channel, usb->index));
1523
1524                 cvmx_usb_write_csr32(usb,
1525                                      CVMX_USBCX_HCINTX(channel, usb->index),
1526                                      usbc_hcint.u32);
1527
1528                 usbc_hcintmsk.u32 = 0;
1529                 usbc_hcintmsk.s.chhltdmsk = 1;
1530                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1531                         /*
1532                          * Channels need these extra interrupts when we aren't
1533                          * in DMA mode.
1534                          */
1535                         usbc_hcintmsk.s.datatglerrmsk = 1;
1536                         usbc_hcintmsk.s.frmovrunmsk = 1;
1537                         usbc_hcintmsk.s.bblerrmsk = 1;
1538                         usbc_hcintmsk.s.xacterrmsk = 1;
1539                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1540                                 /*
1541                                  * Splits don't generate xfercompl, so we need
1542                                  * ACK and NYET.
1543                                  */
1544                                 usbc_hcintmsk.s.nyetmsk = 1;
1545                                 usbc_hcintmsk.s.ackmsk = 1;
1546                         }
1547                         usbc_hcintmsk.s.nakmsk = 1;
1548                         usbc_hcintmsk.s.stallmsk = 1;
1549                         usbc_hcintmsk.s.xfercomplmsk = 1;
1550                 }
1551                 cvmx_usb_write_csr32(usb,
1552                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
1553                                      usbc_hcintmsk.u32);
1554
1555                 /* Enable the channel interrupt to propagate */
1556                 usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
1557                                                         CVMX_USBCX_HAINTMSK(usb->index));
1558                 usbc_haintmsk.s.haintmsk |= 1 << channel;
1559                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
1560                                      usbc_haintmsk.u32);
1561         }
1562
1563         /* Setup the location the DMA engine uses. */
1564         {
1565                 u64 reg;
1566                 u64 dma_address = transaction->buffer +
1567                                   transaction->actual_bytes;
1568
1569                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1570                         dma_address = transaction->buffer +
1571                                         transaction->iso_packets[0].offset +
1572                                         transaction->actual_bytes;
1573
1574                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
1575                         reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
1576                 else
1577                         reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
1578                 cvmx_write64_uint64(reg + channel * 8, dma_address);
1579         }
1580
1581         /* Setup both the size of the transfer and the SPLIT characteristics */
1582         {
1583                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1584                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1585                 int packets_to_transfer;
1586                 int bytes_to_transfer = transaction->buffer_length -
1587                         transaction->actual_bytes;
1588
1589                 /*
1590                  * ISOCHRONOUS transactions store each individual transfer size
1591                  * in the packet structure, not the global buffer_length
1592                  */
1593                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1594                         bytes_to_transfer =
1595                                 transaction->iso_packets[0].length -
1596                                 transaction->actual_bytes;
1597
1598                 /*
1599                  * We need to do split transactions when we are talking to non
1600                  * high speed devices that are behind a high speed hub
1601                  */
1602                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1603                         /*
1604                          * On the start split phase (stage is even) record the
1605                          * frame number we will need to send the split complete.
1606                          * We only store the lower two bits since the time ahead
1607                          * can only be two frames
1608                          */
1609                         if ((transaction->stage & 1) == 0) {
1610                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1611                                         pipe->split_sc_frame =
1612                                                 (usb->frame_number + 1) & 0x7f;
1613                                 else
1614                                         pipe->split_sc_frame =
1615                                                 (usb->frame_number + 2) & 0x7f;
1616                         } else {
1617                                 pipe->split_sc_frame = -1;
1618                         }
1619
1620                         usbc_hcsplt.s.spltena = 1;
1621                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1622                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1623                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1624                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1625
1626                         /*
1627                          * SPLIT transactions can only ever transmit one data
1628                          * packet so limit the transfer size to the max packet
1629                          * size
1630                          */
1631                         if (bytes_to_transfer > pipe->max_packet)
1632                                 bytes_to_transfer = pipe->max_packet;
1633
1634                         /*
1635                          * ISOCHRONOUS OUT splits are unique in that they limit
1636                          * data transfers to 188 byte chunks representing the
1637                          * begin/middle/end of the data or all
1638                          */
1639                         if (!usbc_hcsplt.s.compsplt &&
1640                             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1641                             (pipe->transfer_type ==
1642                              CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1643                                 /*
1644                                  * Clear the split complete frame number as
1645                                  * there isn't going to be a split complete
1646                                  */
1647                                 pipe->split_sc_frame = -1;
1648                                 /*
1649                                  * See if we've started this transfer and sent
1650                                  * data
1651                                  */
1652                                 if (transaction->actual_bytes == 0) {
1653                                         /*
1654                                          * Nothing sent yet, this is either a
1655                                          * begin or the entire payload
1656                                          */
1657                                         if (bytes_to_transfer <= 188)
1658                                                 /* Entire payload in one go */
1659                                                 usbc_hcsplt.s.xactpos = 3;
1660                                         else
1661                                                 /* First part of payload */
1662                                                 usbc_hcsplt.s.xactpos = 2;
1663                                 } else {
1664                                         /*
1665                                          * Continuing the previous data, we must
1666                                          * either be in the middle or at the end
1667                                          */
1668                                         if (bytes_to_transfer <= 188)
1669                                                 /* End of payload */
1670                                                 usbc_hcsplt.s.xactpos = 1;
1671                                         else
1672                                                 /* Middle of payload */
1673                                                 usbc_hcsplt.s.xactpos = 0;
1674                                 }
1675                                 /*
1676                                  * Again, the transfer size is limited to 188
1677                                  * bytes
1678                                  */
1679                                 if (bytes_to_transfer > 188)
1680                                         bytes_to_transfer = 188;
1681                         }
1682                 }
1683
1684                 /*
1685                  * Make sure the transfer never exceeds the byte limit of the
1686                  * hardware. Further bytes will be sent as continued
1687                  * transactions
1688                  */
1689                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1690                         /*
1691                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1692                          * size
1693                          */
1694                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1695                                 pipe->max_packet;
1696                         bytes_to_transfer *= pipe->max_packet;
1697                 }
1698
1699                 /*
1700                  * Calculate the number of packets to transfer. If the length is
1701                  * zero we still need to transfer one packet
1702                  */
1703                 packets_to_transfer =
1704                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1705                 if (packets_to_transfer == 0) {
1706                         packets_to_transfer = 1;
1707                 } else if ((packets_to_transfer > 1) &&
1708                            (usb->init_flags &
1709                             CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1710                         /*
1711                          * Limit to one packet when not using DMA. Channels must
1712                          * be restarted between every packet for IN
1713                          * transactions, so there is no reason to do multiple
1714                          * packets in a row
1715                          */
1716                         packets_to_transfer = 1;
1717                         bytes_to_transfer = packets_to_transfer *
1718                                 pipe->max_packet;
1719                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1720                         /*
1721                          * Limit the number of packet and data transferred to
1722                          * what the hardware can handle
1723                          */
1724                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1725                         bytes_to_transfer = packets_to_transfer *
1726                                 pipe->max_packet;
1727                 }
1728
1729                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1730                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1731
1732                 /* Update the DATA0/DATA1 toggle */
1733                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1734                 /*
1735                  * High speed pipes may need a hardware ping before they start
1736                  */
1737                 if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
1738                         usbc_hctsiz.s.dopng = 1;
1739
1740                 cvmx_usb_write_csr32(usb,
1741                                      CVMX_USBCX_HCSPLTX(channel, usb->index),
1742                                      usbc_hcsplt.u32);
1743                 cvmx_usb_write_csr32(usb,
1744                                      CVMX_USBCX_HCTSIZX(channel, usb->index),
1745                                      usbc_hctsiz.u32);
1746         }
1747
1748         /* Setup the Host Channel Characteristics Register */
1749         {
1750                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1751
1752                 /*
1753                  * Set the startframe odd/even properly. This is only used for
1754                  * periodic
1755                  */
1756                 usbc_hcchar.s.oddfrm = usb->frame_number & 1;
1757
1758                 /*
1759                  * Set the number of back to back packets allowed by this
1760                  * endpoint. Split transactions interpret "ec" as the number of
1761                  * immediate retries of failure. These retries happen too
1762                  * quickly, so we disable these entirely for splits
1763                  */
1764                 if (cvmx_usb_pipe_needs_split(usb, pipe))
1765                         usbc_hcchar.s.ec = 1;
1766                 else if (pipe->multi_count < 1)
1767                         usbc_hcchar.s.ec = 1;
1768                 else if (pipe->multi_count > 3)
1769                         usbc_hcchar.s.ec = 3;
1770                 else
1771                         usbc_hcchar.s.ec = pipe->multi_count;
1772
1773                 /* Set the rest of the endpoint specific settings */
1774                 usbc_hcchar.s.devaddr = pipe->device_addr;
1775                 usbc_hcchar.s.eptype = transaction->type;
1776                 usbc_hcchar.s.lspddev =
1777                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1778                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1779                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1780                 usbc_hcchar.s.mps = pipe->max_packet;
1781                 cvmx_usb_write_csr32(usb,
1782                                      CVMX_USBCX_HCCHARX(channel, usb->index),
1783                                      usbc_hcchar.u32);
1784         }
1785
1786         /* Do transaction type specific fixups as needed */
1787         switch (transaction->type) {
1788         case CVMX_USB_TRANSFER_CONTROL:
1789                 cvmx_usb_start_channel_control(usb, channel, pipe);
1790                 break;
1791         case CVMX_USB_TRANSFER_BULK:
1792         case CVMX_USB_TRANSFER_INTERRUPT:
1793                 break;
1794         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1795                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
1796                         /*
1797                          * ISO transactions require different PIDs depending on
1798                          * direction and how many packets are needed
1799                          */
1800                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1801                                 if (pipe->multi_count < 2) /* Need DATA0 */
1802                                         USB_SET_FIELD32(
1803                                                 CVMX_USBCX_HCTSIZX(channel,
1804                                                                    usb->index),
1805                                                 cvmx_usbcx_hctsizx, pid, 0);
1806                                 else /* Need MDATA */
1807                                         USB_SET_FIELD32(
1808                                                 CVMX_USBCX_HCTSIZX(channel,
1809                                                                    usb->index),
1810                                                 cvmx_usbcx_hctsizx, pid, 3);
1811                         }
1812                 }
1813                 break;
1814         }
1815         {
1816                 union cvmx_usbcx_hctsizx usbc_hctsiz = { .u32 =
1817                         cvmx_usb_read_csr32(usb,
1818                                             CVMX_USBCX_HCTSIZX(channel,
1819                                                                usb->index))
1820                 };
1821                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1822                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1823         }
1824         /* Remember when we start a split transaction */
1825         if (cvmx_usb_pipe_needs_split(usb, pipe))
1826                 usb->active_split = transaction;
1827         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1828                         cvmx_usbcx_hccharx, chena, 1);
1829         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1830                 cvmx_usb_fill_tx_fifo(usb, channel);
1831 }
1832
1833 /**
1834  * Find a pipe that is ready to be scheduled to hardware.
1835  * @usb:         USB device state populated by cvmx_usb_initialize().
1836  * @xfer_type:   Transfer type
1837  *
1838  * Returns: Pipe or NULL if none are ready
1839  */
1840 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(struct octeon_hcd *usb,
1841                                                       enum cvmx_usb_transfer xfer_type)
1842 {
1843         struct list_head *list = usb->active_pipes + xfer_type;
1844         u64 current_frame = usb->frame_number;
1845         struct cvmx_usb_pipe *pipe;
1846
1847         list_for_each_entry(pipe, list, node) {
1848                 struct cvmx_usb_transaction *t =
1849                         list_first_entry(&pipe->transactions, typeof(*t),
1850                                          node);
1851                 if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1852                     (pipe->next_tx_frame <= current_frame) &&
1853                     ((pipe->split_sc_frame == -1) ||
1854                      ((((int)current_frame - pipe->split_sc_frame) & 0x7f) <
1855                       0x40)) &&
1856                     (!usb->active_split || (usb->active_split == t))) {
1857                         prefetch(t);
1858                         return pipe;
1859                 }
1860         }
1861         return NULL;
1862 }
1863
1864 static struct cvmx_usb_pipe *cvmx_usb_next_pipe(struct octeon_hcd *usb,
1865                                                 int is_sof)
1866 {
1867         struct cvmx_usb_pipe *pipe;
1868
1869         /* Find a pipe needing service. */
1870         if (is_sof) {
1871                 /*
1872                  * Only process periodic pipes on SOF interrupts. This way we
1873                  * are sure that the periodic data is sent in the beginning of
1874                  * the frame.
1875                  */
1876                 pipe = cvmx_usb_find_ready_pipe(usb,
1877                                                 CVMX_USB_TRANSFER_ISOCHRONOUS);
1878                 if (pipe)
1879                         return pipe;
1880                 pipe = cvmx_usb_find_ready_pipe(usb,
1881                                                 CVMX_USB_TRANSFER_INTERRUPT);
1882                 if (pipe)
1883                         return pipe;
1884         }
1885         pipe = cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_CONTROL);
1886         if (pipe)
1887                 return pipe;
1888         return cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_BULK);
1889 }
1890
1891 /**
1892  * Called whenever a pipe might need to be scheduled to the
1893  * hardware.
1894  *
1895  * @usb:         USB device state populated by cvmx_usb_initialize().
1896  * @is_sof:      True if this schedule was called on a SOF interrupt.
1897  */
1898 static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
1899 {
1900         int channel;
1901         struct cvmx_usb_pipe *pipe;
1902         int need_sof;
1903         enum cvmx_usb_transfer ttype;
1904
1905         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1906                 /*
1907                  * Without DMA we need to be careful to not schedule something
1908                  * at the end of a frame and cause an overrun.
1909                  */
1910                 union cvmx_usbcx_hfnum hfnum = {
1911                         .u32 = cvmx_usb_read_csr32(usb,
1912                                                 CVMX_USBCX_HFNUM(usb->index))
1913                 };
1914
1915                 union cvmx_usbcx_hfir hfir = {
1916                         .u32 = cvmx_usb_read_csr32(usb,
1917                                                 CVMX_USBCX_HFIR(usb->index))
1918                 };
1919
1920                 if (hfnum.s.frrem < hfir.s.frint / 4)
1921                         goto done;
1922         }
1923
1924         while (usb->idle_hardware_channels) {
1925                 /* Find an idle channel */
1926                 channel = __fls(usb->idle_hardware_channels);
1927                 if (unlikely(channel > 7))
1928                         break;
1929
1930                 pipe = cvmx_usb_next_pipe(usb, is_sof);
1931                 if (!pipe)
1932                         break;
1933
1934                 cvmx_usb_start_channel(usb, channel, pipe);
1935         }
1936
1937 done:
1938         /*
1939          * Only enable SOF interrupts when we have transactions pending in the
1940          * future that might need to be scheduled
1941          */
1942         need_sof = 0;
1943         for (ttype = CVMX_USB_TRANSFER_CONTROL;
1944              ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1945                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1946                         if (pipe->next_tx_frame > usb->frame_number) {
1947                                 need_sof = 1;
1948                                 break;
1949                         }
1950                 }
1951         }
1952         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1953                         cvmx_usbcx_gintmsk, sofmsk, need_sof);
1954 }
1955
1956 static void octeon_usb_urb_complete_callback(struct octeon_hcd *usb,
1957                                              enum cvmx_usb_status status,
1958                                              struct cvmx_usb_pipe *pipe,
1959                                              struct cvmx_usb_transaction
1960                                                 *transaction,
1961                                              int bytes_transferred,
1962                                              struct urb *urb)
1963 {
1964         struct usb_hcd *hcd = octeon_to_hcd(usb);
1965         struct device *dev = hcd->self.controller;
1966
1967         if (likely(status == CVMX_USB_STATUS_OK))
1968                 urb->actual_length = bytes_transferred;
1969         else
1970                 urb->actual_length = 0;
1971
1972         urb->hcpriv = NULL;
1973
1974         /* For Isochronous transactions we need to update the URB packet status
1975          * list from data in our private copy
1976          */
1977         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1978                 int i;
1979                 /*
1980                  * The pointer to the private list is stored in the setup_packet
1981                  * field.
1982                  */
1983                 struct cvmx_usb_iso_packet *iso_packet =
1984                         (struct cvmx_usb_iso_packet *)urb->setup_packet;
1985                 /* Recalculate the transfer size by adding up each packet */
1986                 urb->actual_length = 0;
1987                 for (i = 0; i < urb->number_of_packets; i++) {
1988                         if (iso_packet[i].status == CVMX_USB_STATUS_OK) {
1989                                 urb->iso_frame_desc[i].status = 0;
1990                                 urb->iso_frame_desc[i].actual_length =
1991                                         iso_packet[i].length;
1992                                 urb->actual_length +=
1993                                         urb->iso_frame_desc[i].actual_length;
1994                         } else {
1995                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
1996                                         i, urb->number_of_packets,
1997                                         iso_packet[i].status, pipe,
1998                                         transaction, iso_packet[i].length);
1999                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2000                         }
2001                 }
2002                 /* Free the private list now that we don't need it anymore */
2003                 kfree(iso_packet);
2004                 urb->setup_packet = NULL;
2005         }
2006
2007         switch (status) {
2008         case CVMX_USB_STATUS_OK:
2009                 urb->status = 0;
2010                 break;
2011         case CVMX_USB_STATUS_CANCEL:
2012                 if (urb->status == 0)
2013                         urb->status = -ENOENT;
2014                 break;
2015         case CVMX_USB_STATUS_STALL:
2016                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2017                         pipe, transaction, bytes_transferred);
2018                 urb->status = -EPIPE;
2019                 break;
2020         case CVMX_USB_STATUS_BABBLEERR:
2021                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2022                         pipe, transaction, bytes_transferred);
2023                 urb->status = -EPIPE;
2024                 break;
2025         case CVMX_USB_STATUS_SHORT:
2026                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2027                         pipe, transaction, bytes_transferred);
2028                 urb->status = -EREMOTEIO;
2029                 break;
2030         case CVMX_USB_STATUS_ERROR:
2031         case CVMX_USB_STATUS_XACTERR:
2032         case CVMX_USB_STATUS_DATATGLERR:
2033         case CVMX_USB_STATUS_FRAMEERR:
2034                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2035                         status, pipe, transaction, bytes_transferred);
2036                 urb->status = -EPROTO;
2037                 break;
2038         }
2039         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(usb), urb);
2040         spin_unlock(&usb->lock);
2041         usb_hcd_giveback_urb(octeon_to_hcd(usb), urb, urb->status);
2042         spin_lock(&usb->lock);
2043 }
2044
2045 /**
2046  * Signal the completion of a transaction and free it. The
2047  * transaction will be removed from the pipe transaction list.
2048  *
2049  * @usb:         USB device state populated by cvmx_usb_initialize().
2050  * @pipe:        Pipe the transaction is on
2051  * @transaction:
2052  *               Transaction that completed
2053  * @complete_code:
2054  *               Completion code
2055  */
2056 static void cvmx_usb_complete(struct octeon_hcd *usb,
2057                               struct cvmx_usb_pipe *pipe,
2058                               struct cvmx_usb_transaction *transaction,
2059                               enum cvmx_usb_status complete_code)
2060 {
2061         /* If this was a split then clear our split in progress marker */
2062         if (usb->active_split == transaction)
2063                 usb->active_split = NULL;
2064
2065         /*
2066          * Isochronous transactions need extra processing as they might not be
2067          * done after a single data transfer
2068          */
2069         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2070                 /* Update the number of bytes transferred in this ISO packet */
2071                 transaction->iso_packets[0].length = transaction->actual_bytes;
2072                 transaction->iso_packets[0].status = complete_code;
2073
2074                 /*
2075                  * If there are more ISOs pending and we succeeded, schedule the
2076                  * next one
2077                  */
2078                 if ((transaction->iso_number_packets > 1) &&
2079                     (complete_code == CVMX_USB_STATUS_OK)) {
2080                         /* No bytes transferred for this packet as of yet */
2081                         transaction->actual_bytes = 0;
2082                         /* One less ISO waiting to transfer */
2083                         transaction->iso_number_packets--;
2084                         /* Increment to the next location in our packet array */
2085                         transaction->iso_packets++;
2086                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2087                         return;
2088                 }
2089         }
2090
2091         /* Remove the transaction from the pipe list */
2092         list_del(&transaction->node);
2093         if (list_empty(&pipe->transactions))
2094                 list_move_tail(&pipe->node, &usb->idle_pipes);
2095         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2096                                          transaction,
2097                                          transaction->actual_bytes,
2098                                          transaction->urb);
2099         kfree(transaction);
2100 }
2101
2102 /**
2103  * Submit a usb transaction to a pipe. Called for all types
2104  * of transactions.
2105  *
2106  * @usb:
2107  * @pipe:           Which pipe to submit to.
2108  * @type:           Transaction type
2109  * @buffer:         User buffer for the transaction
2110  * @buffer_length:
2111  *                  User buffer's length in bytes
2112  * @control_header:
2113  *                  For control transactions, the 8 byte standard header
2114  * @iso_start_frame:
2115  *                  For ISO transactions, the start frame
2116  * @iso_number_packets:
2117  *                  For ISO, the number of packet in the transaction.
2118  * @iso_packets:
2119  *                  A description of each ISO packet
2120  * @urb:            URB for the callback
2121  *
2122  * Returns: Transaction or NULL on failure.
2123  */
2124 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
2125                                 struct octeon_hcd *usb,
2126                                 struct cvmx_usb_pipe *pipe,
2127                                 enum cvmx_usb_transfer type,
2128                                 u64 buffer,
2129                                 int buffer_length,
2130                                 u64 control_header,
2131                                 int iso_start_frame,
2132                                 int iso_number_packets,
2133                                 struct cvmx_usb_iso_packet *iso_packets,
2134                                 struct urb *urb)
2135 {
2136         struct cvmx_usb_transaction *transaction;
2137
2138         if (unlikely(pipe->transfer_type != type))
2139                 return NULL;
2140
2141         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2142         if (unlikely(!transaction))
2143                 return NULL;
2144
2145         transaction->type = type;
2146         transaction->buffer = buffer;
2147         transaction->buffer_length = buffer_length;
2148         transaction->control_header = control_header;
2149         /* FIXME: This is not used, implement it. */
2150         transaction->iso_start_frame = iso_start_frame;
2151         transaction->iso_number_packets = iso_number_packets;
2152         transaction->iso_packets = iso_packets;
2153         transaction->urb = urb;
2154         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2155                 transaction->stage = CVMX_USB_STAGE_SETUP;
2156         else
2157                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2158
2159         if (!list_empty(&pipe->transactions)) {
2160                 list_add_tail(&transaction->node, &pipe->transactions);
2161         } else {
2162                 list_add_tail(&transaction->node, &pipe->transactions);
2163                 list_move_tail(&pipe->node,
2164                                &usb->active_pipes[pipe->transfer_type]);
2165
2166                 /*
2167                  * We may need to schedule the pipe if this was the head of the
2168                  * pipe.
2169                  */
2170                 cvmx_usb_schedule(usb, 0);
2171         }
2172
2173         return transaction;
2174 }
2175
2176 /**
2177  * Call to submit a USB Bulk transfer to a pipe.
2178  *
2179  * @usb:            USB device state populated by cvmx_usb_initialize().
2180  * @pipe:           Handle to the pipe for the transfer.
2181  * @urb:            URB.
2182  *
2183  * Returns: A submitted transaction or NULL on failure.
2184  */
2185 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2186                                                 struct octeon_hcd *usb,
2187                                                 struct cvmx_usb_pipe *pipe,
2188                                                 struct urb *urb)
2189 {
2190         return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2191                                            urb->transfer_dma,
2192                                            urb->transfer_buffer_length,
2193                                            0, /* control_header */
2194                                            0, /* iso_start_frame */
2195                                            0, /* iso_number_packets */
2196                                            NULL, /* iso_packets */
2197                                            urb);
2198 }
2199
2200 /**
2201  * Call to submit a USB Interrupt transfer to a pipe.
2202  *
2203  * @usb:            USB device state populated by cvmx_usb_initialize().
2204  * @pipe:           Handle to the pipe for the transfer.
2205  * @urb:            URB returned when the callback is called.
2206  *
2207  * Returns: A submitted transaction or NULL on failure.
2208  */
2209 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2210                                                 struct octeon_hcd *usb,
2211                                                 struct cvmx_usb_pipe *pipe,
2212                                                 struct urb *urb)
2213 {
2214         return cvmx_usb_submit_transaction(usb, pipe,
2215                                            CVMX_USB_TRANSFER_INTERRUPT,
2216                                            urb->transfer_dma,
2217                                            urb->transfer_buffer_length,
2218                                            0, /* control_header */
2219                                            0, /* iso_start_frame */
2220                                            0, /* iso_number_packets */
2221                                            NULL, /* iso_packets */
2222                                            urb);
2223 }
2224
2225 /**
2226  * Call to submit a USB Control transfer to a pipe.
2227  *
2228  * @usb:            USB device state populated by cvmx_usb_initialize().
2229  * @pipe:           Handle to the pipe for the transfer.
2230  * @urb:            URB.
2231  *
2232  * Returns: A submitted transaction or NULL on failure.
2233  */
2234 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2235                                                 struct octeon_hcd *usb,
2236                                                 struct cvmx_usb_pipe *pipe,
2237                                                 struct urb *urb)
2238 {
2239         int buffer_length = urb->transfer_buffer_length;
2240         u64 control_header = urb->setup_dma;
2241         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2242
2243         if ((header->bRequestType & USB_DIR_IN) == 0)
2244                 buffer_length = le16_to_cpu(header->wLength);
2245
2246         return cvmx_usb_submit_transaction(usb, pipe,
2247                                            CVMX_USB_TRANSFER_CONTROL,
2248                                            urb->transfer_dma, buffer_length,
2249                                            control_header,
2250                                            0, /* iso_start_frame */
2251                                            0, /* iso_number_packets */
2252                                            NULL, /* iso_packets */
2253                                            urb);
2254 }
2255
2256 /**
2257  * Call to submit a USB Isochronous transfer to a pipe.
2258  *
2259  * @usb:            USB device state populated by cvmx_usb_initialize().
2260  * @pipe:           Handle to the pipe for the transfer.
2261  * @urb:            URB returned when the callback is called.
2262  *
2263  * Returns: A submitted transaction or NULL on failure.
2264  */
2265 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2266                                                 struct octeon_hcd *usb,
2267                                                 struct cvmx_usb_pipe *pipe,
2268                                                 struct urb *urb)
2269 {
2270         struct cvmx_usb_iso_packet *packets;
2271
2272         packets = (struct cvmx_usb_iso_packet *)urb->setup_packet;
2273         return cvmx_usb_submit_transaction(usb, pipe,
2274                                            CVMX_USB_TRANSFER_ISOCHRONOUS,
2275                                            urb->transfer_dma,
2276                                            urb->transfer_buffer_length,
2277                                            0, /* control_header */
2278                                            urb->start_frame,
2279                                            urb->number_of_packets,
2280                                            packets, urb);
2281 }
2282
2283 /**
2284  * Cancel one outstanding request in a pipe. Canceling a request
2285  * can fail if the transaction has already completed before cancel
2286  * is called. Even after a successful cancel call, it may take
2287  * a frame or two for the cvmx_usb_poll() function to call the
2288  * associated callback.
2289  *
2290  * @usb:         USB device state populated by cvmx_usb_initialize().
2291  * @pipe:        Pipe to cancel requests in.
2292  * @transaction: Transaction to cancel, returned by the submit function.
2293  *
2294  * Returns: 0 or a negative error code.
2295  */
2296 static int cvmx_usb_cancel(struct octeon_hcd *usb,
2297                            struct cvmx_usb_pipe *pipe,
2298                            struct cvmx_usb_transaction *transaction)
2299 {
2300         /*
2301          * If the transaction is the HEAD of the queue and scheduled. We need to
2302          * treat it special
2303          */
2304         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2305             transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2306                 union cvmx_usbcx_hccharx usbc_hcchar;
2307
2308                 usb->pipe_for_channel[pipe->channel] = NULL;
2309                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2310
2311                 CVMX_SYNCW;
2312
2313                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2314                                                       CVMX_USBCX_HCCHARX(pipe->channel,
2315                                                                          usb->index));
2316                 /*
2317                  * If the channel isn't enabled then the transaction already
2318                  * completed.
2319                  */
2320                 if (usbc_hcchar.s.chena) {
2321                         usbc_hcchar.s.chdis = 1;
2322                         cvmx_usb_write_csr32(usb,
2323                                              CVMX_USBCX_HCCHARX(pipe->channel,
2324                                                                 usb->index),
2325                                              usbc_hcchar.u32);
2326                 }
2327         }
2328         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_CANCEL);
2329         return 0;
2330 }
2331
2332 /**
2333  * Cancel all outstanding requests in a pipe. Logically all this
2334  * does is call cvmx_usb_cancel() in a loop.
2335  *
2336  * @usb:         USB device state populated by cvmx_usb_initialize().
2337  * @pipe:        Pipe to cancel requests in.
2338  *
2339  * Returns: 0 or a negative error code.
2340  */
2341 static int cvmx_usb_cancel_all(struct octeon_hcd *usb,
2342                                struct cvmx_usb_pipe *pipe)
2343 {
2344         struct cvmx_usb_transaction *transaction, *next;
2345
2346         /* Simply loop through and attempt to cancel each transaction */
2347         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2348                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2349
2350                 if (unlikely(result != 0))
2351                         return result;
2352         }
2353         return 0;
2354 }
2355
2356 /**
2357  * Close a pipe created with cvmx_usb_open_pipe().
2358  *
2359  * @usb:         USB device state populated by cvmx_usb_initialize().
2360  * @pipe:        Pipe to close.
2361  *
2362  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2363  *          outstanding transfers.
2364  */
2365 static int cvmx_usb_close_pipe(struct octeon_hcd *usb,
2366                                struct cvmx_usb_pipe *pipe)
2367 {
2368         /* Fail if the pipe has pending transactions */
2369         if (!list_empty(&pipe->transactions))
2370                 return -EBUSY;
2371
2372         list_del(&pipe->node);
2373         kfree(pipe);
2374
2375         return 0;
2376 }
2377
2378 /**
2379  * Get the current USB protocol level frame number. The frame
2380  * number is always in the range of 0-0x7ff.
2381  *
2382  * @usb: USB device state populated by cvmx_usb_initialize().
2383  *
2384  * Returns: USB frame number
2385  */
2386 static int cvmx_usb_get_frame_number(struct octeon_hcd *usb)
2387 {
2388         union cvmx_usbcx_hfnum usbc_hfnum;
2389
2390         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2391
2392         return usbc_hfnum.s.frnum;
2393 }
2394
2395 static void cvmx_usb_transfer_control(struct octeon_hcd *usb,
2396                                       struct cvmx_usb_pipe *pipe,
2397                                       struct cvmx_usb_transaction *transaction,
2398                                       union cvmx_usbcx_hccharx usbc_hcchar,
2399                                       int buffer_space_left,
2400                                       int bytes_in_last_packet)
2401 {
2402         switch (transaction->stage) {
2403         case CVMX_USB_STAGE_NON_CONTROL:
2404         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2405                 /* This should be impossible */
2406                 cvmx_usb_complete(usb, pipe, transaction,
2407                                   CVMX_USB_STATUS_ERROR);
2408                 break;
2409         case CVMX_USB_STAGE_SETUP:
2410                 pipe->pid_toggle = 1;
2411                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2412                         transaction->stage =
2413                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2414                 } else {
2415                         struct usb_ctrlrequest *header =
2416                                 cvmx_phys_to_ptr(transaction->control_header);
2417                         if (header->wLength)
2418                                 transaction->stage = CVMX_USB_STAGE_DATA;
2419                         else
2420                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2421                 }
2422                 break;
2423         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2424                 {
2425                         struct usb_ctrlrequest *header =
2426                                 cvmx_phys_to_ptr(transaction->control_header);
2427                         if (header->wLength)
2428                                 transaction->stage = CVMX_USB_STAGE_DATA;
2429                         else
2430                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2431                 }
2432                 break;
2433         case CVMX_USB_STAGE_DATA:
2434                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2435                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2436                         /*
2437                          * For setup OUT data that are splits,
2438                          * the hardware doesn't appear to count
2439                          * transferred data. Here we manually
2440                          * update the data transferred
2441                          */
2442                         if (!usbc_hcchar.s.epdir) {
2443                                 if (buffer_space_left < pipe->max_packet)
2444                                         transaction->actual_bytes +=
2445                                                 buffer_space_left;
2446                                 else
2447                                         transaction->actual_bytes +=
2448                                                 pipe->max_packet;
2449                         }
2450                 } else if ((buffer_space_left == 0) ||
2451                            (bytes_in_last_packet < pipe->max_packet)) {
2452                         pipe->pid_toggle = 1;
2453                         transaction->stage = CVMX_USB_STAGE_STATUS;
2454                 }
2455                 break;
2456         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2457                 if ((buffer_space_left == 0) ||
2458                     (bytes_in_last_packet < pipe->max_packet)) {
2459                         pipe->pid_toggle = 1;
2460                         transaction->stage = CVMX_USB_STAGE_STATUS;
2461                 } else {
2462                         transaction->stage = CVMX_USB_STAGE_DATA;
2463                 }
2464                 break;
2465         case CVMX_USB_STAGE_STATUS:
2466                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2467                         transaction->stage =
2468                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2469                 else
2470                         cvmx_usb_complete(usb, pipe, transaction,
2471                                           CVMX_USB_STATUS_OK);
2472                 break;
2473         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2474                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2475                 break;
2476         }
2477 }
2478
2479 static void cvmx_usb_transfer_bulk(struct octeon_hcd *usb,
2480                                    struct cvmx_usb_pipe *pipe,
2481                                    struct cvmx_usb_transaction *transaction,
2482                                    union cvmx_usbcx_hcintx usbc_hcint,
2483                                    int buffer_space_left,
2484                                    int bytes_in_last_packet)
2485 {
2486         /*
2487          * The only time a bulk transfer isn't complete when it finishes with
2488          * an ACK is during a split transaction. For splits we need to continue
2489          * the transfer if more data is needed.
2490          */
2491         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2492                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2493                         transaction->stage =
2494                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2495                 else if (buffer_space_left &&
2496                          (bytes_in_last_packet == pipe->max_packet))
2497                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2498                 else
2499                         cvmx_usb_complete(usb, pipe, transaction,
2500                                           CVMX_USB_STATUS_OK);
2501         } else {
2502                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2503                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2504                     (usbc_hcint.s.nak))
2505                         pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2506                 if (!buffer_space_left ||
2507                     (bytes_in_last_packet < pipe->max_packet))
2508                         cvmx_usb_complete(usb, pipe, transaction,
2509                                           CVMX_USB_STATUS_OK);
2510         }
2511 }
2512
2513 static void cvmx_usb_transfer_intr(struct octeon_hcd *usb,
2514                                    struct cvmx_usb_pipe *pipe,
2515                                    struct cvmx_usb_transaction *transaction,
2516                                    int buffer_space_left,
2517                                    int bytes_in_last_packet)
2518 {
2519         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2520                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL) {
2521                         transaction->stage =
2522                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2523                 } else if (buffer_space_left &&
2524                            (bytes_in_last_packet == pipe->max_packet)) {
2525                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2526                 } else {
2527                         pipe->next_tx_frame += pipe->interval;
2528                         cvmx_usb_complete(usb, pipe, transaction,
2529                                           CVMX_USB_STATUS_OK);
2530                 }
2531         } else if (!buffer_space_left ||
2532                    (bytes_in_last_packet < pipe->max_packet)) {
2533                 pipe->next_tx_frame += pipe->interval;
2534                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2535         }
2536 }
2537
2538 static void cvmx_usb_transfer_isoc(struct octeon_hcd *usb,
2539                                    struct cvmx_usb_pipe *pipe,
2540                                    struct cvmx_usb_transaction *transaction,
2541                                    int buffer_space_left,
2542                                    int bytes_in_last_packet,
2543                                    int bytes_this_transfer)
2544 {
2545         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2546                 /*
2547                  * ISOCHRONOUS OUT splits don't require a complete split stage.
2548                  * Instead they use a sequence of begin OUT splits to transfer
2549                  * the data 188 bytes at a time. Once the transfer is complete,
2550                  * the pipe sleeps until the next schedule interval.
2551                  */
2552                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2553                         /*
2554                          * If no space left or this wasn't a max size packet
2555                          * then this transfer is complete. Otherwise start it
2556                          * again to send the next 188 bytes
2557                          */
2558                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2559                                 pipe->next_tx_frame += pipe->interval;
2560                                 cvmx_usb_complete(usb, pipe, transaction,
2561                                                   CVMX_USB_STATUS_OK);
2562                         }
2563                         return;
2564                 }
2565                 if (transaction->stage ==
2566                     CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2567                         /*
2568                          * We are in the incoming data phase. Keep getting data
2569                          * until we run out of space or get a small packet
2570                          */
2571                         if ((buffer_space_left == 0) ||
2572                             (bytes_in_last_packet < pipe->max_packet)) {
2573                                 pipe->next_tx_frame += pipe->interval;
2574                                 cvmx_usb_complete(usb, pipe, transaction,
2575                                                   CVMX_USB_STATUS_OK);
2576                         }
2577                 } else {
2578                         transaction->stage =
2579                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2580                 }
2581         } else {
2582                 pipe->next_tx_frame += pipe->interval;
2583                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2584         }
2585 }
2586
2587 /**
2588  * Poll a channel for status
2589  *
2590  * @usb:     USB device
2591  * @channel: Channel to poll
2592  *
2593  * Returns: Zero on success
2594  */
2595 static int cvmx_usb_poll_channel(struct octeon_hcd *usb, int channel)
2596 {
2597         struct usb_hcd *hcd = octeon_to_hcd(usb);
2598         struct device *dev = hcd->self.controller;
2599         union cvmx_usbcx_hcintx usbc_hcint;
2600         union cvmx_usbcx_hctsizx usbc_hctsiz;
2601         union cvmx_usbcx_hccharx usbc_hcchar;
2602         struct cvmx_usb_pipe *pipe;
2603         struct cvmx_usb_transaction *transaction;
2604         int bytes_this_transfer;
2605         int bytes_in_last_packet;
2606         int packets_processed;
2607         int buffer_space_left;
2608
2609         /* Read the interrupt status bits for the channel */
2610         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
2611                                              CVMX_USBCX_HCINTX(channel, usb->index));
2612
2613         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2614                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2615                                                       CVMX_USBCX_HCCHARX(channel,
2616                                                                          usb->index));
2617
2618                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2619                         /*
2620                          * There seems to be a bug in CN31XX which can cause
2621                          * interrupt IN transfers to get stuck until we do a
2622                          * write of HCCHARX without changing things
2623                          */
2624                         cvmx_usb_write_csr32(usb,
2625                                              CVMX_USBCX_HCCHARX(channel,
2626                                                                 usb->index),
2627                                              usbc_hcchar.u32);
2628                         return 0;
2629                 }
2630
2631                 /*
2632                  * In non DMA mode the channels don't halt themselves. We need
2633                  * to manually disable channels that are left running
2634                  */
2635                 if (!usbc_hcint.s.chhltd) {
2636                         if (usbc_hcchar.s.chena) {
2637                                 union cvmx_usbcx_hcintmskx hcintmsk;
2638                                 /* Disable all interrupts except CHHLTD */
2639                                 hcintmsk.u32 = 0;
2640                                 hcintmsk.s.chhltdmsk = 1;
2641                                 cvmx_usb_write_csr32(usb,
2642                                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
2643                                                      hcintmsk.u32);
2644                                 usbc_hcchar.s.chdis = 1;
2645                                 cvmx_usb_write_csr32(usb,
2646                                                      CVMX_USBCX_HCCHARX(channel, usb->index),
2647                                                      usbc_hcchar.u32);
2648                                 return 0;
2649                         } else if (usbc_hcint.s.xfercompl) {
2650                                 /*
2651                                  * Successful IN/OUT with transfer complete.
2652                                  * Channel halt isn't needed.
2653                                  */
2654                         } else {
2655                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2656                                         usb->index, channel);
2657                                 return 0;
2658                         }
2659                 }
2660         } else {
2661                 /*
2662                  * There is are no interrupts that we need to process when the
2663                  * channel is still running
2664                  */
2665                 if (!usbc_hcint.s.chhltd)
2666                         return 0;
2667         }
2668
2669         /* Disable the channel interrupts now that it is done */
2670         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2671         usb->idle_hardware_channels |= (1 << channel);
2672
2673         /* Make sure this channel is tied to a valid pipe */
2674         pipe = usb->pipe_for_channel[channel];
2675         prefetch(pipe);
2676         if (!pipe)
2677                 return 0;
2678         transaction = list_first_entry(&pipe->transactions,
2679                                        typeof(*transaction),
2680                                        node);
2681         prefetch(transaction);
2682
2683         /*
2684          * Disconnect this pipe from the HW channel. Later the schedule
2685          * function will figure out which pipe needs to go
2686          */
2687         usb->pipe_for_channel[channel] = NULL;
2688         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2689
2690         /*
2691          * Read the channel config info so we can figure out how much data
2692          * transferred
2693          */
2694         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2695                                               CVMX_USBCX_HCCHARX(channel, usb->index));
2696         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
2697                                               CVMX_USBCX_HCTSIZX(channel, usb->index));
2698
2699         /*
2700          * Calculating the number of bytes successfully transferred is dependent
2701          * on the transfer direction
2702          */
2703         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2704         if (usbc_hcchar.s.epdir) {
2705                 /*
2706                  * IN transactions are easy. For every byte received the
2707                  * hardware decrements xfersize. All we need to do is subtract
2708                  * the current value of xfersize from its starting value and we
2709                  * know how many bytes were written to the buffer
2710                  */
2711                 bytes_this_transfer = transaction->xfersize -
2712                         usbc_hctsiz.s.xfersize;
2713         } else {
2714                 /*
2715                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2716                  * decremented on every successful packet send. The hardware
2717                  * does this when it receives an ACK, or NYET. If it doesn't
2718                  * receive one of these responses pktcnt doesn't change
2719                  */
2720                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2721                 /*
2722                  * The last packet may not be a full transfer if we didn't have
2723                  * enough data
2724                  */
2725                 if (bytes_this_transfer > transaction->xfersize)
2726                         bytes_this_transfer = transaction->xfersize;
2727         }
2728         /* Figure out how many bytes were in the last packet of the transfer */
2729         if (packets_processed)
2730                 bytes_in_last_packet = bytes_this_transfer -
2731                         (packets_processed - 1) * usbc_hcchar.s.mps;
2732         else
2733                 bytes_in_last_packet = bytes_this_transfer;
2734
2735         /*
2736          * As a special case, setup transactions output the setup header, not
2737          * the user's data. For this reason we don't count setup data as bytes
2738          * transferred
2739          */
2740         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2741             (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2742                 bytes_this_transfer = 0;
2743
2744         /*
2745          * Add the bytes transferred to the running total. It is important that
2746          * bytes_this_transfer doesn't count any data that needs to be
2747          * retransmitted
2748          */
2749         transaction->actual_bytes += bytes_this_transfer;
2750         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2751                 buffer_space_left = transaction->iso_packets[0].length -
2752                         transaction->actual_bytes;
2753         else
2754                 buffer_space_left = transaction->buffer_length -
2755                         transaction->actual_bytes;
2756
2757         /*
2758          * We need to remember the PID toggle state for the next transaction.
2759          * The hardware already updated it for the next transaction
2760          */
2761         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2762
2763         /*
2764          * For high speed bulk out, assume the next transaction will need to do
2765          * a ping before proceeding. If this isn't true the ACK processing below
2766          * will clear this flag
2767          */
2768         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2769             (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2770             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2771                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2772
2773         if (WARN_ON_ONCE(bytes_this_transfer < 0)) {
2774                 /*
2775                  * In some rare cases the DMA engine seems to get stuck and
2776                  * keeps substracting same byte count over and over again. In
2777                  * such case we just need to fail every transaction.
2778                  */
2779                 cvmx_usb_complete(usb, pipe, transaction,
2780                                   CVMX_USB_STATUS_ERROR);
2781                 return 0;
2782         }
2783
2784         if (usbc_hcint.s.stall) {
2785                 /*
2786                  * STALL as a response means this transaction cannot be
2787                  * completed because the device can't process transactions. Tell
2788                  * the user. Any data that was transferred will be counted on
2789                  * the actual bytes transferred
2790                  */
2791                 pipe->pid_toggle = 0;
2792                 cvmx_usb_complete(usb, pipe, transaction,
2793                                   CVMX_USB_STATUS_STALL);
2794         } else if (usbc_hcint.s.xacterr) {
2795                 /*
2796                  * XactErr as a response means the device signaled
2797                  * something wrong with the transfer. For example, PID
2798                  * toggle errors cause these.
2799                  */
2800                 cvmx_usb_complete(usb, pipe, transaction,
2801                                   CVMX_USB_STATUS_XACTERR);
2802         } else if (usbc_hcint.s.bblerr) {
2803                 /* Babble Error (BblErr) */
2804                 cvmx_usb_complete(usb, pipe, transaction,
2805                                   CVMX_USB_STATUS_BABBLEERR);
2806         } else if (usbc_hcint.s.datatglerr) {
2807                 /* Data toggle error */
2808                 cvmx_usb_complete(usb, pipe, transaction,
2809                                   CVMX_USB_STATUS_DATATGLERR);
2810         } else if (usbc_hcint.s.nyet) {
2811                 /*
2812                  * NYET as a response is only allowed in three cases: as a
2813                  * response to a ping, as a response to a split transaction, and
2814                  * as a response to a bulk out. The ping case is handled by
2815                  * hardware, so we only have splits and bulk out
2816                  */
2817                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
2818                         transaction->retries = 0;
2819                         /*
2820                          * If there is more data to go then we need to try
2821                          * again. Otherwise this transaction is complete
2822                          */
2823                         if ((buffer_space_left == 0) ||
2824                             (bytes_in_last_packet < pipe->max_packet))
2825                                 cvmx_usb_complete(usb, pipe,
2826                                                   transaction,
2827                                                   CVMX_USB_STATUS_OK);
2828                 } else {
2829                         /*
2830                          * Split transactions retry the split complete 4 times
2831                          * then rewind to the start split and do the entire
2832                          * transactions again
2833                          */
2834                         transaction->retries++;
2835                         if ((transaction->retries & 0x3) == 0) {
2836                                 /*
2837                                  * Rewind to the beginning of the transaction by
2838                                  * anding off the split complete bit
2839                                  */
2840                                 transaction->stage &= ~1;
2841                                 pipe->split_sc_frame = -1;
2842                         }
2843                 }
2844         } else if (usbc_hcint.s.ack) {
2845                 transaction->retries = 0;
2846                 /*
2847                  * The ACK bit can only be checked after the other error bits.
2848                  * This is because a multi packet transfer may succeed in a
2849                  * number of packets and then get a different response on the
2850                  * last packet. In this case both ACK and the last response bit
2851                  * will be set. If none of the other response bits is set, then
2852                  * the last packet must have been an ACK
2853                  *
2854                  * Since we got an ACK, we know we don't need to do a ping on
2855                  * this pipe
2856                  */
2857                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
2858
2859                 switch (transaction->type) {
2860                 case CVMX_USB_TRANSFER_CONTROL:
2861                         cvmx_usb_transfer_control(usb, pipe, transaction,
2862                                                   usbc_hcchar,
2863                                                   buffer_space_left,
2864                                                   bytes_in_last_packet);
2865                         break;
2866                 case CVMX_USB_TRANSFER_BULK:
2867                         cvmx_usb_transfer_bulk(usb, pipe, transaction,
2868                                                usbc_hcint, buffer_space_left,
2869                                                bytes_in_last_packet);
2870                         break;
2871                 case CVMX_USB_TRANSFER_INTERRUPT:
2872                         cvmx_usb_transfer_intr(usb, pipe, transaction,
2873                                                buffer_space_left,
2874                                                bytes_in_last_packet);
2875                         break;
2876                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2877                         cvmx_usb_transfer_isoc(usb, pipe, transaction,
2878                                                buffer_space_left,
2879                                                bytes_in_last_packet,
2880                                                bytes_this_transfer);
2881                         break;
2882                 }
2883         } else if (usbc_hcint.s.nak) {
2884                 /*
2885                  * If this was a split then clear our split in progress marker.
2886                  */
2887                 if (usb->active_split == transaction)
2888                         usb->active_split = NULL;
2889                 /*
2890                  * NAK as a response means the device couldn't accept the
2891                  * transaction, but it should be retried in the future. Rewind
2892                  * to the beginning of the transaction by anding off the split
2893                  * complete bit. Retry in the next interval
2894                  */
2895                 transaction->retries = 0;
2896                 transaction->stage &= ~1;
2897                 pipe->next_tx_frame += pipe->interval;
2898                 if (pipe->next_tx_frame < usb->frame_number)
2899                         pipe->next_tx_frame = usb->frame_number +
2900                                 pipe->interval -
2901                                 (usb->frame_number - pipe->next_tx_frame) %
2902                                 pipe->interval;
2903         } else {
2904                 struct cvmx_usb_port_status port;
2905
2906                 port = cvmx_usb_get_status(usb);
2907                 if (port.port_enabled) {
2908                         /* We'll retry the exact same transaction again */
2909                         transaction->retries++;
2910                 } else {
2911                         /*
2912                          * We get channel halted interrupts with no result bits
2913                          * sets when the cable is unplugged
2914                          */
2915                         cvmx_usb_complete(usb, pipe, transaction,
2916                                           CVMX_USB_STATUS_ERROR);
2917                 }
2918         }
2919         return 0;
2920 }
2921
2922 static void octeon_usb_port_callback(struct octeon_hcd *usb)
2923 {
2924         spin_unlock(&usb->lock);
2925         usb_hcd_poll_rh_status(octeon_to_hcd(usb));
2926         spin_lock(&usb->lock);
2927 }
2928
2929 /**
2930  * Poll the USB block for status and call all needed callback
2931  * handlers. This function is meant to be called in the interrupt
2932  * handler for the USB controller. It can also be called
2933  * periodically in a loop for non-interrupt based operation.
2934  *
2935  * @usb: USB device state populated by cvmx_usb_initialize().
2936  *
2937  * Returns: 0 or a negative error code.
2938  */
2939 static int cvmx_usb_poll(struct octeon_hcd *usb)
2940 {
2941         union cvmx_usbcx_hfnum usbc_hfnum;
2942         union cvmx_usbcx_gintsts usbc_gintsts;
2943
2944         prefetch_range(usb, sizeof(*usb));
2945
2946         /* Update the frame counter */
2947         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2948         if ((usb->frame_number & 0x3fff) > usbc_hfnum.s.frnum)
2949                 usb->frame_number += 0x4000;
2950         usb->frame_number &= ~0x3fffull;
2951         usb->frame_number |= usbc_hfnum.s.frnum;
2952
2953         /* Read the pending interrupts */
2954         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
2955                                                CVMX_USBCX_GINTSTS(usb->index));
2956
2957         /* Clear the interrupts now that we know about them */
2958         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
2959                              usbc_gintsts.u32);
2960
2961         if (usbc_gintsts.s.rxflvl) {
2962                 /*
2963                  * RxFIFO Non-Empty (RxFLvl)
2964                  * Indicates that there is at least one packet pending to be
2965                  * read from the RxFIFO.
2966                  *
2967                  * In DMA mode this is handled by hardware
2968                  */
2969                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2970                         cvmx_usb_poll_rx_fifo(usb);
2971         }
2972         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
2973                 /* Fill the Tx FIFOs when not in DMA mode */
2974                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2975                         cvmx_usb_poll_tx_fifo(usb);
2976         }
2977         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
2978                 union cvmx_usbcx_hprt usbc_hprt;
2979                 /*
2980                  * Disconnect Detected Interrupt (DisconnInt)
2981                  * Asserted when a device disconnect is detected.
2982                  *
2983                  * Host Port Interrupt (PrtInt)
2984                  * The core sets this bit to indicate a change in port status of
2985                  * one of the O2P USB core ports in Host mode. The application
2986                  * must read the Host Port Control and Status (HPRT) register to
2987                  * determine the exact event that caused this interrupt. The
2988                  * application must clear the appropriate status bit in the Host
2989                  * Port Control and Status register to clear this bit.
2990                  *
2991                  * Call the user's port callback
2992                  */
2993                 octeon_usb_port_callback(usb);
2994                 /* Clear the port change bits */
2995                 usbc_hprt.u32 =
2996                         cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
2997                 usbc_hprt.s.prtena = 0;
2998                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
2999                                      usbc_hprt.u32);
3000         }
3001         if (usbc_gintsts.s.hchint) {
3002                 /*
3003                  * Host Channels Interrupt (HChInt)
3004                  * The core sets this bit to indicate that an interrupt is
3005                  * pending on one of the channels of the core (in Host mode).
3006                  * The application must read the Host All Channels Interrupt
3007                  * (HAINT) register to determine the exact number of the channel
3008                  * on which the interrupt occurred, and then read the
3009                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3010                  * determine the exact cause of the interrupt. The application
3011                  * must clear the appropriate status bit in the HCINTn register
3012                  * to clear this bit.
3013                  */
3014                 union cvmx_usbcx_haint usbc_haint;
3015
3016                 usbc_haint.u32 = cvmx_usb_read_csr32(usb,
3017                                                      CVMX_USBCX_HAINT(usb->index));
3018                 while (usbc_haint.u32) {
3019                         int channel;
3020
3021                         channel = __fls(usbc_haint.u32);
3022                         cvmx_usb_poll_channel(usb, channel);
3023                         usbc_haint.u32 ^= 1 << channel;
3024                 }
3025         }
3026
3027         cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3028
3029         return 0;
3030 }
3031
3032 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3033 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3034 {
3035         return (struct octeon_hcd *)(hcd->hcd_priv);
3036 }
3037
3038 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3039 {
3040         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3041         unsigned long flags;
3042
3043         spin_lock_irqsave(&usb->lock, flags);
3044         cvmx_usb_poll(usb);
3045         spin_unlock_irqrestore(&usb->lock, flags);
3046         return IRQ_HANDLED;
3047 }
3048
3049 static int octeon_usb_start(struct usb_hcd *hcd)
3050 {
3051         hcd->state = HC_STATE_RUNNING;
3052         return 0;
3053 }
3054
3055 static void octeon_usb_stop(struct usb_hcd *hcd)
3056 {
3057         hcd->state = HC_STATE_HALT;
3058 }
3059
3060 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3061 {
3062         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3063
3064         return cvmx_usb_get_frame_number(usb);
3065 }
3066
3067 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3068                                   struct urb *urb,
3069                                   gfp_t mem_flags)
3070 {
3071         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3072         struct device *dev = hcd->self.controller;
3073         struct cvmx_usb_transaction *transaction = NULL;
3074         struct cvmx_usb_pipe *pipe;
3075         unsigned long flags;
3076         struct cvmx_usb_iso_packet *iso_packet;
3077         struct usb_host_endpoint *ep = urb->ep;
3078         int rc;
3079
3080         urb->status = 0;
3081         spin_lock_irqsave(&usb->lock, flags);
3082
3083         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3084         if (rc) {
3085                 spin_unlock_irqrestore(&usb->lock, flags);
3086                 return rc;
3087         }
3088
3089         if (!ep->hcpriv) {
3090                 enum cvmx_usb_transfer transfer_type;
3091                 enum cvmx_usb_speed speed;
3092                 int split_device = 0;
3093                 int split_port = 0;
3094
3095                 switch (usb_pipetype(urb->pipe)) {
3096                 case PIPE_ISOCHRONOUS:
3097                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3098                         break;
3099                 case PIPE_INTERRUPT:
3100                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3101                         break;
3102                 case PIPE_CONTROL:
3103                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3104                         break;
3105                 default:
3106                         transfer_type = CVMX_USB_TRANSFER_BULK;
3107                         break;
3108                 }
3109                 switch (urb->dev->speed) {
3110                 case USB_SPEED_LOW:
3111                         speed = CVMX_USB_SPEED_LOW;
3112                         break;
3113                 case USB_SPEED_FULL:
3114                         speed = CVMX_USB_SPEED_FULL;
3115                         break;
3116                 default:
3117                         speed = CVMX_USB_SPEED_HIGH;
3118                         break;
3119                 }
3120                 /*
3121                  * For slow devices on high speed ports we need to find the hub
3122                  * that does the speed translation so we know where to send the
3123                  * split transactions.
3124                  */
3125                 if (speed != CVMX_USB_SPEED_HIGH) {
3126                         /*
3127                          * Start at this device and work our way up the usb
3128                          * tree.
3129                          */
3130                         struct usb_device *dev = urb->dev;
3131
3132                         while (dev->parent) {
3133                                 /*
3134                                  * If our parent is high speed then he'll
3135                                  * receive the splits.
3136                                  */
3137                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3138                                         split_device = dev->parent->devnum;
3139                                         split_port = dev->portnum;
3140                                         break;
3141                                 }
3142                                 /*
3143                                  * Move up the tree one level. If we make it all
3144                                  * the way up the tree, then the port must not
3145                                  * be in high speed mode and we don't need a
3146                                  * split.
3147                                  */
3148                                 dev = dev->parent;
3149                         }
3150                 }
3151                 pipe = cvmx_usb_open_pipe(usb, usb_pipedevice(urb->pipe),
3152                                           usb_pipeendpoint(urb->pipe), speed,
3153                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3154                                           & 0x7ff,
3155                                           transfer_type,
3156                                           usb_pipein(urb->pipe) ?
3157                                                 CVMX_USB_DIRECTION_IN :
3158                                                 CVMX_USB_DIRECTION_OUT,
3159                                           urb->interval,
3160                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3161                                            >> 11) & 0x3,
3162                                           split_device, split_port);
3163                 if (!pipe) {
3164                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3165                         spin_unlock_irqrestore(&usb->lock, flags);
3166                         dev_dbg(dev, "Failed to create pipe\n");
3167                         return -ENOMEM;
3168                 }
3169                 ep->hcpriv = pipe;
3170         } else {
3171                 pipe = ep->hcpriv;
3172         }
3173
3174         switch (usb_pipetype(urb->pipe)) {
3175         case PIPE_ISOCHRONOUS:
3176                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3177                         usb_pipedevice(urb->pipe),
3178                         usb_pipeendpoint(urb->pipe));
3179                 /*
3180                  * Allocate a structure to use for our private list of
3181                  * isochronous packets.
3182                  */
3183                 iso_packet = kmalloc_array(urb->number_of_packets,
3184                                            sizeof(struct cvmx_usb_iso_packet),
3185                                            GFP_ATOMIC);
3186                 if (iso_packet) {
3187                         int i;
3188                         /* Fill the list with the data from the URB */
3189                         for (i = 0; i < urb->number_of_packets; i++) {
3190                                 iso_packet[i].offset =
3191                                         urb->iso_frame_desc[i].offset;
3192                                 iso_packet[i].length =
3193                                         urb->iso_frame_desc[i].length;
3194                                 iso_packet[i].status = CVMX_USB_STATUS_ERROR;
3195                         }
3196                         /*
3197                          * Store a pointer to the list in the URB setup_packet
3198                          * field. We know this currently isn't being used and
3199                          * this saves us a bunch of logic.
3200                          */
3201                         urb->setup_packet = (char *)iso_packet;
3202                         transaction = cvmx_usb_submit_isochronous(usb,
3203                                                                   pipe, urb);
3204                         /*
3205                          * If submit failed we need to free our private packet
3206                          * list.
3207                          */
3208                         if (!transaction) {
3209                                 urb->setup_packet = NULL;
3210                                 kfree(iso_packet);
3211                         }
3212                 }
3213                 break;
3214         case PIPE_INTERRUPT:
3215                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3216                         usb_pipedevice(urb->pipe),
3217                         usb_pipeendpoint(urb->pipe));
3218                 transaction = cvmx_usb_submit_interrupt(usb, pipe, urb);
3219                 break;
3220         case PIPE_CONTROL:
3221                 dev_dbg(dev, "Submit control to %d.%d\n",
3222                         usb_pipedevice(urb->pipe),
3223                         usb_pipeendpoint(urb->pipe));
3224                 transaction = cvmx_usb_submit_control(usb, pipe, urb);
3225                 break;
3226         case PIPE_BULK:
3227                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3228                         usb_pipedevice(urb->pipe),
3229                         usb_pipeendpoint(urb->pipe));
3230                 transaction = cvmx_usb_submit_bulk(usb, pipe, urb);
3231                 break;
3232         }
3233         if (!transaction) {
3234                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3235                 spin_unlock_irqrestore(&usb->lock, flags);
3236                 dev_dbg(dev, "Failed to submit\n");
3237                 return -ENOMEM;
3238         }
3239         urb->hcpriv = transaction;
3240         spin_unlock_irqrestore(&usb->lock, flags);
3241         return 0;
3242 }
3243
3244 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3245                                   struct urb *urb,
3246                                   int status)
3247 {
3248         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3249         unsigned long flags;
3250         int rc;
3251
3252         if (!urb->dev)
3253                 return -EINVAL;
3254
3255         spin_lock_irqsave(&usb->lock, flags);
3256
3257         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3258         if (rc)
3259                 goto out;
3260
3261         urb->status = status;
3262         cvmx_usb_cancel(usb, urb->ep->hcpriv, urb->hcpriv);
3263
3264 out:
3265         spin_unlock_irqrestore(&usb->lock, flags);
3266
3267         return rc;
3268 }
3269
3270 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3271                                         struct usb_host_endpoint *ep)
3272 {
3273         struct device *dev = hcd->self.controller;
3274
3275         if (ep->hcpriv) {
3276                 struct octeon_hcd *usb = hcd_to_octeon(hcd);
3277                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3278                 unsigned long flags;
3279
3280                 spin_lock_irqsave(&usb->lock, flags);
3281                 cvmx_usb_cancel_all(usb, pipe);
3282                 if (cvmx_usb_close_pipe(usb, pipe))
3283                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3284                 spin_unlock_irqrestore(&usb->lock, flags);
3285                 ep->hcpriv = NULL;
3286         }
3287 }
3288
3289 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3290 {
3291         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3292         struct cvmx_usb_port_status port_status;
3293         unsigned long flags;
3294
3295         spin_lock_irqsave(&usb->lock, flags);
3296         port_status = cvmx_usb_get_status(usb);
3297         spin_unlock_irqrestore(&usb->lock, flags);
3298         buf[0] = port_status.connect_change << 1;
3299
3300         return buf[0] != 0;
3301 }
3302
3303 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3304                                   u16 wIndex, char *buf, u16 wLength)
3305 {
3306         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3307         struct device *dev = hcd->self.controller;
3308         struct cvmx_usb_port_status usb_port_status;
3309         int port_status;
3310         struct usb_hub_descriptor *desc;
3311         unsigned long flags;
3312
3313         switch (typeReq) {
3314         case ClearHubFeature:
3315                 dev_dbg(dev, "ClearHubFeature\n");
3316                 switch (wValue) {
3317                 case C_HUB_LOCAL_POWER:
3318                 case C_HUB_OVER_CURRENT:
3319                         /* Nothing required here */
3320                         break;
3321                 default:
3322                         return -EINVAL;
3323                 }
3324                 break;
3325         case ClearPortFeature:
3326                 dev_dbg(dev, "ClearPortFeature\n");
3327                 if (wIndex != 1) {
3328                         dev_dbg(dev, " INVALID\n");
3329                         return -EINVAL;
3330                 }
3331
3332                 switch (wValue) {
3333                 case USB_PORT_FEAT_ENABLE:
3334                         dev_dbg(dev, " ENABLE\n");
3335                         spin_lock_irqsave(&usb->lock, flags);
3336                         cvmx_usb_disable(usb);
3337                         spin_unlock_irqrestore(&usb->lock, flags);
3338                         break;
3339                 case USB_PORT_FEAT_SUSPEND:
3340                         dev_dbg(dev, " SUSPEND\n");
3341                         /* Not supported on Octeon */
3342                         break;
3343                 case USB_PORT_FEAT_POWER:
3344                         dev_dbg(dev, " POWER\n");
3345                         /* Not supported on Octeon */
3346                         break;
3347                 case USB_PORT_FEAT_INDICATOR:
3348                         dev_dbg(dev, " INDICATOR\n");
3349                         /* Port inidicator not supported */
3350                         break;
3351                 case USB_PORT_FEAT_C_CONNECTION:
3352                         dev_dbg(dev, " C_CONNECTION\n");
3353                         /* Clears drivers internal connect status change flag */
3354                         spin_lock_irqsave(&usb->lock, flags);
3355                         usb->port_status = cvmx_usb_get_status(usb);
3356                         spin_unlock_irqrestore(&usb->lock, flags);
3357                         break;
3358                 case USB_PORT_FEAT_C_RESET:
3359                         dev_dbg(dev, " C_RESET\n");
3360                         /*
3361                          * Clears the driver's internal Port Reset Change flag.
3362                          */
3363                         spin_lock_irqsave(&usb->lock, flags);
3364                         usb->port_status = cvmx_usb_get_status(usb);
3365                         spin_unlock_irqrestore(&usb->lock, flags);
3366                         break;
3367                 case USB_PORT_FEAT_C_ENABLE:
3368                         dev_dbg(dev, " C_ENABLE\n");
3369                         /*
3370                          * Clears the driver's internal Port Enable/Disable
3371                          * Change flag.
3372                          */
3373                         spin_lock_irqsave(&usb->lock, flags);
3374                         usb->port_status = cvmx_usb_get_status(usb);
3375                         spin_unlock_irqrestore(&usb->lock, flags);
3376                         break;
3377                 case USB_PORT_FEAT_C_SUSPEND:
3378                         dev_dbg(dev, " C_SUSPEND\n");
3379                         /*
3380                          * Clears the driver's internal Port Suspend Change
3381                          * flag, which is set when resume signaling on the host
3382                          * port is complete.
3383                          */
3384                         break;
3385                 case USB_PORT_FEAT_C_OVER_CURRENT:
3386                         dev_dbg(dev, " C_OVER_CURRENT\n");
3387                         /* Clears the driver's overcurrent Change flag */
3388                         spin_lock_irqsave(&usb->lock, flags);
3389                         usb->port_status = cvmx_usb_get_status(usb);
3390                         spin_unlock_irqrestore(&usb->lock, flags);
3391                         break;
3392                 default:
3393                         dev_dbg(dev, " UNKNOWN\n");
3394                         return -EINVAL;
3395                 }
3396                 break;
3397         case GetHubDescriptor:
3398                 dev_dbg(dev, "GetHubDescriptor\n");
3399                 desc = (struct usb_hub_descriptor *)buf;
3400                 desc->bDescLength = 9;
3401                 desc->bDescriptorType = 0x29;
3402                 desc->bNbrPorts = 1;
3403                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3404                 desc->bPwrOn2PwrGood = 1;
3405                 desc->bHubContrCurrent = 0;
3406                 desc->u.hs.DeviceRemovable[0] = 0;
3407                 desc->u.hs.DeviceRemovable[1] = 0xff;
3408                 break;
3409         case GetHubStatus:
3410                 dev_dbg(dev, "GetHubStatus\n");
3411                 *(__le32 *)buf = 0;
3412                 break;
3413         case GetPortStatus:
3414                 dev_dbg(dev, "GetPortStatus\n");
3415                 if (wIndex != 1) {
3416                         dev_dbg(dev, " INVALID\n");
3417                         return -EINVAL;
3418                 }
3419
3420                 spin_lock_irqsave(&usb->lock, flags);
3421                 usb_port_status = cvmx_usb_get_status(usb);
3422                 spin_unlock_irqrestore(&usb->lock, flags);
3423                 port_status = 0;
3424
3425                 if (usb_port_status.connect_change) {
3426                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3427                         dev_dbg(dev, " C_CONNECTION\n");
3428                 }
3429
3430                 if (usb_port_status.port_enabled) {
3431                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3432                         dev_dbg(dev, " C_ENABLE\n");
3433                 }
3434
3435                 if (usb_port_status.connected) {
3436                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3437                         dev_dbg(dev, " CONNECTION\n");
3438                 }
3439
3440                 if (usb_port_status.port_enabled) {
3441                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3442                         dev_dbg(dev, " ENABLE\n");
3443                 }
3444
3445                 if (usb_port_status.port_over_current) {
3446                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3447                         dev_dbg(dev, " OVER_CURRENT\n");
3448                 }
3449
3450                 if (usb_port_status.port_powered) {
3451                         port_status |= (1 << USB_PORT_FEAT_POWER);
3452                         dev_dbg(dev, " POWER\n");
3453                 }
3454
3455                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3456                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3457                         dev_dbg(dev, " HIGHSPEED\n");
3458                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3459                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3460                         dev_dbg(dev, " LOWSPEED\n");
3461                 }
3462
3463                 *((__le32 *)buf) = cpu_to_le32(port_status);
3464                 break;
3465         case SetHubFeature:
3466                 dev_dbg(dev, "SetHubFeature\n");
3467                 /* No HUB features supported */
3468                 break;
3469         case SetPortFeature:
3470                 dev_dbg(dev, "SetPortFeature\n");
3471                 if (wIndex != 1) {
3472                         dev_dbg(dev, " INVALID\n");
3473                         return -EINVAL;
3474                 }
3475
3476                 switch (wValue) {
3477                 case USB_PORT_FEAT_SUSPEND:
3478                         dev_dbg(dev, " SUSPEND\n");
3479                         return -EINVAL;
3480                 case USB_PORT_FEAT_POWER:
3481                         dev_dbg(dev, " POWER\n");
3482                         /*
3483                          * Program the port power bit to drive VBUS on the USB.
3484                          */
3485                         spin_lock_irqsave(&usb->lock, flags);
3486                         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
3487                                         cvmx_usbcx_hprt, prtpwr, 1);
3488                         spin_unlock_irqrestore(&usb->lock, flags);
3489                         return 0;
3490                 case USB_PORT_FEAT_RESET:
3491                         dev_dbg(dev, " RESET\n");
3492                         spin_lock_irqsave(&usb->lock, flags);
3493                         cvmx_usb_reset_port(usb);
3494                         spin_unlock_irqrestore(&usb->lock, flags);
3495                         return 0;
3496                 case USB_PORT_FEAT_INDICATOR:
3497                         dev_dbg(dev, " INDICATOR\n");
3498                         /* Not supported */
3499                         break;
3500                 default:
3501                         dev_dbg(dev, " UNKNOWN\n");
3502                         return -EINVAL;
3503                 }
3504                 break;
3505         default:
3506                 dev_dbg(dev, "Unknown root hub request\n");
3507                 return -EINVAL;
3508         }
3509         return 0;
3510 }
3511
3512 static const struct hc_driver octeon_hc_driver = {
3513         .description            = "Octeon USB",
3514         .product_desc           = "Octeon Host Controller",
3515         .hcd_priv_size          = sizeof(struct octeon_hcd),
3516         .irq                    = octeon_usb_irq,
3517         .flags                  = HCD_MEMORY | HCD_DMA | HCD_USB2,
3518         .start                  = octeon_usb_start,
3519         .stop                   = octeon_usb_stop,
3520         .urb_enqueue            = octeon_usb_urb_enqueue,
3521         .urb_dequeue            = octeon_usb_urb_dequeue,
3522         .endpoint_disable       = octeon_usb_endpoint_disable,
3523         .get_frame_number       = octeon_usb_get_frame_number,
3524         .hub_status_data        = octeon_usb_hub_status_data,
3525         .hub_control            = octeon_usb_hub_control,
3526         .map_urb_for_dma        = octeon_map_urb_for_dma,
3527         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3528 };
3529
3530 static int octeon_usb_probe(struct platform_device *pdev)
3531 {
3532         int status;
3533         int initialize_flags;
3534         int usb_num;
3535         struct resource *res_mem;
3536         struct device_node *usbn_node;
3537         int irq = platform_get_irq(pdev, 0);
3538         struct device *dev = &pdev->dev;
3539         struct octeon_hcd *usb;
3540         struct usb_hcd *hcd;
3541         u32 clock_rate = 48000000;
3542         bool is_crystal_clock = false;
3543         const char *clock_type;
3544         int i;
3545
3546         if (!dev->of_node) {
3547                 dev_err(dev, "Error: empty of_node\n");
3548                 return -ENXIO;
3549         }
3550         usbn_node = dev->of_node->parent;
3551
3552         i = of_property_read_u32(usbn_node,
3553                                  "clock-frequency", &clock_rate);
3554         if (i)
3555                 i = of_property_read_u32(usbn_node,
3556                                          "refclk-frequency", &clock_rate);
3557         if (i) {
3558                 dev_err(dev, "No USBN \"clock-frequency\"\n");
3559                 return -ENXIO;
3560         }
3561         switch (clock_rate) {
3562         case 12000000:
3563                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3564                 break;
3565         case 24000000:
3566                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3567                 break;
3568         case 48000000:
3569                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3570                 break;
3571         default:
3572                 dev_err(dev, "Illegal USBN \"clock-frequency\" %u\n",
3573                         clock_rate);
3574                 return -ENXIO;
3575         }
3576
3577         i = of_property_read_string(usbn_node,
3578                                     "cavium,refclk-type", &clock_type);
3579         if (i)
3580                 i = of_property_read_string(usbn_node,
3581                                             "refclk-type", &clock_type);
3582
3583         if (!i && strcmp("crystal", clock_type) == 0)
3584                 is_crystal_clock = true;
3585
3586         if (is_crystal_clock)
3587                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3588         else
3589                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3590
3591         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3592         if (!res_mem) {
3593                 dev_err(dev, "found no memory resource\n");
3594                 return -ENXIO;
3595         }
3596         usb_num = (res_mem->start >> 44) & 1;
3597
3598         if (irq < 0) {
3599                 /* Defective device tree, but we know how to fix it. */
3600                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3601
3602                 irq = irq_create_mapping(NULL, hwirq);
3603         }
3604
3605         /*
3606          * Set the DMA mask to 64bits so we get buffers already translated for
3607          * DMA.
3608          */
3609         i = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
3610         if (i)
3611                 return i;
3612
3613         /*
3614          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3615          * IOB priority registers.  Under heavy network load USB
3616          * hardware can be starved by the IOB causing a crash.  Give
3617          * it a priority boost if it has been waiting more than 400
3618          * cycles to avoid this situation.
3619          *
3620          * Testing indicates that a cnt_val of 8192 is not sufficient,
3621          * but no failures are seen with 4096.  We choose a value of
3622          * 400 to give a safety factor of 10.
3623          */
3624         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3625                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3626
3627                 pri_cnt.u64 = 0;
3628                 pri_cnt.s.cnt_enb = 1;
3629                 pri_cnt.s.cnt_val = 400;
3630                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3631         }
3632
3633         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3634         if (!hcd) {
3635                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3636                 return -1;
3637         }
3638         hcd->uses_new_polling = 1;
3639         usb = (struct octeon_hcd *)hcd->hcd_priv;
3640
3641         spin_lock_init(&usb->lock);
3642
3643         usb->init_flags = initialize_flags;
3644
3645         /* Initialize the USB state structure */
3646         usb->index = usb_num;
3647         INIT_LIST_HEAD(&usb->idle_pipes);
3648         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
3649                 INIT_LIST_HEAD(&usb->active_pipes[i]);
3650
3651         /* Due to an errata, CN31XX doesn't support DMA */
3652         if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
3653                 usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
3654                 /* Only use one channel with non DMA */
3655                 usb->idle_hardware_channels = 0x1;
3656         } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
3657                 /* CN5XXX have an errata with channel 3 */
3658                 usb->idle_hardware_channels = 0xf7;
3659         } else {
3660                 usb->idle_hardware_channels = 0xff;
3661         }
3662
3663         status = cvmx_usb_initialize(dev, usb);
3664         if (status) {
3665                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3666                 usb_put_hcd(hcd);
3667                 return -1;
3668         }
3669
3670         status = usb_add_hcd(hcd, irq, 0);
3671         if (status) {
3672                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3673                 usb_put_hcd(hcd);
3674                 return -1;
3675         }
3676         device_wakeup_enable(hcd->self.controller);
3677
3678         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3679
3680         return 0;
3681 }
3682
3683 static int octeon_usb_remove(struct platform_device *pdev)
3684 {
3685         int status;
3686         struct device *dev = &pdev->dev;
3687         struct usb_hcd *hcd = dev_get_drvdata(dev);
3688         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3689         unsigned long flags;
3690
3691         usb_remove_hcd(hcd);
3692         spin_lock_irqsave(&usb->lock, flags);
3693         status = cvmx_usb_shutdown(usb);
3694         spin_unlock_irqrestore(&usb->lock, flags);
3695         if (status)
3696                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3697
3698         usb_put_hcd(hcd);
3699
3700         return 0;
3701 }
3702
3703 static const struct of_device_id octeon_usb_match[] = {
3704         {
3705                 .compatible = "cavium,octeon-5750-usbc",
3706         },
3707         {},
3708 };
3709 MODULE_DEVICE_TABLE(of, octeon_usb_match);
3710
3711 static struct platform_driver octeon_usb_driver = {
3712         .driver = {
3713                 .name           = "octeon-hcd",
3714                 .of_match_table = octeon_usb_match,
3715         },
3716         .probe      = octeon_usb_probe,
3717         .remove     = octeon_usb_remove,
3718 };
3719
3720 static int __init octeon_usb_driver_init(void)
3721 {
3722         if (usb_disabled())
3723                 return 0;
3724
3725         return platform_driver_register(&octeon_usb_driver);
3726 }
3727 module_init(octeon_usb_driver_init);
3728
3729 static void __exit octeon_usb_driver_exit(void)
3730 {
3731         if (usb_disabled())
3732                 return;
3733
3734         platform_driver_unregister(&octeon_usb_driver);
3735 }
3736 module_exit(octeon_usb_driver_exit);
3737
3738 MODULE_LICENSE("GPL");
3739 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3740 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");