drivers/staging/octeon-usb/octeon-hcd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file is subject to the terms and conditions of the GNU General Public
   4  * License.  See the file "COPYING" in the main directory of this archive
   5  * for more details.
   6  *
   7  * Copyright (C) 2008 Cavium Networks
   8  *
   9  * Some parts of the code were originally released under BSD license:
  10  *
  11  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  12  * reserved.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions are
  16  * met:
  17  *
  18  *   * Redistributions of source code must retain the above copyright
  19  *     notice, this list of conditions and the following disclaimer.
  20  *
  21  *   * Redistributions in binary form must reproduce the above
  22  *     copyright notice, this list of conditions and the following
  23  *     disclaimer in the documentation and/or other materials provided
  24  *     with the distribution.
  25  *
  26  *   * Neither the name of Cavium Networks nor the names of
  27  *     its contributors may be used to endorse or promote products
  28  *     derived from this software without specific prior written
  29  *     permission.
  30  *
  31  * This Software, including technical data, may be subject to U.S. export
  32  * control laws, including the U.S. Export Administration Act and its associated
  33  * regulations, and may be subject to export or import regulations in other
  34  * countries.
  35  *
  36  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  37  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  38  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  39  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  40  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  41  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  42  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  43  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  44  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  45  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  46  */
  47
  48 #include <linux/usb.h>
  49 #include <linux/slab.h>
  50 #include <linux/module.h>
  51 #include <linux/usb/hcd.h>
  52 #include <linux/prefetch.h>
  53 #include <linux/platform_device.h>
  54
  55 #include <asm/octeon/octeon.h>
  56
  57 #include "octeon-hcd.h"
  58
  59 /**
  60  * enum cvmx_usb_speed - the possible USB device speeds
  61  *
  62  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  63  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  64  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  65  */
  66 enum cvmx_usb_speed {
  67         CVMX_USB_SPEED_HIGH = 0,
  68         CVMX_USB_SPEED_FULL = 1,
  69         CVMX_USB_SPEED_LOW = 2,
  70 };
  71
  72 /**
  73  * enum cvmx_usb_transfer - the possible USB transfer types
  74  *
  75  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  76  *                                 transfers
  77  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  78  *                                 priority periodic transfers
  79  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  80  *                                 transfers
  81  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  82  *                                 periodic transfers
  83  */
  84 enum cvmx_usb_transfer {
  85         CVMX_USB_TRANSFER_CONTROL = 0,
  86         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
  87         CVMX_USB_TRANSFER_BULK = 2,
  88         CVMX_USB_TRANSFER_INTERRUPT = 3,
  89 };
  90
  91 /**
  92  * enum cvmx_usb_direction - the transfer directions
  93  *
  94  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
  95  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
  96  */
  97 enum cvmx_usb_direction {
  98         CVMX_USB_DIRECTION_OUT,
  99         CVMX_USB_DIRECTION_IN,
 100 };
 101
 102 /**
 103  * enum cvmx_usb_status - possible callback function status codes
 104  *
 105  * @CVMX_USB_STATUS_OK:           The transaction / operation finished without
 106  *                                any errors
 107  * @CVMX_USB_STATUS_SHORT:        FIXME: This is currently not implemented
 108  * @CVMX_USB_STATUS_CANCEL:       The transaction was canceled while in flight
 109  *                                by a user call to cvmx_usb_cancel
 110  * @CVMX_USB_STATUS_ERROR:        The transaction aborted with an unexpected
 111  *                                error status
 112  * @CVMX_USB_STATUS_STALL:        The transaction received a USB STALL response
 113  *                                from the device
 114  * @CVMX_USB_STATUS_XACTERR:      The transaction failed with an error from the
 115  *                                device even after a number of retries
 116  * @CVMX_USB_STATUS_DATATGLERR:   The transaction failed with a data toggle
 117  *                                error even after a number of retries
 118  * @CVMX_USB_STATUS_BABBLEERR:    The transaction failed with a babble error
 119  * @CVMX_USB_STATUS_FRAMEERR:     The transaction failed with a frame error
 120  *                                even after a number of retries
 121  */
 122 enum cvmx_usb_status {
 123         CVMX_USB_STATUS_OK,
 124         CVMX_USB_STATUS_SHORT,
 125         CVMX_USB_STATUS_CANCEL,
 126         CVMX_USB_STATUS_ERROR,
 127         CVMX_USB_STATUS_STALL,
 128         CVMX_USB_STATUS_XACTERR,
 129         CVMX_USB_STATUS_DATATGLERR,
 130         CVMX_USB_STATUS_BABBLEERR,
 131         CVMX_USB_STATUS_FRAMEERR,
 132 };
 133
 134 /**
 135  * struct cvmx_usb_port_status - the USB port status information
 136  *
 137  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 138  * @port_over_current:  1 = Over current detected, 0 = Over current not
 139  *                      detected. Octeon doesn't support over current detection.
 140  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 141  *                      power is off. Octeon doesn't support turning port power
 142  *                      off.
 143  * @port_speed:         Current port speed.
 144  * @connected:          1 = A device is connected to the port, 0 = No device is
 145  *                      connected.
 146  * @connect_change:     1 = Device connected state changed since the last set
 147  *                      status call.
 148  */
 149 struct cvmx_usb_port_status {
 150         u32 reserved                    : 25;
 151         u32 port_enabled                : 1;
 152         u32 port_over_current           : 1;
 153         u32 port_powered                : 1;
 154         enum cvmx_usb_speed port_speed  : 2;
 155         u32 connected                   : 1;
 156         u32 connect_change              : 1;
 157 };
 158
 159 /**
 160  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 161  *
 162  * @offset:     This is the offset in bytes into the main buffer where this data
 163  *              is stored.
 164  * @length:     This is the length in bytes of the data.
 165  * @status:     This is the status of this individual packet transfer.
 166  */
 167 struct cvmx_usb_iso_packet {
 168         int offset;
 169         int length;
 170         enum cvmx_usb_status status;
 171 };
 172
 173 /**
 174  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 175  *
 176  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 177  *                                            as clock source at USB_XO and
 178  *                                            USB_XI.
 179  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 180  *                                            board clock source at USB_XO.
 181  *                                            USB_XI should be tied to GND.
 182  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 183  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 184  *                                            crystal
 185  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 186  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 187  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 188  *                                            data transfer use for the USB
 189  */
 190 enum cvmx_usb_initialize_flags {
 191         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 192         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 193         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 194         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 195         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 196         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 197         /* Bits 3-4 used to encode the clock frequency */
 198         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 199 };
 200
 201 /**
 202  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 203  *
 204  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 205  *                                 actively using hardware.
 206  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
 207  *                                 pipe is in the ping state.
 208  */
 209 enum cvmx_usb_pipe_flags {
 210         CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
 211         CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
 212 };
 213
 214 /* Maximum number of times to retry failed transactions */
 215 #define MAX_RETRIES             3
 216
 217 /* Maximum number of hardware channels supported by the USB block */
 218 #define MAX_CHANNELS            8
 219
 220 /*
 221  * The low level hardware can transfer a maximum of this number of bytes in each
 222  * transfer. The field is 19 bits wide
 223  */
 224 #define MAX_TRANSFER_BYTES      ((1 << 19) - 1)
 225
 226 /*
 227  * The low level hardware can transfer a maximum of this number of packets in
 228  * each transfer. The field is 10 bits wide
 229  */
 230 #define MAX_TRANSFER_PACKETS    ((1 << 10) - 1)
 231
 232 /**
 233  * Logical transactions may take numerous low level
 234  * transactions, especially when splits are concerned. This
 235  * enum represents all of the possible stages a transaction can
 236  * be in. Note that split completes are always even. This is so
 237  * the NAK handler can backup to the previous low level
 238  * transaction with a simple clearing of bit 0.
 239  */
 240 enum cvmx_usb_stage {
 241         CVMX_USB_STAGE_NON_CONTROL,
 242         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 243         CVMX_USB_STAGE_SETUP,
 244         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 245         CVMX_USB_STAGE_DATA,
 246         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 247         CVMX_USB_STAGE_STATUS,
 248         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 249 };
 250
 251 /**
 252  * struct cvmx_usb_transaction - describes each pending USB transaction
 253  *                               regardless of type. These are linked together
 254  *                               to form a list of pending requests for a pipe.
 255  *
 256  * @node:               List node for transactions in the pipe.
 257  * @type:               Type of transaction, duplicated of the pipe.
 258  * @flags:              State flags for this transaction.
 259  * @buffer:             User's physical buffer address to read/write.
 260  * @buffer_length:      Size of the user's buffer in bytes.
 261  * @control_header:     For control transactions, physical address of the 8
 262  *                      byte standard header.
 263  * @iso_start_frame:    For ISO transactions, the starting frame number.
 264  * @iso_number_packets: For ISO transactions, the number of packets in the
 265  *                      request.
 266  * @iso_packets:        For ISO transactions, the sub packets in the request.
 267  * @actual_bytes:       Actual bytes transfer for this transaction.
 268  * @stage:              For control transactions, the current stage.
 269  * @urb:                URB.
 270  */
 271 struct cvmx_usb_transaction {
 272         struct list_head node;
 273         enum cvmx_usb_transfer type;
 274         u64 buffer;
 275         int buffer_length;
 276         u64 control_header;
 277         int iso_start_frame;
 278         int iso_number_packets;
 279         struct cvmx_usb_iso_packet *iso_packets;
 280         int xfersize;
 281         int pktcnt;
 282         int retries;
 283         int actual_bytes;
 284         enum cvmx_usb_stage stage;
 285         struct urb *urb;
 286 };
 287
 288 /**
 289  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 290  *                        and some USB device. It contains a list of pending
 291  *                        request to the device.
 292  *
 293  * @node:               List node for pipe list
 294  * @next:               Pipe after this one in the list
 295  * @transactions:       List of pending transactions
 296  * @interval:           For periodic pipes, the interval between packets in
 297  *                      frames
 298  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 299  * @flags:              State flags for this pipe
 300  * @device_speed:       Speed of device connected to this pipe
 301  * @transfer_type:      Type of transaction supported by this pipe
 302  * @transfer_dir:       IN or OUT. Ignored for Control
 303  * @multi_count:        Max packet in a row for the device
 304  * @max_packet:         The device's maximum packet size in bytes
 305  * @device_addr:        USB device address at other end of pipe
 306  * @endpoint_num:       USB endpoint number at other end of pipe
 307  * @hub_device_addr:    Hub address this device is connected to
 308  * @hub_port:           Hub port this device is connected to
 309  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 310  *                      the data pid needed
 311  * @channel:            Hardware DMA channel for this pipe
 312  * @split_sc_frame:     The low order bits of the frame number the split
 313  *                      complete should be sent on
 314  */
 315 struct cvmx_usb_pipe {
 316         struct list_head node;
 317         struct list_head transactions;
 318         u64 interval;
 319         u64 next_tx_frame;
 320         enum cvmx_usb_pipe_flags flags;
 321         enum cvmx_usb_speed device_speed;
 322         enum cvmx_usb_transfer transfer_type;
 323         enum cvmx_usb_direction transfer_dir;
 324         int multi_count;
 325         u16 max_packet;
 326         u8 device_addr;
 327         u8 endpoint_num;
 328         u8 hub_device_addr;
 329         u8 hub_port;
 330         u8 pid_toggle;
 331         u8 channel;
 332         s8 split_sc_frame;
 333 };
 334
 335 struct cvmx_usb_tx_fifo {
 336         struct {
 337                 int channel;
 338                 int size;
 339                 u64 address;
 340         } entry[MAX_CHANNELS + 1];
 341         int head;
 342         int tail;
 343 };
 344
 345 /**
 346  * struct octeon_hcd - the state of the USB block
 347  *
 348  * lock:                   Serialization lock.
 349  * init_flags:             Flags passed to initialize.
 350  * index:                  Which USB block this is for.
 351  * idle_hardware_channels: Bit set for every idle hardware channel.
 352  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 353  *                         determine splits.
 354  * pipe_for_channel:       Map channels to pipes.
 355  * pipe:                   Storage for pipes.
 356  * indent:                 Used by debug output to indent functions.
 357  * port_status:            Last port status used for change notification.
 358  * idle_pipes:             List of open pipes that have no transactions.
 359  * active_pipes:           Active pipes indexed by transfer type.
 360  * frame_number:           Increments every SOF interrupt for time keeping.
 361  * active_split:           Points to the current active split, or NULL.
 362  */
 363 struct octeon_hcd {
 364         spinlock_t lock; /* serialization lock */
 365         int init_flags;
 366         int index;
 367         int idle_hardware_channels;
 368         union cvmx_usbcx_hprt usbcx_hprt;
 369         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 370         int indent;
 371         struct cvmx_usb_port_status port_status;
 372         struct list_head idle_pipes;
 373         struct list_head active_pipes[4];
 374         u64 frame_number;
 375         struct cvmx_usb_transaction *active_split;
 376         struct cvmx_usb_tx_fifo periodic;
 377         struct cvmx_usb_tx_fifo nonperiodic;
 378 };
 379
 380 /* This macro spins on a register waiting for it to reach a condition. */
 381 #define CVMX_WAIT_FOR_FIELD32(address, _union, cond, timeout_usec)          \
 382         ({int result;                                                       \
 383         do {                                                                \
 384                 u64 done = cvmx_get_cycle() + (u64)timeout_usec *           \
 385                            octeon_get_clock_rate() / 1000000;               \
 386                 union _union c;                                             \
 387                                                                             \
 388                 while (1) {                                                 \
 389                         c.u32 = cvmx_usb_read_csr32(usb, address);          \
 390                                                                             \
 391                         if (cond) {                                         \
 392                                 result = 0;                                 \
 393                                 break;                                      \
 394                         } else if (cvmx_get_cycle() > done) {               \
 395                                 result = -1;                                \
 396                                 break;                                      \
 397                         } else                                              \
 398                                 __delay(100);                               \
 399                 }                                                           \
 400         } while (0);                                                        \
 401         result; })
 402
 403 /*
 404  * This macro logically sets a single field in a CSR. It does the sequence
 405  * read, modify, and write
 406  */
 407 #define USB_SET_FIELD32(address, _union, field, value)          \
 408         do {                                                    \
 409                 union _union c;                                 \
 410                                                                 \
 411                 c.u32 = cvmx_usb_read_csr32(usb, address);      \
 412                 c.s.field = value;                              \
 413                 cvmx_usb_write_csr32(usb, address, c.u32);      \
 414         } while (0)
 415
 416 /* Returns the IO address to push/pop stuff data from the FIFOs */
 417 #define USB_FIFO_ADDRESS(channel, usb_index) \
 418         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel) + 1) * 0x1000)
 419
 420 /**
 421  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 422  * @orig_buffer: the original buffer passed by the USB stack
 423  * @data:        the newly allocated temporary buffer (excluding meta-data)
 424  *
 425  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 426  * the buffer is too short, we need to allocate a temporary one, and this struct
 427  * represents it.
 428  */
 429 struct octeon_temp_buffer {
 430         void *orig_buffer;
 431         u8 data[0];
 432 };
 433
 434 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
 435 {
 436         return container_of((void *)p, struct usb_hcd, hcd_priv);
 437 }
 438
 439 /**
 440  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 441  *                            (if needed)
 442  * @urb:        URB.
 443  * @mem_flags:  Memory allocation flags.
 444  *
 445  * This function allocates a temporary bounce buffer whenever it's needed
 446  * due to HW limitations.
 447  */
 448 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 449 {
 450         struct octeon_temp_buffer *temp;
 451
 452         if (urb->num_sgs || urb->sg ||
 453             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 454             !(urb->transfer_buffer_length % sizeof(u32)))
 455                 return 0;
 456
 457         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 458                        sizeof(*temp), mem_flags);
 459         if (!temp)
 460                 return -ENOMEM;
 461
 462         temp->orig_buffer = urb->transfer_buffer;
 463         if (usb_urb_dir_out(urb))
 464                 memcpy(temp->data, urb->transfer_buffer,
 465                        urb->transfer_buffer_length);
 466         urb->transfer_buffer = temp->data;
 467         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 468
 469         return 0;
 470 }
 471
 472 /**
 473  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 474  * @urb: URB.
 475  *
 476  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 477  */
 478 static void octeon_free_temp_buffer(struct urb *urb)
 479 {
 480         struct octeon_temp_buffer *temp;
 481         size_t length;
 482
 483         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 484                 return;
 485
 486         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 487                             data);
 488         if (usb_urb_dir_in(urb)) {
 489                 if (usb_pipeisoc(urb->pipe))
 490                         length = urb->transfer_buffer_length;
 491                 else
 492                         length = urb->actual_length;
 493
 494                 memcpy(temp->orig_buffer, urb->transfer_buffer, length);
 495         }
 496         urb->transfer_buffer = temp->orig_buffer;
 497         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 498         kfree(temp);
 499 }
 500
 501 /**
 502  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 503  * @hcd:        USB HCD structure.
 504  * @urb:        URB.
 505  * @mem_flags:  Memory allocation flags.
 506  */
 507 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 508                                   gfp_t mem_flags)
 509 {
 510         int ret;
 511
 512         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 513         if (ret)
 514                 return ret;
 515
 516         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 517         if (ret)
 518                 octeon_free_temp_buffer(urb);
 519
 520         return ret;
 521 }
 522
 523 /**
 524  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 525  * @hcd:        USB HCD structure.
 526  * @urb:        URB.
 527  */
 528 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 529 {
 530         usb_hcd_unmap_urb_for_dma(hcd, urb);
 531         octeon_free_temp_buffer(urb);
 532 }
 533
 534 /**
 535  * Read a USB 32bit CSR. It performs the necessary address swizzle
 536  * for 32bit CSRs and logs the value in a readable format if
 537  * debugging is on.
 538  *
 539  * @usb:     USB block this access is for
 540  * @address: 64bit address to read
 541  *
 542  * Returns: Result of the read
 543  */
 544 static inline u32 cvmx_usb_read_csr32(struct octeon_hcd *usb, u64 address)
 545 {
 546         u32 result = cvmx_read64_uint32(address ^ 4);
 547         return result;
 548 }
 549
 550 /**
 551  * Write a USB 32bit CSR. It performs the necessary address
 552  * swizzle for 32bit CSRs and logs the value in a readable format
 553  * if debugging is on.
 554  *
 555  * @usb:     USB block this access is for
 556  * @address: 64bit address to write
 557  * @value:   Value to write
 558  */
 559 static inline void cvmx_usb_write_csr32(struct octeon_hcd *usb,
 560                                         u64 address, u32 value)
 561 {
 562         cvmx_write64_uint32(address ^ 4, value);
 563         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 564 }
 565
 566 /**
 567  * Return non zero if this pipe connects to a non HIGH speed
 568  * device through a high speed hub.
 569  *
 570  * @usb:    USB block this access is for
 571  * @pipe:   Pipe to check
 572  *
 573  * Returns: Non zero if we need to do split transactions
 574  */
 575 static inline int cvmx_usb_pipe_needs_split(struct octeon_hcd *usb,
 576                                             struct cvmx_usb_pipe *pipe)
 577 {
 578         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 579                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 580 }
 581
 582 /**
 583  * Trivial utility function to return the correct PID for a pipe
 584  *
 585  * @pipe:   pipe to check
 586  *
 587  * Returns: PID for pipe
 588  */
 589 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 590 {
 591         if (pipe->pid_toggle)
 592                 return 2; /* Data1 */
 593         return 0; /* Data0 */
 594 }
 595
 596 static void cvmx_fifo_setup(struct octeon_hcd *usb)
 597 {
 598         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
 599         union cvmx_usbcx_gnptxfsiz npsiz;
 600         union cvmx_usbcx_hptxfsiz psiz;
 601
 602         usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
 603                                                 CVMX_USBCX_GHWCFG3(usb->index));
 604
 605         /*
 606          * Program the USBC_GRXFSIZ register to select the size of the receive
 607          * FIFO (25%).
 608          */
 609         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
 610                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
 611
 612         /*
 613          * Program the USBC_GNPTXFSIZ register to select the size and the start
 614          * address of the non-periodic transmit FIFO for nonperiodic
 615          * transactions (50%).
 616          */
 617         npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
 618         npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
 619         npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
 620         cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
 621
 622         /*
 623          * Program the USBC_HPTXFSIZ register to select the size and start
 624          * address of the periodic transmit FIFO for periodic transactions
 625          * (25%).
 626          */
 627         psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
 628         psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
 629         psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
 630         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
 631
 632         /* Flush all FIFOs */
 633         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 634                         cvmx_usbcx_grstctl, txfnum, 0x10);
 635         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 636                         cvmx_usbcx_grstctl, txfflsh, 1);
 637         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 638                               cvmx_usbcx_grstctl, c.s.txfflsh == 0, 100);
 639         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 640                         cvmx_usbcx_grstctl, rxfflsh, 1);
 641         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
 642                               cvmx_usbcx_grstctl, c.s.rxfflsh == 0, 100);
 643 }
 644
 645 /**
 646  * Shutdown a USB port after a call to cvmx_usb_initialize().
 647  * The port should be disabled with all pipes closed when this
 648  * function is called.
 649  *
 650  * @usb: USB device state populated by cvmx_usb_initialize().
 651  *
 652  * Returns: 0 or a negative error code.
 653  */
 654 static int cvmx_usb_shutdown(struct octeon_hcd *usb)
 655 {
 656         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 657
 658         /* Make sure all pipes are closed */
 659         if (!list_empty(&usb->idle_pipes) ||
 660             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 661             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 662             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 663             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 664                 return -EBUSY;
 665
 666         /* Disable the clocks and put them in power on reset */
 667         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 668         usbn_clk_ctl.s.enable = 1;
 669         usbn_clk_ctl.s.por = 1;
 670         usbn_clk_ctl.s.hclk_rst = 1;
 671         usbn_clk_ctl.s.prst = 0;
 672         usbn_clk_ctl.s.hrst = 0;
 673         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 674         return 0;
 675 }
 676
 677 /**
 678  * Initialize a USB port for use. This must be called before any
 679  * other access to the Octeon USB port is made. The port starts
 680  * off in the disabled state.
 681  *
 682  * @dev:         Pointer to struct device for logging purposes.
 683  * @usb:         Pointer to struct octeon_hcd.
 684  *
 685  * Returns: 0 or a negative error code.
 686  */
 687 static int cvmx_usb_initialize(struct device *dev,
 688                                struct octeon_hcd *usb)
 689 {
 690         int channel;
 691         int divisor;
 692         int retries = 0;
 693         union cvmx_usbcx_hcfg usbcx_hcfg;
 694         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 695         union cvmx_usbcx_gintsts usbc_gintsts;
 696         union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 697         union cvmx_usbcx_gintmsk usbcx_gintmsk;
 698         union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 699         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 700
 701 retry:
 702         /*
 703          * Power On Reset and PHY Initialization
 704          *
 705          * 1. Wait for DCOK to assert (nothing to do)
 706          *
 707          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 708          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 709          */
 710         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
 711         usbn_clk_ctl.s.por = 1;
 712         usbn_clk_ctl.s.hrst = 0;
 713         usbn_clk_ctl.s.prst = 0;
 714         usbn_clk_ctl.s.hclk_rst = 0;
 715         usbn_clk_ctl.s.enable = 0;
 716         /*
 717          * 2b. Select the USB reference clock/crystal parameters by writing
 718          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 719          */
 720         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 721                 /*
 722                  * The USB port uses 12/24/48MHz 2.5V board clock
 723                  * source at USB_XO. USB_XI should be tied to GND.
 724                  * Most Octeon evaluation boards require this setting
 725                  */
 726                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 727                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 728                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 729                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 730                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 731                 else
 732                         /* From CN52XX manual */
 733                         usbn_clk_ctl.s.p_rtype = 1;
 734
 735                 switch (usb->init_flags &
 736                         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 737                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 738                         usbn_clk_ctl.s.p_c_sel = 0;
 739                         break;
 740                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 741                         usbn_clk_ctl.s.p_c_sel = 1;
 742                         break;
 743                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 744                         usbn_clk_ctl.s.p_c_sel = 2;
 745                         break;
 746                 }
 747         } else {
 748                 /*
 749                  * The USB port uses a 12MHz crystal as clock source
 750                  * at USB_XO and USB_XI
 751                  */
 752                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 753                         /* From CN31XX,CN30XX manual */
 754                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 755                 else
 756                         /* From CN56XX,CN52XX,CN50XX manuals. */
 757                         usbn_clk_ctl.s.p_rtype = 0;
 758
 759                 usbn_clk_ctl.s.p_c_sel = 0;
 760         }
 761         /*
 762          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 763          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 764          *     such that USB is as close as possible to 125Mhz
 765          */
 766         divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
 767         /* Lower than 4 doesn't seem to work properly */
 768         if (divisor < 4)
 769                 divisor = 4;
 770         usbn_clk_ctl.s.divide = divisor;
 771         usbn_clk_ctl.s.divide2 = 0;
 772         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 773
 774         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 775         usbn_clk_ctl.s.hclk_rst = 1;
 776         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 777         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 778         __delay(64);
 779         /*
 780          * 3. Program the power-on reset field in the USBN clock-control
 781          *    register:
 782          *    USBN_CLK_CTL[POR] = 0
 783          */
 784         usbn_clk_ctl.s.por = 0;
 785         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 786         /* 4. Wait 1 ms for PHY clock to start */
 787         mdelay(1);
 788         /*
 789          * 5. Program the Reset input from automatic test equipment field in the
 790          *    USBP control and status register:
 791          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 792          */
 793         usbn_usbp_ctl_status.u64 =
 794                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 795         usbn_usbp_ctl_status.s.ate_reset = 1;
 796         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 797                             usbn_usbp_ctl_status.u64);
 798         /* 6. Wait 10 cycles */
 799         __delay(10);
 800         /*
 801          * 7. Clear ATE_RESET field in the USBN clock-control register:
 802          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 803          */
 804         usbn_usbp_ctl_status.s.ate_reset = 0;
 805         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 806                             usbn_usbp_ctl_status.u64);
 807         /*
 808          * 8. Program the PHY reset field in the USBN clock-control register:
 809          *    USBN_CLK_CTL[PRST] = 1
 810          */
 811         usbn_clk_ctl.s.prst = 1;
 812         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 813         /*
 814          * 9. Program the USBP control and status register to select host or
 815          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 816          *    device
 817          */
 818         usbn_usbp_ctl_status.s.hst_mode = 0;
 819         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 820                             usbn_usbp_ctl_status.u64);
 821         /* 10. Wait 1 us */
 822         udelay(1);
 823         /*
 824          * 11. Program the hreset_n field in the USBN clock-control register:
 825          *     USBN_CLK_CTL[HRST] = 1
 826          */
 827         usbn_clk_ctl.s.hrst = 1;
 828         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 829         /* 12. Proceed to USB core initialization */
 830         usbn_clk_ctl.s.enable = 1;
 831         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
 832         udelay(1);
 833
 834         /*
 835          * USB Core Initialization
 836          *
 837          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 838          *    determine USB core configuration parameters.
 839          *
 840          *    Nothing needed
 841          *
 842          * 2. Program the following fields in the global AHB configuration
 843          *    register (USBC_GAHBCFG)
 844          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 845          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 846          *    Nonperiodic TxFIFO empty level (slave mode only),
 847          *    USBC_GAHBCFG[NPTXFEMPLVL]
 848          *    Periodic TxFIFO empty level (slave mode only),
 849          *    USBC_GAHBCFG[PTXFEMPLVL]
 850          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 851          */
 852         usbcx_gahbcfg.u32 = 0;
 853         usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 854                                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 855         usbcx_gahbcfg.s.hbstlen = 0;
 856         usbcx_gahbcfg.s.nptxfemplvl = 1;
 857         usbcx_gahbcfg.s.ptxfemplvl = 1;
 858         usbcx_gahbcfg.s.glblintrmsk = 1;
 859         cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 860                              usbcx_gahbcfg.u32);
 861
 862         /*
 863          * 3. Program the following fields in USBC_GUSBCFG register.
 864          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 865          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 866          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 867          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 868          */
 869         usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
 870                                                 CVMX_USBCX_GUSBCFG(usb->index));
 871         usbcx_gusbcfg.s.toutcal = 0;
 872         usbcx_gusbcfg.s.ddrsel = 0;
 873         usbcx_gusbcfg.s.usbtrdtim = 0x5;
 874         usbcx_gusbcfg.s.phylpwrclksel = 0;
 875         cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 876                              usbcx_gusbcfg.u32);
 877
 878         /*
 879          * 4. The software must unmask the following bits in the USBC_GINTMSK
 880          *    register.
 881          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 882          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 883          */
 884         usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
 885                                                 CVMX_USBCX_GINTMSK(usb->index));
 886         usbcx_gintmsk.s.otgintmsk = 1;
 887         usbcx_gintmsk.s.modemismsk = 1;
 888         usbcx_gintmsk.s.hchintmsk = 1;
 889         usbcx_gintmsk.s.sofmsk = 0;
 890         /* We need RX FIFO interrupts if we don't have DMA */
 891         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 892                 usbcx_gintmsk.s.rxflvlmsk = 1;
 893         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 894                              usbcx_gintmsk.u32);
 895
 896         /*
 897          * Disable all channel interrupts. We'll enable them per channel later.
 898          */
 899         for (channel = 0; channel < 8; channel++)
 900                 cvmx_usb_write_csr32(usb,
 901                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
 902                                      0);
 903
 904         /*
 905          * Host Port Initialization
 906          *
 907          * 1. Program the host-port interrupt-mask field to unmask,
 908          *    USBC_GINTMSK[PRTINT] = 1
 909          */
 910         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 911                         cvmx_usbcx_gintmsk, prtintmsk, 1);
 912         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 913                         cvmx_usbcx_gintmsk, disconnintmsk, 1);
 914
 915         /*
 916          * 2. Program the USBC_HCFG register to select full-speed host
 917          *    or high-speed host.
 918          */
 919         usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
 920         usbcx_hcfg.s.fslssupp = 0;
 921         usbcx_hcfg.s.fslspclksel = 0;
 922         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
 923
 924         cvmx_fifo_setup(usb);
 925
 926         /*
 927          * If the controller is getting port events right after the reset, it
 928          * means the initialization failed. Try resetting the controller again
 929          * in such case. This is seen to happen after cold boot on DSR-1000N.
 930          */
 931         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
 932                                                CVMX_USBCX_GINTSTS(usb->index));
 933         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
 934                              usbc_gintsts.u32);
 935         dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
 936         if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
 937                 return 0;
 938         if (retries++ >= 5)
 939                 return -EAGAIN;
 940         dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
 941                  (int)usbc_gintsts.u32);
 942         msleep(50);
 943         cvmx_usb_shutdown(usb);
 944         msleep(50);
 945         goto retry;
 946 }
 947
 948 /**
 949  * Reset a USB port. After this call succeeds, the USB port is
 950  * online and servicing requests.
 951  *
 952  * @usb: USB device state populated by cvmx_usb_initialize().
 953  */
 954 static void cvmx_usb_reset_port(struct octeon_hcd *usb)
 955 {
 956         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 957                                                   CVMX_USBCX_HPRT(usb->index));
 958
 959         /* Program the port reset bit to start the reset process */
 960         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 961                         prtrst, 1);
 962
 963         /*
 964          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
 965          * process to complete.
 966          */
 967         mdelay(50);
 968
 969         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
 970         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 971                         prtrst, 0);
 972
 973         /*
 974          * Read the port speed field to get the enumerated speed,
 975          * USBC_HPRT[PRTSPD].
 976          */
 977         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
 978                                                   CVMX_USBCX_HPRT(usb->index));
 979 }
 980
 981 /**
 982  * Disable a USB port. After this call the USB port will not
 983  * generate data transfers and will not generate events.
 984  * Transactions in process will fail and call their
 985  * associated callbacks.
 986  *
 987  * @usb: USB device state populated by cvmx_usb_initialize().
 988  *
 989  * Returns: 0 or a negative error code.
 990  */
 991 static int cvmx_usb_disable(struct octeon_hcd *usb)
 992 {
 993         /* Disable the port */
 994         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
 995                         prtena, 1);
 996         return 0;
 997 }
 998
 999 /**
1000  * Get the current state of the USB port. Use this call to
1001  * determine if the usb port has anything connected, is enabled,
1002  * or has some sort of error condition. The return value of this
1003  * call has "changed" bits to signal of the value of some fields
1004  * have changed between calls.
1005  *
1006  * @usb: USB device state populated by cvmx_usb_initialize().
1007  *
1008  * Returns: Port status information
1009  */
1010 static struct cvmx_usb_port_status cvmx_usb_get_status(struct octeon_hcd *usb)
1011 {
1012         union cvmx_usbcx_hprt usbc_hprt;
1013         struct cvmx_usb_port_status result;
1014
1015         memset(&result, 0, sizeof(result));
1016
1017         usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1018         result.port_enabled = usbc_hprt.s.prtena;
1019         result.port_over_current = usbc_hprt.s.prtovrcurract;
1020         result.port_powered = usbc_hprt.s.prtpwr;
1021         result.port_speed = usbc_hprt.s.prtspd;
1022         result.connected = usbc_hprt.s.prtconnsts;
1023         result.connect_change =
1024                 result.connected != usb->port_status.connected;
1025
1026         return result;
1027 }
1028
1029 /**
1030  * Open a virtual pipe between the host and a USB device. A pipe
1031  * must be opened before data can be transferred between a device
1032  * and Octeon.
1033  *
1034  * @usb:             USB device state populated by cvmx_usb_initialize().
1035  * @device_addr:
1036  *                   USB device address to open the pipe to
1037  *                   (0-127).
1038  * @endpoint_num:
1039  *                   USB endpoint number to open the pipe to
1040  *                   (0-15).
1041  * @device_speed:
1042  *                   The speed of the device the pipe is going
1043  *                   to. This must match the device's speed,
1044  *                   which may be different than the port speed.
1045  * @max_packet:      The maximum packet length the device can
1046  *                   transmit/receive (low speed=0-8, full
1047  *                   speed=0-1023, high speed=0-1024). This value
1048  *                   comes from the standard endpoint descriptor
1049  *                   field wMaxPacketSize bits <10:0>.
1050  * @transfer_type:
1051  *                   The type of transfer this pipe is for.
1052  * @transfer_dir:
1053  *                   The direction the pipe is in. This is not
1054  *                   used for control pipes.
1055  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1056  *                   this is how often the transfer is scheduled
1057  *                   for. All other transfers should specify
1058  *                   zero. The units are in frames (8000/sec at
1059  *                   high speed, 1000/sec for full speed).
1060  * @multi_count:
1061  *                   For high speed devices, this is the maximum
1062  *                   allowed number of packet per microframe.
1063  *                   Specify zero for non high speed devices. This
1064  *                   value comes from the standard endpoint descriptor
1065  *                   field wMaxPacketSize bits <12:11>.
1066  * @hub_device_addr:
1067  *                   Hub device address this device is connected
1068  *                   to. Devices connected directly to Octeon
1069  *                   use zero. This is only used when the device
1070  *                   is full/low speed behind a high speed hub.
1071  *                   The address will be of the high speed hub,
1072  *                   not and full speed hubs after it.
1073  * @hub_port:        Which port on the hub the device is
1074  *                   connected. Use zero for devices connected
1075  *                   directly to Octeon. Like hub_device_addr,
1076  *                   this is only used for full/low speed
1077  *                   devices behind a high speed hub.
1078  *
1079  * Returns: A non-NULL value is a pipe. NULL means an error.
1080  */
1081 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct octeon_hcd *usb,
1082                                                 int device_addr,
1083                                                 int endpoint_num,
1084                                                 enum cvmx_usb_speed
1085                                                         device_speed,
1086                                                 int max_packet,
1087                                                 enum cvmx_usb_transfer
1088                                                         transfer_type,
1089                                                 enum cvmx_usb_direction
1090                                                         transfer_dir,
1091                                                 int interval, int multi_count,
1092                                                 int hub_device_addr,
1093                                                 int hub_port)
1094 {
1095         struct cvmx_usb_pipe *pipe;
1096
1097         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1098         if (!pipe)
1099                 return NULL;
1100         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1101             (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1102             (transfer_type == CVMX_USB_TRANSFER_BULK))
1103                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
1104         pipe->device_addr = device_addr;
1105         pipe->endpoint_num = endpoint_num;
1106         pipe->device_speed = device_speed;
1107         pipe->max_packet = max_packet;
1108         pipe->transfer_type = transfer_type;
1109         pipe->transfer_dir = transfer_dir;
1110         INIT_LIST_HEAD(&pipe->transactions);
1111
1112         /*
1113          * All pipes use interval to rate limit NAK processing. Force an
1114          * interval if one wasn't supplied
1115          */
1116         if (!interval)
1117                 interval = 1;
1118         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1119                 pipe->interval = interval * 8;
1120                 /* Force start splits to be schedule on uFrame 0 */
1121                 pipe->next_tx_frame = ((usb->frame_number + 7) & ~7) +
1122                                         pipe->interval;
1123         } else {
1124                 pipe->interval = interval;
1125                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1126         }
1127         pipe->multi_count = multi_count;
1128         pipe->hub_device_addr = hub_device_addr;
1129         pipe->hub_port = hub_port;
1130         pipe->pid_toggle = 0;
1131         pipe->split_sc_frame = -1;
1132         list_add_tail(&pipe->node, &usb->idle_pipes);
1133
1134         /*
1135          * We don't need to tell the hardware about this pipe yet since
1136          * it doesn't have any submitted requests
1137          */
1138
1139         return pipe;
1140 }
1141
1142 /**
1143  * Poll the RX FIFOs and remove data as needed. This function is only used
1144  * in non DMA mode. It is very important that this function be called quickly
1145  * enough to prevent FIFO overflow.
1146  *
1147  * @usb:        USB device state populated by cvmx_usb_initialize().
1148  */
1149 static void cvmx_usb_poll_rx_fifo(struct octeon_hcd *usb)
1150 {
1151         union cvmx_usbcx_grxstsph rx_status;
1152         int channel;
1153         int bytes;
1154         u64 address;
1155         u32 *ptr;
1156
1157         rx_status.u32 = cvmx_usb_read_csr32(usb,
1158                                             CVMX_USBCX_GRXSTSPH(usb->index));
1159         /* Only read data if IN data is there */
1160         if (rx_status.s.pktsts != 2)
1161                 return;
1162         /* Check if no data is available */
1163         if (!rx_status.s.bcnt)
1164                 return;
1165
1166         channel = rx_status.s.chnum;
1167         bytes = rx_status.s.bcnt;
1168         if (!bytes)
1169                 return;
1170
1171         /* Get where the DMA engine would have written this data */
1172         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1173                                      channel * 8);
1174
1175         ptr = cvmx_phys_to_ptr(address);
1176         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1177                             address + bytes);
1178
1179         /* Loop writing the FIFO data for this packet into memory */
1180         while (bytes > 0) {
1181                 *ptr++ = cvmx_usb_read_csr32(usb,
1182                                         USB_FIFO_ADDRESS(channel, usb->index));
1183                 bytes -= 4;
1184         }
1185         CVMX_SYNCW;
1186 }
1187
1188 /**
1189  * Fill the TX hardware fifo with data out of the software
1190  * fifos
1191  *
1192  * @usb:            USB device state populated by cvmx_usb_initialize().
1193  * @fifo:           Software fifo to use
1194  * @available:      Amount of space in the hardware fifo
1195  *
1196  * Returns: Non zero if the hardware fifo was too small and needs
1197  *          to be serviced again.
1198  */
1199 static int cvmx_usb_fill_tx_hw(struct octeon_hcd *usb,
1200                                struct cvmx_usb_tx_fifo *fifo, int available)
1201 {
1202         /*
1203          * We're done either when there isn't anymore space or the software FIFO
1204          * is empty
1205          */
1206         while (available && (fifo->head != fifo->tail)) {
1207                 int i = fifo->tail;
1208                 const u32 *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1209                 u64 csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1210                                                    usb->index) ^ 4;
1211                 int words = available;
1212
1213                 /* Limit the amount of data to what the SW fifo has */
1214                 if (fifo->entry[i].size <= available) {
1215                         words = fifo->entry[i].size;
1216                         fifo->tail++;
1217                         if (fifo->tail > MAX_CHANNELS)
1218                                 fifo->tail = 0;
1219                 }
1220
1221                 /* Update the next locations and counts */
1222                 available -= words;
1223                 fifo->entry[i].address += words * 4;
1224                 fifo->entry[i].size -= words;
1225
1226                 /*
1227                  * Write the HW fifo data. The read every three writes is due
1228                  * to an errata on CN3XXX chips
1229                  */
1230                 while (words > 3) {
1231                         cvmx_write64_uint32(csr_address, *ptr++);
1232                         cvmx_write64_uint32(csr_address, *ptr++);
1233                         cvmx_write64_uint32(csr_address, *ptr++);
1234                         cvmx_read64_uint64(
1235                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1236                         words -= 3;
1237                 }
1238                 cvmx_write64_uint32(csr_address, *ptr++);
1239                 if (--words) {
1240                         cvmx_write64_uint32(csr_address, *ptr++);
1241                         if (--words)
1242                                 cvmx_write64_uint32(csr_address, *ptr++);
1243                 }
1244                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1245         }
1246         return fifo->head != fifo->tail;
1247 }
1248
1249 /**
1250  * Check the hardware FIFOs and fill them as needed
1251  *
1252  * @usb:        USB device state populated by cvmx_usb_initialize().
1253  */
1254 static void cvmx_usb_poll_tx_fifo(struct octeon_hcd *usb)
1255 {
1256         if (usb->periodic.head != usb->periodic.tail) {
1257                 union cvmx_usbcx_hptxsts tx_status;
1258
1259                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1260                                         CVMX_USBCX_HPTXSTS(usb->index));
1261                 if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1262                                         tx_status.s.ptxfspcavail))
1263                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1264                                         cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1265                 else
1266                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1267                                         cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1268         }
1269
1270         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1271                 union cvmx_usbcx_gnptxsts tx_status;
1272
1273                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1274                                         CVMX_USBCX_GNPTXSTS(usb->index));
1275                 if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1276                                         tx_status.s.nptxfspcavail))
1277                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1278                                         cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1279                 else
1280                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1281                                         cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1282         }
1283 }
1284
1285 /**
1286  * Fill the TX FIFO with an outgoing packet
1287  *
1288  * @usb:          USB device state populated by cvmx_usb_initialize().
1289  * @channel:      Channel number to get packet from
1290  */
1291 static void cvmx_usb_fill_tx_fifo(struct octeon_hcd *usb, int channel)
1292 {
1293         union cvmx_usbcx_hccharx hcchar;
1294         union cvmx_usbcx_hcspltx usbc_hcsplt;
1295         union cvmx_usbcx_hctsizx usbc_hctsiz;
1296         struct cvmx_usb_tx_fifo *fifo;
1297
1298         /* We only need to fill data on outbound channels */
1299         hcchar.u32 = cvmx_usb_read_csr32(usb,
1300                         CVMX_USBCX_HCCHARX(channel, usb->index));
1301         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1302                 return;
1303
1304         /* OUT Splits only have data on the start and not the complete */
1305         usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
1306                                 CVMX_USBCX_HCSPLTX(channel, usb->index));
1307         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1308                 return;
1309
1310         /*
1311          * Find out how many bytes we need to fill and convert it into 32bit
1312          * words.
1313          */
1314         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1315                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1316         if (!usbc_hctsiz.s.xfersize)
1317                 return;
1318
1319         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1320             (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1321                 fifo = &usb->periodic;
1322         else
1323                 fifo = &usb->nonperiodic;
1324
1325         fifo->entry[fifo->head].channel = channel;
1326         fifo->entry[fifo->head].address =
1327                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1328                                    channel * 8);
1329         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize + 3) >> 2;
1330         fifo->head++;
1331         if (fifo->head > MAX_CHANNELS)
1332                 fifo->head = 0;
1333
1334         cvmx_usb_poll_tx_fifo(usb);
1335 }
1336
1337 /**
1338  * Perform channel specific setup for Control transactions. All
1339  * the generic stuff will already have been done in cvmx_usb_start_channel().
1340  *
1341  * @usb:          USB device state populated by cvmx_usb_initialize().
1342  * @channel:      Channel to setup
1343  * @pipe:         Pipe for control transaction
1344  */
1345 static void cvmx_usb_start_channel_control(struct octeon_hcd *usb,
1346                                            int channel,
1347                                            struct cvmx_usb_pipe *pipe)
1348 {
1349         struct usb_hcd *hcd = octeon_to_hcd(usb);
1350         struct device *dev = hcd->self.controller;
1351         struct cvmx_usb_transaction *transaction =
1352                 list_first_entry(&pipe->transactions, typeof(*transaction),
1353                                  node);
1354         struct usb_ctrlrequest *header =
1355                 cvmx_phys_to_ptr(transaction->control_header);
1356         int bytes_to_transfer = transaction->buffer_length -
1357                 transaction->actual_bytes;
1358         int packets_to_transfer;
1359         union cvmx_usbcx_hctsizx usbc_hctsiz;
1360
1361         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1362                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1363
1364         switch (transaction->stage) {
1365         case CVMX_USB_STAGE_NON_CONTROL:
1366         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1367                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1368                 break;
1369         case CVMX_USB_STAGE_SETUP:
1370                 usbc_hctsiz.s.pid = 3; /* Setup */
1371                 bytes_to_transfer = sizeof(*header);
1372                 /* All Control operations start with a setup going OUT */
1373                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1374                                 cvmx_usbcx_hccharx, epdir,
1375                                 CVMX_USB_DIRECTION_OUT);
1376                 /*
1377                  * Setup send the control header instead of the buffer data. The
1378                  * buffer data will be used in the next stage
1379                  */
1380                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1381                                         channel * 8,
1382                                     transaction->control_header);
1383                 break;
1384         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1385                 usbc_hctsiz.s.pid = 3; /* Setup */
1386                 bytes_to_transfer = 0;
1387                 /* All Control operations start with a setup going OUT */
1388                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1389                                 cvmx_usbcx_hccharx, epdir,
1390                                 CVMX_USB_DIRECTION_OUT);
1391
1392                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1393                                 cvmx_usbcx_hcspltx, compsplt, 1);
1394                 break;
1395         case CVMX_USB_STAGE_DATA:
1396                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1397                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1398                         if (header->bRequestType & USB_DIR_IN)
1399                                 bytes_to_transfer = 0;
1400                         else if (bytes_to_transfer > pipe->max_packet)
1401                                 bytes_to_transfer = pipe->max_packet;
1402                 }
1403                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1404                                 cvmx_usbcx_hccharx, epdir,
1405                                 ((header->bRequestType & USB_DIR_IN) ?
1406                                         CVMX_USB_DIRECTION_IN :
1407                                         CVMX_USB_DIRECTION_OUT));
1408                 break;
1409         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1410                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1411                 if (!(header->bRequestType & USB_DIR_IN))
1412                         bytes_to_transfer = 0;
1413                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1414                                 cvmx_usbcx_hccharx, epdir,
1415                                 ((header->bRequestType & USB_DIR_IN) ?
1416                                         CVMX_USB_DIRECTION_IN :
1417                                         CVMX_USB_DIRECTION_OUT));
1418                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1419                                 cvmx_usbcx_hcspltx, compsplt, 1);
1420                 break;
1421         case CVMX_USB_STAGE_STATUS:
1422                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1423                 bytes_to_transfer = 0;
1424                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1425                                 cvmx_usbcx_hccharx, epdir,
1426                                 ((header->bRequestType & USB_DIR_IN) ?
1427                                         CVMX_USB_DIRECTION_OUT :
1428                                         CVMX_USB_DIRECTION_IN));
1429                 break;
1430         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1431                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1432                 bytes_to_transfer = 0;
1433                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1434                                 cvmx_usbcx_hccharx, epdir,
1435                                 ((header->bRequestType & USB_DIR_IN) ?
1436                                         CVMX_USB_DIRECTION_OUT :
1437                                         CVMX_USB_DIRECTION_IN));
1438                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1439                                 cvmx_usbcx_hcspltx, compsplt, 1);
1440                 break;
1441         }
1442
1443         /*
1444          * Make sure the transfer never exceeds the byte limit of the hardware.
1445          * Further bytes will be sent as continued transactions
1446          */
1447         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1448                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1449                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1450                 bytes_to_transfer *= pipe->max_packet;
1451         }
1452
1453         /*
1454          * Calculate the number of packets to transfer. If the length is zero
1455          * we still need to transfer one packet
1456          */
1457         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1458                                            pipe->max_packet);
1459         if (packets_to_transfer == 0) {
1460                 packets_to_transfer = 1;
1461         } else if ((packets_to_transfer > 1) &&
1462                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1463                 /*
1464                  * Limit to one packet when not using DMA. Channels must be
1465                  * restarted between every packet for IN transactions, so there
1466                  * is no reason to do multiple packets in a row
1467                  */
1468                 packets_to_transfer = 1;
1469                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1470         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1471                 /*
1472                  * Limit the number of packet and data transferred to what the
1473                  * hardware can handle
1474                  */
1475                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1476                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1477         }
1478
1479         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1480         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1481
1482         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1483                              usbc_hctsiz.u32);
1484 }
1485
1486 /**
1487  * Start a channel to perform the pipe's head transaction
1488  *
1489  * @usb:          USB device state populated by cvmx_usb_initialize().
1490  * @channel:      Channel to setup
1491  * @pipe:         Pipe to start
1492  */
1493 static void cvmx_usb_start_channel(struct octeon_hcd *usb, int channel,
1494                                    struct cvmx_usb_pipe *pipe)
1495 {
1496         struct cvmx_usb_transaction *transaction =
1497                 list_first_entry(&pipe->transactions, typeof(*transaction),
1498                                  node);
1499
1500         /* Make sure all writes to the DMA region get flushed */
1501         CVMX_SYNCW;
1502
1503         /* Attach the channel to the pipe */
1504         usb->pipe_for_channel[channel] = pipe;
1505         pipe->channel = channel;
1506         pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
1507
1508         /* Mark this channel as in use */
1509         usb->idle_hardware_channels &= ~(1 << channel);
1510
1511         /* Enable the channel interrupt bits */
1512         {
1513                 union cvmx_usbcx_hcintx usbc_hcint;
1514                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1515                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1516
1517                 /* Clear all channel status bits */
1518                 usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
1519                                         CVMX_USBCX_HCINTX(channel, usb->index));
1520
1521                 cvmx_usb_write_csr32(usb,
1522                                      CVMX_USBCX_HCINTX(channel, usb->index),
1523                                      usbc_hcint.u32);
1524
1525                 usbc_hcintmsk.u32 = 0;
1526                 usbc_hcintmsk.s.chhltdmsk = 1;
1527                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1528                         /*
1529                          * Channels need these extra interrupts when we aren't
1530                          * in DMA mode.
1531                          */
1532                         usbc_hcintmsk.s.datatglerrmsk = 1;
1533                         usbc_hcintmsk.s.frmovrunmsk = 1;
1534                         usbc_hcintmsk.s.bblerrmsk = 1;
1535                         usbc_hcintmsk.s.xacterrmsk = 1;
1536                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1537                                 /*
1538                                  * Splits don't generate xfercompl, so we need
1539                                  * ACK and NYET.
1540                                  */
1541                                 usbc_hcintmsk.s.nyetmsk = 1;
1542                                 usbc_hcintmsk.s.ackmsk = 1;
1543                         }
1544                         usbc_hcintmsk.s.nakmsk = 1;
1545                         usbc_hcintmsk.s.stallmsk = 1;
1546                         usbc_hcintmsk.s.xfercomplmsk = 1;
1547                 }
1548                 cvmx_usb_write_csr32(usb,
1549                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
1550                                      usbc_hcintmsk.u32);
1551
1552                 /* Enable the channel interrupt to propagate */
1553                 usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
1554                                         CVMX_USBCX_HAINTMSK(usb->index));
1555                 usbc_haintmsk.s.haintmsk |= 1 << channel;
1556                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
1557                                      usbc_haintmsk.u32);
1558         }
1559
1560         /* Setup the location the DMA engine uses. */
1561         {
1562                 u64 reg;
1563                 u64 dma_address = transaction->buffer +
1564                                   transaction->actual_bytes;
1565
1566                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1567                         dma_address = transaction->buffer +
1568                                         transaction->iso_packets[0].offset +
1569                                         transaction->actual_bytes;
1570
1571                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
1572                         reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
1573                 else
1574                         reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
1575                 cvmx_write64_uint64(reg + channel * 8, dma_address);
1576         }
1577
1578         /* Setup both the size of the transfer and the SPLIT characteristics */
1579         {
1580                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1581                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1582                 int packets_to_transfer;
1583                 int bytes_to_transfer = transaction->buffer_length -
1584                         transaction->actual_bytes;
1585
1586                 /*
1587                  * ISOCHRONOUS transactions store each individual transfer size
1588                  * in the packet structure, not the global buffer_length
1589                  */
1590                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1591                         bytes_to_transfer =
1592                                 transaction->iso_packets[0].length -
1593                                 transaction->actual_bytes;
1594
1595                 /*
1596                  * We need to do split transactions when we are talking to non
1597                  * high speed devices that are behind a high speed hub
1598                  */
1599                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1600                         /*
1601                          * On the start split phase (stage is even) record the
1602                          * frame number we will need to send the split complete.
1603                          * We only store the lower two bits since the time ahead
1604                          * can only be two frames
1605                          */
1606                         if ((transaction->stage & 1) == 0) {
1607                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1608                                         pipe->split_sc_frame =
1609                                                 (usb->frame_number + 1) & 0x7f;
1610                                 else
1611                                         pipe->split_sc_frame =
1612                                                 (usb->frame_number + 2) & 0x7f;
1613                         } else {
1614                                 pipe->split_sc_frame = -1;
1615                         }
1616
1617                         usbc_hcsplt.s.spltena = 1;
1618                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1619                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1620                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1621                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1622
1623                         /*
1624                          * SPLIT transactions can only ever transmit one data
1625                          * packet so limit the transfer size to the max packet
1626                          * size
1627                          */
1628                         if (bytes_to_transfer > pipe->max_packet)
1629                                 bytes_to_transfer = pipe->max_packet;
1630
1631                         /*
1632                          * ISOCHRONOUS OUT splits are unique in that they limit
1633                          * data transfers to 188 byte chunks representing the
1634                          * begin/middle/end of the data or all
1635                          */
1636                         if (!usbc_hcsplt.s.compsplt &&
1637                             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1638                             (pipe->transfer_type ==
1639                              CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1640                                 /*
1641                                  * Clear the split complete frame number as
1642                                  * there isn't going to be a split complete
1643                                  */
1644                                 pipe->split_sc_frame = -1;
1645                                 /*
1646                                  * See if we've started this transfer and sent
1647                                  * data
1648                                  */
1649                                 if (transaction->actual_bytes == 0) {
1650                                         /*
1651                                          * Nothing sent yet, this is either a
1652                                          * begin or the entire payload
1653                                          */
1654                                         if (bytes_to_transfer <= 188)
1655                                                 /* Entire payload in one go */
1656                                                 usbc_hcsplt.s.xactpos = 3;
1657                                         else
1658                                                 /* First part of payload */
1659                                                 usbc_hcsplt.s.xactpos = 2;
1660                                 } else {
1661                                         /*
1662                                          * Continuing the previous data, we must
1663                                          * either be in the middle or at the end
1664                                          */
1665                                         if (bytes_to_transfer <= 188)
1666                                                 /* End of payload */
1667                                                 usbc_hcsplt.s.xactpos = 1;
1668                                         else
1669                                                 /* Middle of payload */
1670                                                 usbc_hcsplt.s.xactpos = 0;
1671                                 }
1672                                 /*
1673                                  * Again, the transfer size is limited to 188
1674                                  * bytes
1675                                  */
1676                                 if (bytes_to_transfer > 188)
1677                                         bytes_to_transfer = 188;
1678                         }
1679                 }
1680
1681                 /*
1682                  * Make sure the transfer never exceeds the byte limit of the
1683                  * hardware. Further bytes will be sent as continued
1684                  * transactions
1685                  */
1686                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1687                         /*
1688                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1689                          * size
1690                          */
1691                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1692                                 pipe->max_packet;
1693                         bytes_to_transfer *= pipe->max_packet;
1694                 }
1695
1696                 /*
1697                  * Calculate the number of packets to transfer. If the length is
1698                  * zero we still need to transfer one packet
1699                  */
1700                 packets_to_transfer =
1701                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1702                 if (packets_to_transfer == 0) {
1703                         packets_to_transfer = 1;
1704                 } else if ((packets_to_transfer > 1) &&
1705                            (usb->init_flags &
1706                             CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1707                         /*
1708                          * Limit to one packet when not using DMA. Channels must
1709                          * be restarted between every packet for IN
1710                          * transactions, so there is no reason to do multiple
1711                          * packets in a row
1712                          */
1713                         packets_to_transfer = 1;
1714                         bytes_to_transfer = packets_to_transfer *
1715                                 pipe->max_packet;
1716                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1717                         /*
1718                          * Limit the number of packet and data transferred to
1719                          * what the hardware can handle
1720                          */
1721                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1722                         bytes_to_transfer = packets_to_transfer *
1723                                 pipe->max_packet;
1724                 }
1725
1726                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1727                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1728
1729                 /* Update the DATA0/DATA1 toggle */
1730                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1731                 /*
1732                  * High speed pipes may need a hardware ping before they start
1733                  */
1734                 if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
1735                         usbc_hctsiz.s.dopng = 1;
1736
1737                 cvmx_usb_write_csr32(usb,
1738                                      CVMX_USBCX_HCSPLTX(channel, usb->index),
1739                                      usbc_hcsplt.u32);
1740                 cvmx_usb_write_csr32(usb,
1741                                      CVMX_USBCX_HCTSIZX(channel, usb->index),
1742                                      usbc_hctsiz.u32);
1743         }
1744
1745         /* Setup the Host Channel Characteristics Register */
1746         {
1747                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1748
1749                 /*
1750                  * Set the startframe odd/even properly. This is only used for
1751                  * periodic
1752                  */
1753                 usbc_hcchar.s.oddfrm = usb->frame_number & 1;
1754
1755                 /*
1756                  * Set the number of back to back packets allowed by this
1757                  * endpoint. Split transactions interpret "ec" as the number of
1758                  * immediate retries of failure. These retries happen too
1759                  * quickly, so we disable these entirely for splits
1760                  */
1761                 if (cvmx_usb_pipe_needs_split(usb, pipe))
1762                         usbc_hcchar.s.ec = 1;
1763                 else if (pipe->multi_count < 1)
1764                         usbc_hcchar.s.ec = 1;
1765                 else if (pipe->multi_count > 3)
1766                         usbc_hcchar.s.ec = 3;
1767                 else
1768                         usbc_hcchar.s.ec = pipe->multi_count;
1769
1770                 /* Set the rest of the endpoint specific settings */
1771                 usbc_hcchar.s.devaddr = pipe->device_addr;
1772                 usbc_hcchar.s.eptype = transaction->type;
1773                 usbc_hcchar.s.lspddev =
1774                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1775                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1776                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1777                 usbc_hcchar.s.mps = pipe->max_packet;
1778                 cvmx_usb_write_csr32(usb,
1779                                      CVMX_USBCX_HCCHARX(channel, usb->index),
1780                                      usbc_hcchar.u32);
1781         }
1782
1783         /* Do transaction type specific fixups as needed */
1784         switch (transaction->type) {
1785         case CVMX_USB_TRANSFER_CONTROL:
1786                 cvmx_usb_start_channel_control(usb, channel, pipe);
1787                 break;
1788         case CVMX_USB_TRANSFER_BULK:
1789         case CVMX_USB_TRANSFER_INTERRUPT:
1790                 break;
1791         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1792                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
1793                         /*
1794                          * ISO transactions require different PIDs depending on
1795                          * direction and how many packets are needed
1796                          */
1797                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1798                                 if (pipe->multi_count < 2) /* Need DATA0 */
1799                                         USB_SET_FIELD32(
1800                                                 CVMX_USBCX_HCTSIZX(channel,
1801                                                                    usb->index),
1802                                                 cvmx_usbcx_hctsizx, pid, 0);
1803                                 else /* Need MDATA */
1804                                         USB_SET_FIELD32(
1805                                                 CVMX_USBCX_HCTSIZX(channel,
1806                                                                    usb->index),
1807                                                 cvmx_usbcx_hctsizx, pid, 3);
1808                         }
1809                 }
1810                 break;
1811         }
1812         {
1813                 union cvmx_usbcx_hctsizx usbc_hctsiz = { .u32 =
1814                         cvmx_usb_read_csr32(usb,
1815                                             CVMX_USBCX_HCTSIZX(channel,
1816                                                                usb->index))
1817                 };
1818                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1819                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1820         }
1821         /* Remember when we start a split transaction */
1822         if (cvmx_usb_pipe_needs_split(usb, pipe))
1823                 usb->active_split = transaction;
1824         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1825                         cvmx_usbcx_hccharx, chena, 1);
1826         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1827                 cvmx_usb_fill_tx_fifo(usb, channel);
1828 }
1829
1830 /**
1831  * Find a pipe that is ready to be scheduled to hardware.
1832  * @usb:         USB device state populated by cvmx_usb_initialize().
1833  * @xfer_type:   Transfer type
1834  *
1835  * Returns: Pipe or NULL if none are ready
1836  */
1837 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(
1838                 struct octeon_hcd *usb,
1839                 enum cvmx_usb_transfer xfer_type)
1840 {
1841         struct list_head *list = usb->active_pipes + xfer_type;
1842         u64 current_frame = usb->frame_number;
1843         struct cvmx_usb_pipe *pipe;
1844
1845         list_for_each_entry(pipe, list, node) {
1846                 struct cvmx_usb_transaction *t =
1847                         list_first_entry(&pipe->transactions, typeof(*t),
1848                                          node);
1849                 if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1850                     (pipe->next_tx_frame <= current_frame) &&
1851                     ((pipe->split_sc_frame == -1) ||
1852                      ((((int)current_frame - pipe->split_sc_frame) & 0x7f) <
1853                       0x40)) &&
1854                     (!usb->active_split || (usb->active_split == t))) {
1855                         prefetch(t);
1856                         return pipe;
1857                 }
1858         }
1859         return NULL;
1860 }
1861
1862 static struct cvmx_usb_pipe *cvmx_usb_next_pipe(struct octeon_hcd *usb,
1863                                                 int is_sof)
1864 {
1865         struct cvmx_usb_pipe *pipe;
1866
1867         /* Find a pipe needing service. */
1868         if (is_sof) {
1869                 /*
1870                  * Only process periodic pipes on SOF interrupts. This way we
1871                  * are sure that the periodic data is sent in the beginning of
1872                  * the frame.
1873                  */
1874                 pipe = cvmx_usb_find_ready_pipe(usb,
1875                                                 CVMX_USB_TRANSFER_ISOCHRONOUS);
1876                 if (pipe)
1877                         return pipe;
1878                 pipe = cvmx_usb_find_ready_pipe(usb,
1879                                                 CVMX_USB_TRANSFER_INTERRUPT);
1880                 if (pipe)
1881                         return pipe;
1882         }
1883         pipe = cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_CONTROL);
1884         if (pipe)
1885                 return pipe;
1886         return cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_BULK);
1887 }
1888
1889 /**
1890  * Called whenever a pipe might need to be scheduled to the
1891  * hardware.
1892  *
1893  * @usb:         USB device state populated by cvmx_usb_initialize().
1894  * @is_sof:      True if this schedule was called on a SOF interrupt.
1895  */
1896 static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
1897 {
1898         int channel;
1899         struct cvmx_usb_pipe *pipe;
1900         int need_sof;
1901         enum cvmx_usb_transfer ttype;
1902
1903         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1904                 /*
1905                  * Without DMA we need to be careful to not schedule something
1906                  * at the end of a frame and cause an overrun.
1907                  */
1908                 union cvmx_usbcx_hfnum hfnum = {
1909                         .u32 = cvmx_usb_read_csr32(usb,
1910                                                 CVMX_USBCX_HFNUM(usb->index))
1911                 };
1912
1913                 union cvmx_usbcx_hfir hfir = {
1914                         .u32 = cvmx_usb_read_csr32(usb,
1915                                                 CVMX_USBCX_HFIR(usb->index))
1916                 };
1917
1918                 if (hfnum.s.frrem < hfir.s.frint / 4)
1919                         goto done;
1920         }
1921
1922         while (usb->idle_hardware_channels) {
1923                 /* Find an idle channel */
1924                 channel = __fls(usb->idle_hardware_channels);
1925                 if (unlikely(channel > 7))
1926                         break;
1927
1928                 pipe = cvmx_usb_next_pipe(usb, is_sof);
1929                 if (!pipe)
1930                         break;
1931
1932                 cvmx_usb_start_channel(usb, channel, pipe);
1933         }
1934
1935 done:
1936         /*
1937          * Only enable SOF interrupts when we have transactions pending in the
1938          * future that might need to be scheduled
1939          */
1940         need_sof = 0;
1941         for (ttype = CVMX_USB_TRANSFER_CONTROL;
1942              ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1943                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1944                         if (pipe->next_tx_frame > usb->frame_number) {
1945                                 need_sof = 1;
1946                                 break;
1947                         }
1948                 }
1949         }
1950         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1951                         cvmx_usbcx_gintmsk, sofmsk, need_sof);
1952 }
1953
1954 static void octeon_usb_urb_complete_callback(struct octeon_hcd *usb,
1955                                              enum cvmx_usb_status status,
1956                                              struct cvmx_usb_pipe *pipe,
1957                                              struct cvmx_usb_transaction
1958                                                 *transaction,
1959                                              int bytes_transferred,
1960                                              struct urb *urb)
1961 {
1962         struct usb_hcd *hcd = octeon_to_hcd(usb);
1963         struct device *dev = hcd->self.controller;
1964
1965         if (likely(status == CVMX_USB_STATUS_OK))
1966                 urb->actual_length = bytes_transferred;
1967         else
1968                 urb->actual_length = 0;
1969
1970         urb->hcpriv = NULL;
1971
1972         /* For Isochronous transactions we need to update the URB packet status
1973          * list from data in our private copy
1974          */
1975         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
1976                 int i;
1977                 /*
1978                  * The pointer to the private list is stored in the setup_packet
1979                  * field.
1980                  */
1981                 struct cvmx_usb_iso_packet *iso_packet =
1982                         (struct cvmx_usb_iso_packet *)urb->setup_packet;
1983                 /* Recalculate the transfer size by adding up each packet */
1984                 urb->actual_length = 0;
1985                 for (i = 0; i < urb->number_of_packets; i++) {
1986                         if (iso_packet[i].status == CVMX_USB_STATUS_OK) {
1987                                 urb->iso_frame_desc[i].status = 0;
1988                                 urb->iso_frame_desc[i].actual_length =
1989                                         iso_packet[i].length;
1990                                 urb->actual_length +=
1991                                         urb->iso_frame_desc[i].actual_length;
1992                         } else {
1993                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
1994                                         i, urb->number_of_packets,
1995                                         iso_packet[i].status, pipe,
1996                                         transaction, iso_packet[i].length);
1997                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
1998                         }
1999                 }
2000                 /* Free the private list now that we don't need it anymore */
2001                 kfree(iso_packet);
2002                 urb->setup_packet = NULL;
2003         }
2004
2005         switch (status) {
2006         case CVMX_USB_STATUS_OK:
2007                 urb->status = 0;
2008                 break;
2009         case CVMX_USB_STATUS_CANCEL:
2010                 if (urb->status == 0)
2011                         urb->status = -ENOENT;
2012                 break;
2013         case CVMX_USB_STATUS_STALL:
2014                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2015                         pipe, transaction, bytes_transferred);
2016                 urb->status = -EPIPE;
2017                 break;
2018         case CVMX_USB_STATUS_BABBLEERR:
2019                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2020                         pipe, transaction, bytes_transferred);
2021                 urb->status = -EPIPE;
2022                 break;
2023         case CVMX_USB_STATUS_SHORT:
2024                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2025                         pipe, transaction, bytes_transferred);
2026                 urb->status = -EREMOTEIO;
2027                 break;
2028         case CVMX_USB_STATUS_ERROR:
2029         case CVMX_USB_STATUS_XACTERR:
2030         case CVMX_USB_STATUS_DATATGLERR:
2031         case CVMX_USB_STATUS_FRAMEERR:
2032                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2033                         status, pipe, transaction, bytes_transferred);
2034                 urb->status = -EPROTO;
2035                 break;
2036         }
2037         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(usb), urb);
2038         spin_unlock(&usb->lock);
2039         usb_hcd_giveback_urb(octeon_to_hcd(usb), urb, urb->status);
2040         spin_lock(&usb->lock);
2041 }
2042
2043 /**
2044  * Signal the completion of a transaction and free it. The
2045  * transaction will be removed from the pipe transaction list.
2046  *
2047  * @usb:         USB device state populated by cvmx_usb_initialize().
2048  * @pipe:        Pipe the transaction is on
2049  * @transaction:
2050  *               Transaction that completed
2051  * @complete_code:
2052  *               Completion code
2053  */
2054 static void cvmx_usb_complete(struct octeon_hcd *usb,
2055                               struct cvmx_usb_pipe *pipe,
2056                               struct cvmx_usb_transaction *transaction,
2057                               enum cvmx_usb_status complete_code)
2058 {
2059         /* If this was a split then clear our split in progress marker */
2060         if (usb->active_split == transaction)
2061                 usb->active_split = NULL;
2062
2063         /*
2064          * Isochronous transactions need extra processing as they might not be
2065          * done after a single data transfer
2066          */
2067         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2068                 /* Update the number of bytes transferred in this ISO packet */
2069                 transaction->iso_packets[0].length = transaction->actual_bytes;
2070                 transaction->iso_packets[0].status = complete_code;
2071
2072                 /*
2073                  * If there are more ISOs pending and we succeeded, schedule the
2074                  * next one
2075                  */
2076                 if ((transaction->iso_number_packets > 1) &&
2077                     (complete_code == CVMX_USB_STATUS_OK)) {
2078                         /* No bytes transferred for this packet as of yet */
2079                         transaction->actual_bytes = 0;
2080                         /* One less ISO waiting to transfer */
2081                         transaction->iso_number_packets--;
2082                         /* Increment to the next location in our packet array */
2083                         transaction->iso_packets++;
2084                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2085                         return;
2086                 }
2087         }
2088
2089         /* Remove the transaction from the pipe list */
2090         list_del(&transaction->node);
2091         if (list_empty(&pipe->transactions))
2092                 list_move_tail(&pipe->node, &usb->idle_pipes);
2093         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2094                                          transaction,
2095                                          transaction->actual_bytes,
2096                                          transaction->urb);
2097         kfree(transaction);
2098 }
2099
2100 /**
2101  * Submit a usb transaction to a pipe. Called for all types
2102  * of transactions.
2103  *
2104  * @usb:
2105  * @pipe:           Which pipe to submit to.
2106  * @type:           Transaction type
2107  * @buffer:         User buffer for the transaction
2108  * @buffer_length:
2109  *                  User buffer's length in bytes
2110  * @control_header:
2111  *                  For control transactions, the 8 byte standard header
2112  * @iso_start_frame:
2113  *                  For ISO transactions, the start frame
2114  * @iso_number_packets:
2115  *                  For ISO, the number of packet in the transaction.
2116  * @iso_packets:
2117  *                  A description of each ISO packet
2118  * @urb:            URB for the callback
2119  *
2120  * Returns: Transaction or NULL on failure.
2121  */
2122 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
2123                                 struct octeon_hcd *usb,
2124                                 struct cvmx_usb_pipe *pipe,
2125                                 enum cvmx_usb_transfer type,
2126                                 u64 buffer,
2127                                 int buffer_length,
2128                                 u64 control_header,
2129                                 int iso_start_frame,
2130                                 int iso_number_packets,
2131                                 struct cvmx_usb_iso_packet *iso_packets,
2132                                 struct urb *urb)
2133 {
2134         struct cvmx_usb_transaction *transaction;
2135
2136         if (unlikely(pipe->transfer_type != type))
2137                 return NULL;
2138
2139         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2140         if (unlikely(!transaction))
2141                 return NULL;
2142
2143         transaction->type = type;
2144         transaction->buffer = buffer;
2145         transaction->buffer_length = buffer_length;
2146         transaction->control_header = control_header;
2147         /* FIXME: This is not used, implement it. */
2148         transaction->iso_start_frame = iso_start_frame;
2149         transaction->iso_number_packets = iso_number_packets;
2150         transaction->iso_packets = iso_packets;
2151         transaction->urb = urb;
2152         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2153                 transaction->stage = CVMX_USB_STAGE_SETUP;
2154         else
2155                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2156
2157         if (!list_empty(&pipe->transactions)) {
2158                 list_add_tail(&transaction->node, &pipe->transactions);
2159         } else {
2160                 list_add_tail(&transaction->node, &pipe->transactions);
2161                 list_move_tail(&pipe->node,
2162                                &usb->active_pipes[pipe->transfer_type]);
2163
2164                 /*
2165                  * We may need to schedule the pipe if this was the head of the
2166                  * pipe.
2167                  */
2168                 cvmx_usb_schedule(usb, 0);
2169         }
2170
2171         return transaction;
2172 }
2173
2174 /**
2175  * Call to submit a USB Bulk transfer to a pipe.
2176  *
2177  * @usb:            USB device state populated by cvmx_usb_initialize().
2178  * @pipe:           Handle to the pipe for the transfer.
2179  * @urb:            URB.
2180  *
2181  * Returns: A submitted transaction or NULL on failure.
2182  */
2183 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2184                                                 struct octeon_hcd *usb,
2185                                                 struct cvmx_usb_pipe *pipe,
2186                                                 struct urb *urb)
2187 {
2188         return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2189                                            urb->transfer_dma,
2190                                            urb->transfer_buffer_length,
2191                                            0, /* control_header */
2192                                            0, /* iso_start_frame */
2193                                            0, /* iso_number_packets */
2194                                            NULL, /* iso_packets */
2195                                            urb);
2196 }
2197
2198 /**
2199  * Call to submit a USB Interrupt transfer to a pipe.
2200  *
2201  * @usb:            USB device state populated by cvmx_usb_initialize().
2202  * @pipe:           Handle to the pipe for the transfer.
2203  * @urb:            URB returned when the callback is called.
2204  *
2205  * Returns: A submitted transaction or NULL on failure.
2206  */
2207 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2208                                                 struct octeon_hcd *usb,
2209                                                 struct cvmx_usb_pipe *pipe,
2210                                                 struct urb *urb)
2211 {
2212         return cvmx_usb_submit_transaction(usb, pipe,
2213                                            CVMX_USB_TRANSFER_INTERRUPT,
2214                                            urb->transfer_dma,
2215                                            urb->transfer_buffer_length,
2216                                            0, /* control_header */
2217                                            0, /* iso_start_frame */
2218                                            0, /* iso_number_packets */
2219                                            NULL, /* iso_packets */
2220                                            urb);
2221 }
2222
2223 /**
2224  * Call to submit a USB Control transfer to a pipe.
2225  *
2226  * @usb:            USB device state populated by cvmx_usb_initialize().
2227  * @pipe:           Handle to the pipe for the transfer.
2228  * @urb:            URB.
2229  *
2230  * Returns: A submitted transaction or NULL on failure.
2231  */
2232 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2233                                                 struct octeon_hcd *usb,
2234                                                 struct cvmx_usb_pipe *pipe,
2235                                                 struct urb *urb)
2236 {
2237         int buffer_length = urb->transfer_buffer_length;
2238         u64 control_header = urb->setup_dma;
2239         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2240
2241         if ((header->bRequestType & USB_DIR_IN) == 0)
2242                 buffer_length = le16_to_cpu(header->wLength);
2243
2244         return cvmx_usb_submit_transaction(usb, pipe,
2245                                            CVMX_USB_TRANSFER_CONTROL,
2246                                            urb->transfer_dma, buffer_length,
2247                                            control_header,
2248                                            0, /* iso_start_frame */
2249                                            0, /* iso_number_packets */
2250                                            NULL, /* iso_packets */
2251                                            urb);
2252 }
2253
2254 /**
2255  * Call to submit a USB Isochronous transfer to a pipe.
2256  *
2257  * @usb:            USB device state populated by cvmx_usb_initialize().
2258  * @pipe:           Handle to the pipe for the transfer.
2259  * @urb:            URB returned when the callback is called.
2260  *
2261  * Returns: A submitted transaction or NULL on failure.
2262  */
2263 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2264                                                 struct octeon_hcd *usb,
2265                                                 struct cvmx_usb_pipe *pipe,
2266                                                 struct urb *urb)
2267 {
2268         struct cvmx_usb_iso_packet *packets;
2269
2270         packets = (struct cvmx_usb_iso_packet *)urb->setup_packet;
2271         return cvmx_usb_submit_transaction(usb, pipe,
2272                                            CVMX_USB_TRANSFER_ISOCHRONOUS,
2273                                            urb->transfer_dma,
2274                                            urb->transfer_buffer_length,
2275                                            0, /* control_header */
2276                                            urb->start_frame,
2277                                            urb->number_of_packets,
2278                                            packets, urb);
2279 }
2280
2281 /**
2282  * Cancel one outstanding request in a pipe. Canceling a request
2283  * can fail if the transaction has already completed before cancel
2284  * is called. Even after a successful cancel call, it may take
2285  * a frame or two for the cvmx_usb_poll() function to call the
2286  * associated callback.
2287  *
2288  * @usb:         USB device state populated by cvmx_usb_initialize().
2289  * @pipe:        Pipe to cancel requests in.
2290  * @transaction: Transaction to cancel, returned by the submit function.
2291  *
2292  * Returns: 0 or a negative error code.
2293  */
2294 static int cvmx_usb_cancel(struct octeon_hcd *usb,
2295                            struct cvmx_usb_pipe *pipe,
2296                            struct cvmx_usb_transaction *transaction)
2297 {
2298         /*
2299          * If the transaction is the HEAD of the queue and scheduled. We need to
2300          * treat it special
2301          */
2302         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2303             transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2304                 union cvmx_usbcx_hccharx usbc_hcchar;
2305
2306                 usb->pipe_for_channel[pipe->channel] = NULL;
2307                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2308
2309                 CVMX_SYNCW;
2310
2311                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2312                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2313                 /*
2314                  * If the channel isn't enabled then the transaction already
2315                  * completed.
2316                  */
2317                 if (usbc_hcchar.s.chena) {
2318                         usbc_hcchar.s.chdis = 1;
2319                         cvmx_usb_write_csr32(usb,
2320                                              CVMX_USBCX_HCCHARX(pipe->channel,
2321                                                                 usb->index),
2322                                              usbc_hcchar.u32);
2323                 }
2324         }
2325         cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_CANCEL);
2326         return 0;
2327 }
2328
2329 /**
2330  * Cancel all outstanding requests in a pipe. Logically all this
2331  * does is call cvmx_usb_cancel() in a loop.
2332  *
2333  * @usb:         USB device state populated by cvmx_usb_initialize().
2334  * @pipe:        Pipe to cancel requests in.
2335  *
2336  * Returns: 0 or a negative error code.
2337  */
2338 static int cvmx_usb_cancel_all(struct octeon_hcd *usb,
2339                                struct cvmx_usb_pipe *pipe)
2340 {
2341         struct cvmx_usb_transaction *transaction, *next;
2342
2343         /* Simply loop through and attempt to cancel each transaction */
2344         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2345                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2346
2347                 if (unlikely(result != 0))
2348                         return result;
2349         }
2350         return 0;
2351 }
2352
2353 /**
2354  * Close a pipe created with cvmx_usb_open_pipe().
2355  *
2356  * @usb:         USB device state populated by cvmx_usb_initialize().
2357  * @pipe:        Pipe to close.
2358  *
2359  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2360  *          outstanding transfers.
2361  */
2362 static int cvmx_usb_close_pipe(struct octeon_hcd *usb,
2363                                struct cvmx_usb_pipe *pipe)
2364 {
2365         /* Fail if the pipe has pending transactions */
2366         if (!list_empty(&pipe->transactions))
2367                 return -EBUSY;
2368
2369         list_del(&pipe->node);
2370         kfree(pipe);
2371
2372         return 0;
2373 }
2374
2375 /**
2376  * Get the current USB protocol level frame number. The frame
2377  * number is always in the range of 0-0x7ff.
2378  *
2379  * @usb: USB device state populated by cvmx_usb_initialize().
2380  *
2381  * Returns: USB frame number
2382  */
2383 static int cvmx_usb_get_frame_number(struct octeon_hcd *usb)
2384 {
2385         int frame_number;
2386         union cvmx_usbcx_hfnum usbc_hfnum;
2387
2388         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2389         frame_number = usbc_hfnum.s.frnum;
2390
2391         return frame_number;
2392 }
2393
2394 static void cvmx_usb_transfer_control(struct octeon_hcd *usb,
2395                                       struct cvmx_usb_pipe *pipe,
2396                                       struct cvmx_usb_transaction *transaction,
2397                                       union cvmx_usbcx_hccharx usbc_hcchar,
2398                                       int buffer_space_left,
2399                                       int bytes_in_last_packet)
2400 {
2401         switch (transaction->stage) {
2402         case CVMX_USB_STAGE_NON_CONTROL:
2403         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2404                 /* This should be impossible */
2405                 cvmx_usb_complete(usb, pipe, transaction,
2406                                   CVMX_USB_STATUS_ERROR);
2407                 break;
2408         case CVMX_USB_STAGE_SETUP:
2409                 pipe->pid_toggle = 1;
2410                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2411                         transaction->stage =
2412                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2413                 } else {
2414                         struct usb_ctrlrequest *header =
2415                                 cvmx_phys_to_ptr(transaction->control_header);
2416                         if (header->wLength)
2417                                 transaction->stage = CVMX_USB_STAGE_DATA;
2418                         else
2419                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2420                 }
2421                 break;
2422         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2423                 {
2424                         struct usb_ctrlrequest *header =
2425                                 cvmx_phys_to_ptr(transaction->control_header);
2426                         if (header->wLength)
2427                                 transaction->stage = CVMX_USB_STAGE_DATA;
2428                         else
2429                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2430                 }
2431                 break;
2432         case CVMX_USB_STAGE_DATA:
2433                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2434                         transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2435                         /*
2436                          * For setup OUT data that are splits,
2437                          * the hardware doesn't appear to count
2438                          * transferred data. Here we manually
2439                          * update the data transferred
2440                          */
2441                         if (!usbc_hcchar.s.epdir) {
2442                                 if (buffer_space_left < pipe->max_packet)
2443                                         transaction->actual_bytes +=
2444                                                 buffer_space_left;
2445                                 else
2446                                         transaction->actual_bytes +=
2447                                                 pipe->max_packet;
2448                         }
2449                 } else if ((buffer_space_left == 0) ||
2450                            (bytes_in_last_packet < pipe->max_packet)) {
2451                         pipe->pid_toggle = 1;
2452                         transaction->stage = CVMX_USB_STAGE_STATUS;
2453                 }
2454                 break;
2455         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2456                 if ((buffer_space_left == 0) ||
2457                     (bytes_in_last_packet < pipe->max_packet)) {
2458                         pipe->pid_toggle = 1;
2459                         transaction->stage = CVMX_USB_STAGE_STATUS;
2460                 } else {
2461                         transaction->stage = CVMX_USB_STAGE_DATA;
2462                 }
2463                 break;
2464         case CVMX_USB_STAGE_STATUS:
2465                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2466                         transaction->stage =
2467                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2468                 else
2469                         cvmx_usb_complete(usb, pipe, transaction,
2470                                           CVMX_USB_STATUS_OK);
2471                 break;
2472         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2473                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2474                 break;
2475         }
2476 }
2477
2478 static void cvmx_usb_transfer_bulk(struct octeon_hcd *usb,
2479                                    struct cvmx_usb_pipe *pipe,
2480                                    struct cvmx_usb_transaction *transaction,
2481                                    union cvmx_usbcx_hcintx usbc_hcint,
2482                                    int buffer_space_left,
2483                                    int bytes_in_last_packet)
2484 {
2485         /*
2486          * The only time a bulk transfer isn't complete when it finishes with
2487          * an ACK is during a split transaction. For splits we need to continue
2488          * the transfer if more data is needed.
2489          */
2490         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2491                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
2492                         transaction->stage =
2493                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2494                 else if (buffer_space_left &&
2495                          (bytes_in_last_packet == pipe->max_packet))
2496                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2497                 else
2498                         cvmx_usb_complete(usb, pipe, transaction,
2499                                           CVMX_USB_STATUS_OK);
2500         } else {
2501                 if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2502                     (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
2503                     (usbc_hcint.s.nak))
2504                         pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2505                 if (!buffer_space_left ||
2506                     (bytes_in_last_packet < pipe->max_packet))
2507                         cvmx_usb_complete(usb, pipe, transaction,
2508                                           CVMX_USB_STATUS_OK);
2509         }
2510 }
2511
2512 static void cvmx_usb_transfer_intr(struct octeon_hcd *usb,
2513                                    struct cvmx_usb_pipe *pipe,
2514                                    struct cvmx_usb_transaction *transaction,
2515                                    int buffer_space_left,
2516                                    int bytes_in_last_packet)
2517 {
2518         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2519                 if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL) {
2520                         transaction->stage =
2521                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2522                 } else if (buffer_space_left &&
2523                            (bytes_in_last_packet == pipe->max_packet)) {
2524                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2525                 } else {
2526                         pipe->next_tx_frame += pipe->interval;
2527                         cvmx_usb_complete(usb, pipe, transaction,
2528                                           CVMX_USB_STATUS_OK);
2529                 }
2530         } else if (!buffer_space_left ||
2531                    (bytes_in_last_packet < pipe->max_packet)) {
2532                 pipe->next_tx_frame += pipe->interval;
2533                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2534         }
2535 }
2536
2537 static void cvmx_usb_transfer_isoc(struct octeon_hcd *usb,
2538                                    struct cvmx_usb_pipe *pipe,
2539                                    struct cvmx_usb_transaction *transaction,
2540                                    int buffer_space_left,
2541                                    int bytes_in_last_packet,
2542                                    int bytes_this_transfer)
2543 {
2544         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2545                 /*
2546                  * ISOCHRONOUS OUT splits don't require a complete split stage.
2547                  * Instead they use a sequence of begin OUT splits to transfer
2548                  * the data 188 bytes at a time. Once the transfer is complete,
2549                  * the pipe sleeps until the next schedule interval.
2550                  */
2551                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
2552                         /*
2553                          * If no space left or this wasn't a max size packet
2554                          * then this transfer is complete. Otherwise start it
2555                          * again to send the next 188 bytes
2556                          */
2557                         if (!buffer_space_left || (bytes_this_transfer < 188)) {
2558                                 pipe->next_tx_frame += pipe->interval;
2559                                 cvmx_usb_complete(usb, pipe, transaction,
2560                                                   CVMX_USB_STATUS_OK);
2561                         }
2562                         return;
2563                 }
2564                 if (transaction->stage ==
2565                     CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2566                         /*
2567                          * We are in the incoming data phase. Keep getting data
2568                          * until we run out of space or get a small packet
2569                          */
2570                         if ((buffer_space_left == 0) ||
2571                             (bytes_in_last_packet < pipe->max_packet)) {
2572                                 pipe->next_tx_frame += pipe->interval;
2573                                 cvmx_usb_complete(usb, pipe, transaction,
2574                                                   CVMX_USB_STATUS_OK);
2575                         }
2576                 } else {
2577                         transaction->stage =
2578                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2579                 }
2580         } else {
2581                 pipe->next_tx_frame += pipe->interval;
2582                 cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
2583         }
2584 }
2585
2586 /**
2587  * Poll a channel for status
2588  *
2589  * @usb:     USB device
2590  * @channel: Channel to poll
2591  *
2592  * Returns: Zero on success
2593  */
2594 static int cvmx_usb_poll_channel(struct octeon_hcd *usb, int channel)
2595 {
2596         struct usb_hcd *hcd = octeon_to_hcd(usb);
2597         struct device *dev = hcd->self.controller;
2598         union cvmx_usbcx_hcintx usbc_hcint;
2599         union cvmx_usbcx_hctsizx usbc_hctsiz;
2600         union cvmx_usbcx_hccharx usbc_hcchar;
2601         struct cvmx_usb_pipe *pipe;
2602         struct cvmx_usb_transaction *transaction;
2603         int bytes_this_transfer;
2604         int bytes_in_last_packet;
2605         int packets_processed;
2606         int buffer_space_left;
2607
2608         /* Read the interrupt status bits for the channel */
2609         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
2610                                 CVMX_USBCX_HCINTX(channel, usb->index));
2611
2612         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2613                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2614                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2615
2616                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2617                         /*
2618                          * There seems to be a bug in CN31XX which can cause
2619                          * interrupt IN transfers to get stuck until we do a
2620                          * write of HCCHARX without changing things
2621                          */
2622                         cvmx_usb_write_csr32(usb,
2623                                              CVMX_USBCX_HCCHARX(channel,
2624                                                                 usb->index),
2625                                              usbc_hcchar.u32);
2626                         return 0;
2627                 }
2628
2629                 /*
2630                  * In non DMA mode the channels don't halt themselves. We need
2631                  * to manually disable channels that are left running
2632                  */
2633                 if (!usbc_hcint.s.chhltd) {
2634                         if (usbc_hcchar.s.chena) {
2635                                 union cvmx_usbcx_hcintmskx hcintmsk;
2636                                 /* Disable all interrupts except CHHLTD */
2637                                 hcintmsk.u32 = 0;
2638                                 hcintmsk.s.chhltdmsk = 1;
2639                                 cvmx_usb_write_csr32(usb,
2640                                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
2641                                                      hcintmsk.u32);
2642                                 usbc_hcchar.s.chdis = 1;
2643                                 cvmx_usb_write_csr32(usb,
2644                                                      CVMX_USBCX_HCCHARX(channel, usb->index),
2645                                                      usbc_hcchar.u32);
2646                                 return 0;
2647                         } else if (usbc_hcint.s.xfercompl) {
2648                                 /*
2649                                  * Successful IN/OUT with transfer complete.
2650                                  * Channel halt isn't needed.
2651                                  */
2652                         } else {
2653                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2654                                         usb->index, channel);
2655                                 return 0;
2656                         }
2657                 }
2658         } else {
2659                 /*
2660                  * There is are no interrupts that we need to process when the
2661                  * channel is still running
2662                  */
2663                 if (!usbc_hcint.s.chhltd)
2664                         return 0;
2665         }
2666
2667         /* Disable the channel interrupts now that it is done */
2668         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2669         usb->idle_hardware_channels |= (1 << channel);
2670
2671         /* Make sure this channel is tied to a valid pipe */
2672         pipe = usb->pipe_for_channel[channel];
2673         prefetch(pipe);
2674         if (!pipe)
2675                 return 0;
2676         transaction = list_first_entry(&pipe->transactions,
2677                                        typeof(*transaction),
2678                                        node);
2679         prefetch(transaction);
2680
2681         /*
2682          * Disconnect this pipe from the HW channel. Later the schedule
2683          * function will figure out which pipe needs to go
2684          */
2685         usb->pipe_for_channel[channel] = NULL;
2686         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2687
2688         /*
2689          * Read the channel config info so we can figure out how much data
2690          * transferred
2691          */
2692         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2693                         CVMX_USBCX_HCCHARX(channel, usb->index));
2694         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
2695                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2696
2697         /*
2698          * Calculating the number of bytes successfully transferred is dependent
2699          * on the transfer direction
2700          */
2701         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2702         if (usbc_hcchar.s.epdir) {
2703                 /*
2704                  * IN transactions are easy. For every byte received the
2705                  * hardware decrements xfersize. All we need to do is subtract
2706                  * the current value of xfersize from its starting value and we
2707                  * know how many bytes were written to the buffer
2708                  */
2709                 bytes_this_transfer = transaction->xfersize -
2710                         usbc_hctsiz.s.xfersize;
2711         } else {
2712                 /*
2713                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2714                  * decremented on every successful packet send. The hardware
2715                  * does this when it receives an ACK, or NYET. If it doesn't
2716                  * receive one of these responses pktcnt doesn't change
2717                  */
2718                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2719                 /*
2720                  * The last packet may not be a full transfer if we didn't have
2721                  * enough data
2722                  */
2723                 if (bytes_this_transfer > transaction->xfersize)
2724                         bytes_this_transfer = transaction->xfersize;
2725         }
2726         /* Figure out how many bytes were in the last packet of the transfer */
2727         if (packets_processed)
2728                 bytes_in_last_packet = bytes_this_transfer -
2729                         (packets_processed - 1) * usbc_hcchar.s.mps;
2730         else
2731                 bytes_in_last_packet = bytes_this_transfer;
2732
2733         /*
2734          * As a special case, setup transactions output the setup header, not
2735          * the user's data. For this reason we don't count setup data as bytes
2736          * transferred
2737          */
2738         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2739             (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2740                 bytes_this_transfer = 0;
2741
2742         /*
2743          * Add the bytes transferred to the running total. It is important that
2744          * bytes_this_transfer doesn't count any data that needs to be
2745          * retransmitted
2746          */
2747         transaction->actual_bytes += bytes_this_transfer;
2748         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2749                 buffer_space_left = transaction->iso_packets[0].length -
2750                         transaction->actual_bytes;
2751         else
2752                 buffer_space_left = transaction->buffer_length -
2753                         transaction->actual_bytes;
2754
2755         /*
2756          * We need to remember the PID toggle state for the next transaction.
2757          * The hardware already updated it for the next transaction
2758          */
2759         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2760
2761         /*
2762          * For high speed bulk out, assume the next transaction will need to do
2763          * a ping before proceeding. If this isn't true the ACK processing below
2764          * will clear this flag
2765          */
2766         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2767             (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2768             (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2769                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2770
2771         if (unlikely(WARN_ON_ONCE(bytes_this_transfer < 0))) {
2772                 /*
2773                  * In some rare cases the DMA engine seems to get stuck and
2774                  * keeps substracting same byte count over and over again. In
2775                  * such case we just need to fail every transaction.
2776                  */
2777                 cvmx_usb_complete(usb, pipe, transaction,
2778                                   CVMX_USB_STATUS_ERROR);
2779                 return 0;
2780         }
2781
2782         if (usbc_hcint.s.stall) {
2783                 /*
2784                  * STALL as a response means this transaction cannot be
2785                  * completed because the device can't process transactions. Tell
2786                  * the user. Any data that was transferred will be counted on
2787                  * the actual bytes transferred
2788                  */
2789                 pipe->pid_toggle = 0;
2790                 cvmx_usb_complete(usb, pipe, transaction,
2791                                   CVMX_USB_STATUS_STALL);
2792         } else if (usbc_hcint.s.xacterr) {
2793                 /*
2794                  * XactErr as a response means the device signaled
2795                  * something wrong with the transfer. For example, PID
2796                  * toggle errors cause these.
2797                  */
2798                 cvmx_usb_complete(usb, pipe, transaction,
2799                                   CVMX_USB_STATUS_XACTERR);
2800         } else if (usbc_hcint.s.bblerr) {
2801                 /* Babble Error (BblErr) */
2802                 cvmx_usb_complete(usb, pipe, transaction,
2803                                   CVMX_USB_STATUS_BABBLEERR);
2804         } else if (usbc_hcint.s.datatglerr) {
2805                 /* Data toggle error */
2806                 cvmx_usb_complete(usb, pipe, transaction,
2807                                   CVMX_USB_STATUS_DATATGLERR);
2808         } else if (usbc_hcint.s.nyet) {
2809                 /*
2810                  * NYET as a response is only allowed in three cases: as a
2811                  * response to a ping, as a response to a split transaction, and
2812                  * as a response to a bulk out. The ping case is handled by
2813                  * hardware, so we only have splits and bulk out
2814                  */
2815                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
2816                         transaction->retries = 0;
2817                         /*
2818                          * If there is more data to go then we need to try
2819                          * again. Otherwise this transaction is complete
2820                          */
2821                         if ((buffer_space_left == 0) ||
2822                             (bytes_in_last_packet < pipe->max_packet))
2823                                 cvmx_usb_complete(usb, pipe,
2824                                                   transaction,
2825                                                   CVMX_USB_STATUS_OK);
2826                 } else {
2827                         /*
2828                          * Split transactions retry the split complete 4 times
2829                          * then rewind to the start split and do the entire
2830                          * transactions again
2831                          */
2832                         transaction->retries++;
2833                         if ((transaction->retries & 0x3) == 0) {
2834                                 /*
2835                                  * Rewind to the beginning of the transaction by
2836                                  * anding off the split complete bit
2837                                  */
2838                                 transaction->stage &= ~1;
2839                                 pipe->split_sc_frame = -1;
2840                         }
2841                 }
2842         } else if (usbc_hcint.s.ack) {
2843                 transaction->retries = 0;
2844                 /*
2845                  * The ACK bit can only be checked after the other error bits.
2846                  * This is because a multi packet transfer may succeed in a
2847                  * number of packets and then get a different response on the
2848                  * last packet. In this case both ACK and the last response bit
2849                  * will be set. If none of the other response bits is set, then
2850                  * the last packet must have been an ACK
2851                  *
2852                  * Since we got an ACK, we know we don't need to do a ping on
2853                  * this pipe
2854                  */
2855                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
2856
2857                 switch (transaction->type) {
2858                 case CVMX_USB_TRANSFER_CONTROL:
2859                         cvmx_usb_transfer_control(usb, pipe, transaction,
2860                                                   usbc_hcchar,
2861                                                   buffer_space_left,
2862                                                   bytes_in_last_packet);
2863                         break;
2864                 case CVMX_USB_TRANSFER_BULK:
2865                         cvmx_usb_transfer_bulk(usb, pipe, transaction,
2866                                                usbc_hcint, buffer_space_left,
2867                                                bytes_in_last_packet);
2868                         break;
2869                 case CVMX_USB_TRANSFER_INTERRUPT:
2870                         cvmx_usb_transfer_intr(usb, pipe, transaction,
2871                                                buffer_space_left,
2872                                                bytes_in_last_packet);
2873                         break;
2874                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2875                         cvmx_usb_transfer_isoc(usb, pipe, transaction,
2876                                                buffer_space_left,
2877                                                bytes_in_last_packet,
2878                                                bytes_this_transfer);
2879                         break;
2880                 }
2881         } else if (usbc_hcint.s.nak) {
2882                 /*
2883                  * If this was a split then clear our split in progress marker.
2884                  */
2885                 if (usb->active_split == transaction)
2886                         usb->active_split = NULL;
2887                 /*
2888                  * NAK as a response means the device couldn't accept the
2889                  * transaction, but it should be retried in the future. Rewind
2890                  * to the beginning of the transaction by anding off the split
2891                  * complete bit. Retry in the next interval
2892                  */
2893                 transaction->retries = 0;
2894                 transaction->stage &= ~1;
2895                 pipe->next_tx_frame += pipe->interval;
2896                 if (pipe->next_tx_frame < usb->frame_number)
2897                         pipe->next_tx_frame = usb->frame_number +
2898                                 pipe->interval -
2899                                 (usb->frame_number - pipe->next_tx_frame) %
2900                                 pipe->interval;
2901         } else {
2902                 struct cvmx_usb_port_status port;
2903
2904                 port = cvmx_usb_get_status(usb);
2905                 if (port.port_enabled) {
2906                         /* We'll retry the exact same transaction again */
2907                         transaction->retries++;
2908                 } else {
2909                         /*
2910                          * We get channel halted interrupts with no result bits
2911                          * sets when the cable is unplugged
2912                          */
2913                         cvmx_usb_complete(usb, pipe, transaction,
2914                                           CVMX_USB_STATUS_ERROR);
2915                 }
2916         }
2917         return 0;
2918 }
2919
2920 static void octeon_usb_port_callback(struct octeon_hcd *usb)
2921 {
2922         spin_unlock(&usb->lock);
2923         usb_hcd_poll_rh_status(octeon_to_hcd(usb));
2924         spin_lock(&usb->lock);
2925 }
2926
2927 /**
2928  * Poll the USB block for status and call all needed callback
2929  * handlers. This function is meant to be called in the interrupt
2930  * handler for the USB controller. It can also be called
2931  * periodically in a loop for non-interrupt based operation.
2932  *
2933  * @usb: USB device state populated by cvmx_usb_initialize().
2934  *
2935  * Returns: 0 or a negative error code.
2936  */
2937 static int cvmx_usb_poll(struct octeon_hcd *usb)
2938 {
2939         union cvmx_usbcx_hfnum usbc_hfnum;
2940         union cvmx_usbcx_gintsts usbc_gintsts;
2941
2942         prefetch_range(usb, sizeof(*usb));
2943
2944         /* Update the frame counter */
2945         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2946         if ((usb->frame_number & 0x3fff) > usbc_hfnum.s.frnum)
2947                 usb->frame_number += 0x4000;
2948         usb->frame_number &= ~0x3fffull;
2949         usb->frame_number |= usbc_hfnum.s.frnum;
2950
2951         /* Read the pending interrupts */
2952         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
2953                                                CVMX_USBCX_GINTSTS(usb->index));
2954
2955         /* Clear the interrupts now that we know about them */
2956         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
2957                              usbc_gintsts.u32);
2958
2959         if (usbc_gintsts.s.rxflvl) {
2960                 /*
2961                  * RxFIFO Non-Empty (RxFLvl)
2962                  * Indicates that there is at least one packet pending to be
2963                  * read from the RxFIFO.
2964                  *
2965                  * In DMA mode this is handled by hardware
2966                  */
2967                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2968                         cvmx_usb_poll_rx_fifo(usb);
2969         }
2970         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
2971                 /* Fill the Tx FIFOs when not in DMA mode */
2972                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2973                         cvmx_usb_poll_tx_fifo(usb);
2974         }
2975         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
2976                 union cvmx_usbcx_hprt usbc_hprt;
2977                 /*
2978                  * Disconnect Detected Interrupt (DisconnInt)
2979                  * Asserted when a device disconnect is detected.
2980                  *
2981                  * Host Port Interrupt (PrtInt)
2982                  * The core sets this bit to indicate a change in port status of
2983                  * one of the O2P USB core ports in Host mode. The application
2984                  * must read the Host Port Control and Status (HPRT) register to
2985                  * determine the exact event that caused this interrupt. The
2986                  * application must clear the appropriate status bit in the Host
2987                  * Port Control and Status register to clear this bit.
2988                  *
2989                  * Call the user's port callback
2990                  */
2991                 octeon_usb_port_callback(usb);
2992                 /* Clear the port change bits */
2993                 usbc_hprt.u32 =
2994                         cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
2995                 usbc_hprt.s.prtena = 0;
2996                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
2997                                      usbc_hprt.u32);
2998         }
2999         if (usbc_gintsts.s.hchint) {
3000                 /*
3001                  * Host Channels Interrupt (HChInt)
3002                  * The core sets this bit to indicate that an interrupt is
3003                  * pending on one of the channels of the core (in Host mode).
3004                  * The application must read the Host All Channels Interrupt
3005                  * (HAINT) register to determine the exact number of the channel
3006                  * on which the interrupt occurred, and then read the
3007                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3008                  * determine the exact cause of the interrupt. The application
3009                  * must clear the appropriate status bit in the HCINTn register
3010                  * to clear this bit.
3011                  */
3012                 union cvmx_usbcx_haint usbc_haint;
3013
3014                 usbc_haint.u32 = cvmx_usb_read_csr32(usb,
3015                                         CVMX_USBCX_HAINT(usb->index));
3016                 while (usbc_haint.u32) {
3017                         int channel;
3018
3019                         channel = __fls(usbc_haint.u32);
3020                         cvmx_usb_poll_channel(usb, channel);
3021                         usbc_haint.u32 ^= 1 << channel;
3022                 }
3023         }
3024
3025         cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3026
3027         return 0;
3028 }
3029
3030 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3031 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3032 {
3033         return (struct octeon_hcd *)(hcd->hcd_priv);
3034 }
3035
3036 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3037 {
3038         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3039         unsigned long flags;
3040
3041         spin_lock_irqsave(&usb->lock, flags);
3042         cvmx_usb_poll(usb);
3043         spin_unlock_irqrestore(&usb->lock, flags);
3044         return IRQ_HANDLED;
3045 }
3046
3047 static int octeon_usb_start(struct usb_hcd *hcd)
3048 {
3049         hcd->state = HC_STATE_RUNNING;
3050         return 0;
3051 }
3052
3053 static void octeon_usb_stop(struct usb_hcd *hcd)
3054 {
3055         hcd->state = HC_STATE_HALT;
3056 }
3057
3058 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3059 {
3060         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3061
3062         return cvmx_usb_get_frame_number(usb);
3063 }
3064
3065 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3066                                   struct urb *urb,
3067                                   gfp_t mem_flags)
3068 {
3069         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3070         struct device *dev = hcd->self.controller;
3071         struct cvmx_usb_transaction *transaction = NULL;
3072         struct cvmx_usb_pipe *pipe;
3073         unsigned long flags;
3074         struct cvmx_usb_iso_packet *iso_packet;
3075         struct usb_host_endpoint *ep = urb->ep;
3076         int rc;
3077
3078         urb->status = 0;
3079         spin_lock_irqsave(&usb->lock, flags);
3080
3081         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3082         if (rc) {
3083                 spin_unlock_irqrestore(&usb->lock, flags);
3084                 return rc;
3085         }
3086
3087         if (!ep->hcpriv) {
3088                 enum cvmx_usb_transfer transfer_type;
3089                 enum cvmx_usb_speed speed;
3090                 int split_device = 0;
3091                 int split_port = 0;
3092
3093                 switch (usb_pipetype(urb->pipe)) {
3094                 case PIPE_ISOCHRONOUS:
3095                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3096                         break;
3097                 case PIPE_INTERRUPT:
3098                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3099                         break;
3100                 case PIPE_CONTROL:
3101                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3102                         break;
3103                 default:
3104                         transfer_type = CVMX_USB_TRANSFER_BULK;
3105                         break;
3106                 }
3107                 switch (urb->dev->speed) {
3108                 case USB_SPEED_LOW:
3109                         speed = CVMX_USB_SPEED_LOW;
3110                         break;
3111                 case USB_SPEED_FULL:
3112                         speed = CVMX_USB_SPEED_FULL;
3113                         break;
3114                 default:
3115                         speed = CVMX_USB_SPEED_HIGH;
3116                         break;
3117                 }
3118                 /*
3119                  * For slow devices on high speed ports we need to find the hub
3120                  * that does the speed translation so we know where to send the
3121                  * split transactions.
3122                  */
3123                 if (speed != CVMX_USB_SPEED_HIGH) {
3124                         /*
3125                          * Start at this device and work our way up the usb
3126                          * tree.
3127                          */
3128                         struct usb_device *dev = urb->dev;
3129
3130                         while (dev->parent) {
3131                                 /*
3132                                  * If our parent is high speed then he'll
3133                                  * receive the splits.
3134                                  */
3135                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3136                                         split_device = dev->parent->devnum;
3137                                         split_port = dev->portnum;
3138                                         break;
3139                                 }
3140                                 /*
3141                                  * Move up the tree one level. If we make it all
3142                                  * the way up the tree, then the port must not
3143                                  * be in high speed mode and we don't need a
3144                                  * split.
3145                                  */
3146                                 dev = dev->parent;
3147                         }
3148                 }
3149                 pipe = cvmx_usb_open_pipe(usb, usb_pipedevice(urb->pipe),
3150                                           usb_pipeendpoint(urb->pipe), speed,
3151                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3152                                           & 0x7ff,
3153                                           transfer_type,
3154                                           usb_pipein(urb->pipe) ?
3155                                                 CVMX_USB_DIRECTION_IN :
3156                                                 CVMX_USB_DIRECTION_OUT,
3157                                           urb->interval,
3158                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3159                                            >> 11) & 0x3,
3160                                           split_device, split_port);
3161                 if (!pipe) {
3162                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3163                         spin_unlock_irqrestore(&usb->lock, flags);
3164                         dev_dbg(dev, "Failed to create pipe\n");
3165                         return -ENOMEM;
3166                 }
3167                 ep->hcpriv = pipe;
3168         } else {
3169                 pipe = ep->hcpriv;
3170         }
3171
3172         switch (usb_pipetype(urb->pipe)) {
3173         case PIPE_ISOCHRONOUS:
3174                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3175                         usb_pipedevice(urb->pipe),
3176                         usb_pipeendpoint(urb->pipe));
3177                 /*
3178                  * Allocate a structure to use for our private list of
3179                  * isochronous packets.
3180                  */
3181                 iso_packet = kmalloc_array(urb->number_of_packets,
3182                                            sizeof(struct cvmx_usb_iso_packet),
3183                                            GFP_ATOMIC);
3184                 if (iso_packet) {
3185                         int i;
3186                         /* Fill the list with the data from the URB */
3187                         for (i = 0; i < urb->number_of_packets; i++) {
3188                                 iso_packet[i].offset =
3189                                         urb->iso_frame_desc[i].offset;
3190                                 iso_packet[i].length =
3191                                         urb->iso_frame_desc[i].length;
3192                                 iso_packet[i].status = CVMX_USB_STATUS_ERROR;
3193                         }
3194                         /*
3195                          * Store a pointer to the list in the URB setup_packet
3196                          * field. We know this currently isn't being used and
3197                          * this saves us a bunch of logic.
3198                          */
3199                         urb->setup_packet = (char *)iso_packet;
3200                         transaction = cvmx_usb_submit_isochronous(usb,
3201                                                                   pipe, urb);
3202                         /*
3203                          * If submit failed we need to free our private packet
3204                          * list.
3205                          */
3206                         if (!transaction) {
3207                                 urb->setup_packet = NULL;
3208                                 kfree(iso_packet);
3209                         }
3210                 }
3211                 break;
3212         case PIPE_INTERRUPT:
3213                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3214                         usb_pipedevice(urb->pipe),
3215                         usb_pipeendpoint(urb->pipe));
3216                 transaction = cvmx_usb_submit_interrupt(usb, pipe, urb);
3217                 break;
3218         case PIPE_CONTROL:
3219                 dev_dbg(dev, "Submit control to %d.%d\n",
3220                         usb_pipedevice(urb->pipe),
3221                         usb_pipeendpoint(urb->pipe));
3222                 transaction = cvmx_usb_submit_control(usb, pipe, urb);
3223                 break;
3224         case PIPE_BULK:
3225                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3226                         usb_pipedevice(urb->pipe),
3227                         usb_pipeendpoint(urb->pipe));
3228                 transaction = cvmx_usb_submit_bulk(usb, pipe, urb);
3229                 break;
3230         }
3231         if (!transaction) {
3232                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3233                 spin_unlock_irqrestore(&usb->lock, flags);
3234                 dev_dbg(dev, "Failed to submit\n");
3235                 return -ENOMEM;
3236         }
3237         urb->hcpriv = transaction;
3238         spin_unlock_irqrestore(&usb->lock, flags);
3239         return 0;
3240 }
3241
3242 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3243                                   struct urb *urb,
3244                                   int status)
3245 {
3246         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3247         unsigned long flags;
3248         int rc;
3249
3250         if (!urb->dev)
3251                 return -EINVAL;
3252
3253         spin_lock_irqsave(&usb->lock, flags);
3254
3255         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3256         if (rc)
3257                 goto out;
3258
3259         urb->status = status;
3260         cvmx_usb_cancel(usb, urb->ep->hcpriv, urb->hcpriv);
3261
3262 out:
3263         spin_unlock_irqrestore(&usb->lock, flags);
3264
3265         return rc;
3266 }
3267
3268 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3269                                         struct usb_host_endpoint *ep)
3270 {
3271         struct device *dev = hcd->self.controller;
3272
3273         if (ep->hcpriv) {
3274                 struct octeon_hcd *usb = hcd_to_octeon(hcd);
3275                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3276                 unsigned long flags;
3277
3278                 spin_lock_irqsave(&usb->lock, flags);
3279                 cvmx_usb_cancel_all(usb, pipe);
3280                 if (cvmx_usb_close_pipe(usb, pipe))
3281                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3282                 spin_unlock_irqrestore(&usb->lock, flags);
3283                 ep->hcpriv = NULL;
3284         }
3285 }
3286
3287 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3288 {
3289         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3290         struct cvmx_usb_port_status port_status;
3291         unsigned long flags;
3292
3293         spin_lock_irqsave(&usb->lock, flags);
3294         port_status = cvmx_usb_get_status(usb);
3295         spin_unlock_irqrestore(&usb->lock, flags);
3296         buf[0] = port_status.connect_change << 1;
3297
3298         return buf[0] != 0;
3299 }
3300
3301 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3302                                   u16 wIndex, char *buf, u16 wLength)
3303 {
3304         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3305         struct device *dev = hcd->self.controller;
3306         struct cvmx_usb_port_status usb_port_status;
3307         int port_status;
3308         struct usb_hub_descriptor *desc;
3309         unsigned long flags;
3310
3311         switch (typeReq) {
3312         case ClearHubFeature:
3313                 dev_dbg(dev, "ClearHubFeature\n");
3314                 switch (wValue) {
3315                 case C_HUB_LOCAL_POWER:
3316                 case C_HUB_OVER_CURRENT:
3317                         /* Nothing required here */
3318                         break;
3319                 default:
3320                         return -EINVAL;
3321                 }
3322                 break;
3323         case ClearPortFeature:
3324                 dev_dbg(dev, "ClearPortFeature\n");
3325                 if (wIndex != 1) {
3326                         dev_dbg(dev, " INVALID\n");
3327                         return -EINVAL;
3328                 }
3329
3330                 switch (wValue) {
3331                 case USB_PORT_FEAT_ENABLE:
3332                         dev_dbg(dev, " ENABLE\n");
3333                         spin_lock_irqsave(&usb->lock, flags);
3334                         cvmx_usb_disable(usb);
3335                         spin_unlock_irqrestore(&usb->lock, flags);
3336                         break;
3337                 case USB_PORT_FEAT_SUSPEND:
3338                         dev_dbg(dev, " SUSPEND\n");
3339                         /* Not supported on Octeon */
3340                         break;
3341                 case USB_PORT_FEAT_POWER:
3342                         dev_dbg(dev, " POWER\n");
3343                         /* Not supported on Octeon */
3344                         break;
3345                 case USB_PORT_FEAT_INDICATOR:
3346                         dev_dbg(dev, " INDICATOR\n");
3347                         /* Port inidicator not supported */
3348                         break;
3349                 case USB_PORT_FEAT_C_CONNECTION:
3350                         dev_dbg(dev, " C_CONNECTION\n");
3351                         /* Clears drivers internal connect status change flag */
3352                         spin_lock_irqsave(&usb->lock, flags);
3353                         usb->port_status = cvmx_usb_get_status(usb);
3354                         spin_unlock_irqrestore(&usb->lock, flags);
3355                         break;
3356                 case USB_PORT_FEAT_C_RESET:
3357                         dev_dbg(dev, " C_RESET\n");
3358                         /*
3359                          * Clears the driver's internal Port Reset Change flag.
3360                          */
3361                         spin_lock_irqsave(&usb->lock, flags);
3362                         usb->port_status = cvmx_usb_get_status(usb);
3363                         spin_unlock_irqrestore(&usb->lock, flags);
3364                         break;
3365                 case USB_PORT_FEAT_C_ENABLE:
3366                         dev_dbg(dev, " C_ENABLE\n");
3367                         /*
3368                          * Clears the driver's internal Port Enable/Disable
3369                          * Change flag.
3370                          */
3371                         spin_lock_irqsave(&usb->lock, flags);
3372                         usb->port_status = cvmx_usb_get_status(usb);
3373                         spin_unlock_irqrestore(&usb->lock, flags);
3374                         break;
3375                 case USB_PORT_FEAT_C_SUSPEND:
3376                         dev_dbg(dev, " C_SUSPEND\n");
3377                         /*
3378                          * Clears the driver's internal Port Suspend Change
3379                          * flag, which is set when resume signaling on the host
3380                          * port is complete.
3381                          */
3382                         break;
3383                 case USB_PORT_FEAT_C_OVER_CURRENT:
3384                         dev_dbg(dev, " C_OVER_CURRENT\n");
3385                         /* Clears the driver's overcurrent Change flag */
3386                         spin_lock_irqsave(&usb->lock, flags);
3387                         usb->port_status = cvmx_usb_get_status(usb);
3388                         spin_unlock_irqrestore(&usb->lock, flags);
3389                         break;
3390                 default:
3391                         dev_dbg(dev, " UNKNOWN\n");
3392                         return -EINVAL;
3393                 }
3394                 break;
3395         case GetHubDescriptor:
3396                 dev_dbg(dev, "GetHubDescriptor\n");
3397                 desc = (struct usb_hub_descriptor *)buf;
3398                 desc->bDescLength = 9;
3399                 desc->bDescriptorType = 0x29;
3400                 desc->bNbrPorts = 1;
3401                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3402                 desc->bPwrOn2PwrGood = 1;
3403                 desc->bHubContrCurrent = 0;
3404                 desc->u.hs.DeviceRemovable[0] = 0;
3405                 desc->u.hs.DeviceRemovable[1] = 0xff;
3406                 break;
3407         case GetHubStatus:
3408                 dev_dbg(dev, "GetHubStatus\n");
3409                 *(__le32 *)buf = 0;
3410                 break;
3411         case GetPortStatus:
3412                 dev_dbg(dev, "GetPortStatus\n");
3413                 if (wIndex != 1) {
3414                         dev_dbg(dev, " INVALID\n");
3415                         return -EINVAL;
3416                 }
3417
3418                 spin_lock_irqsave(&usb->lock, flags);
3419                 usb_port_status = cvmx_usb_get_status(usb);
3420                 spin_unlock_irqrestore(&usb->lock, flags);
3421                 port_status = 0;
3422
3423                 if (usb_port_status.connect_change) {
3424                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3425                         dev_dbg(dev, " C_CONNECTION\n");
3426                 }
3427
3428                 if (usb_port_status.port_enabled) {
3429                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3430                         dev_dbg(dev, " C_ENABLE\n");
3431                 }
3432
3433                 if (usb_port_status.connected) {
3434                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3435                         dev_dbg(dev, " CONNECTION\n");
3436                 }
3437
3438                 if (usb_port_status.port_enabled) {
3439                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3440                         dev_dbg(dev, " ENABLE\n");
3441                 }
3442
3443                 if (usb_port_status.port_over_current) {
3444                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3445                         dev_dbg(dev, " OVER_CURRENT\n");
3446                 }
3447
3448                 if (usb_port_status.port_powered) {
3449                         port_status |= (1 << USB_PORT_FEAT_POWER);
3450                         dev_dbg(dev, " POWER\n");
3451                 }
3452
3453                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3454                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3455                         dev_dbg(dev, " HIGHSPEED\n");
3456                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3457                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3458                         dev_dbg(dev, " LOWSPEED\n");
3459                 }
3460
3461                 *((__le32 *)buf) = cpu_to_le32(port_status);
3462                 break;
3463         case SetHubFeature:
3464                 dev_dbg(dev, "SetHubFeature\n");
3465                 /* No HUB features supported */
3466                 break;
3467         case SetPortFeature:
3468                 dev_dbg(dev, "SetPortFeature\n");
3469                 if (wIndex != 1) {
3470                         dev_dbg(dev, " INVALID\n");
3471                         return -EINVAL;
3472                 }
3473
3474                 switch (wValue) {
3475                 case USB_PORT_FEAT_SUSPEND:
3476                         dev_dbg(dev, " SUSPEND\n");
3477                         return -EINVAL;
3478                 case USB_PORT_FEAT_POWER:
3479                         dev_dbg(dev, " POWER\n");
3480                         /*
3481                          * Program the port power bit to drive VBUS on the USB.
3482                          */
3483                         spin_lock_irqsave(&usb->lock, flags);
3484                         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
3485                                         cvmx_usbcx_hprt, prtpwr, 1);
3486                         spin_unlock_irqrestore(&usb->lock, flags);
3487                         return 0;
3488                 case USB_PORT_FEAT_RESET:
3489                         dev_dbg(dev, " RESET\n");
3490                         spin_lock_irqsave(&usb->lock, flags);
3491                         cvmx_usb_reset_port(usb);
3492                         spin_unlock_irqrestore(&usb->lock, flags);
3493                         return 0;
3494                 case USB_PORT_FEAT_INDICATOR:
3495                         dev_dbg(dev, " INDICATOR\n");
3496                         /* Not supported */
3497                         break;
3498                 default:
3499                         dev_dbg(dev, " UNKNOWN\n");
3500                         return -EINVAL;
3501                 }
3502                 break;
3503         default:
3504                 dev_dbg(dev, "Unknown root hub request\n");
3505                 return -EINVAL;
3506         }
3507         return 0;
3508 }
3509
3510 static const struct hc_driver octeon_hc_driver = {
3511         .description            = "Octeon USB",
3512         .product_desc           = "Octeon Host Controller",
3513         .hcd_priv_size          = sizeof(struct octeon_hcd),
3514         .irq                    = octeon_usb_irq,
3515         .flags                  = HCD_MEMORY | HCD_USB2,
3516         .start                  = octeon_usb_start,
3517         .stop                   = octeon_usb_stop,
3518         .urb_enqueue            = octeon_usb_urb_enqueue,
3519         .urb_dequeue            = octeon_usb_urb_dequeue,
3520         .endpoint_disable       = octeon_usb_endpoint_disable,
3521         .get_frame_number       = octeon_usb_get_frame_number,
3522         .hub_status_data        = octeon_usb_hub_status_data,
3523         .hub_control            = octeon_usb_hub_control,
3524         .map_urb_for_dma        = octeon_map_urb_for_dma,
3525         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3526 };
3527
3528 static int octeon_usb_probe(struct platform_device *pdev)
3529 {
3530         int status;
3531         int initialize_flags;
3532         int usb_num;
3533         struct resource *res_mem;
3534         struct device_node *usbn_node;
3535         int irq = platform_get_irq(pdev, 0);
3536         struct device *dev = &pdev->dev;
3537         struct octeon_hcd *usb;
3538         struct usb_hcd *hcd;
3539         u32 clock_rate = 48000000;
3540         bool is_crystal_clock = false;
3541         const char *clock_type;
3542         int i;
3543
3544         if (!dev->of_node) {
3545                 dev_err(dev, "Error: empty of_node\n");
3546                 return -ENXIO;
3547         }
3548         usbn_node = dev->of_node->parent;
3549
3550         i = of_property_read_u32(usbn_node,
3551                                  "clock-frequency", &clock_rate);
3552         if (i)
3553                 i = of_property_read_u32(usbn_node,
3554                                          "refclk-frequency", &clock_rate);
3555         if (i) {
3556                 dev_err(dev, "No USBN \"clock-frequency\"\n");
3557                 return -ENXIO;
3558         }
3559         switch (clock_rate) {
3560         case 12000000:
3561                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3562                 break;
3563         case 24000000:
3564                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3565                 break;
3566         case 48000000:
3567                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3568                 break;
3569         default:
3570                 dev_err(dev, "Illegal USBN \"clock-frequency\" %u\n",
3571                         clock_rate);
3572                 return -ENXIO;
3573         }
3574
3575         i = of_property_read_string(usbn_node,
3576                                     "cavium,refclk-type", &clock_type);
3577         if (i)
3578                 i = of_property_read_string(usbn_node,
3579                                             "refclk-type", &clock_type);
3580
3581         if (!i && strcmp("crystal", clock_type) == 0)
3582                 is_crystal_clock = true;
3583
3584         if (is_crystal_clock)
3585                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3586         else
3587                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3588
3589         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3590         if (!res_mem) {
3591                 dev_err(dev, "found no memory resource\n");
3592                 return -ENXIO;
3593         }
3594         usb_num = (res_mem->start >> 44) & 1;
3595
3596         if (irq < 0) {
3597                 /* Defective device tree, but we know how to fix it. */
3598                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3599
3600                 irq = irq_create_mapping(NULL, hwirq);
3601         }
3602
3603         /*
3604          * Set the DMA mask to 64bits so we get buffers already translated for
3605          * DMA.
3606          */
3607         dev->coherent_dma_mask = ~0;
3608         dev->dma_mask = &dev->coherent_dma_mask;
3609
3610         /*
3611          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3612          * IOB priority registers.  Under heavy network load USB
3613          * hardware can be starved by the IOB causing a crash.  Give
3614          * it a priority boost if it has been waiting more than 400
3615          * cycles to avoid this situation.
3616          *
3617          * Testing indicates that a cnt_val of 8192 is not sufficient,
3618          * but no failures are seen with 4096.  We choose a value of
3619          * 400 to give a safety factor of 10.
3620          */
3621         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3622                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3623
3624                 pri_cnt.u64 = 0;
3625                 pri_cnt.s.cnt_enb = 1;
3626                 pri_cnt.s.cnt_val = 400;
3627                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3628         }
3629
3630         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3631         if (!hcd) {
3632                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3633                 return -1;
3634         }
3635         hcd->uses_new_polling = 1;
3636         usb = (struct octeon_hcd *)hcd->hcd_priv;
3637
3638         spin_lock_init(&usb->lock);
3639
3640         usb->init_flags = initialize_flags;
3641
3642         /* Initialize the USB state structure */
3643         usb->index = usb_num;
3644         INIT_LIST_HEAD(&usb->idle_pipes);
3645         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
3646                 INIT_LIST_HEAD(&usb->active_pipes[i]);
3647
3648         /* Due to an errata, CN31XX doesn't support DMA */
3649         if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
3650                 usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
3651                 /* Only use one channel with non DMA */
3652                 usb->idle_hardware_channels = 0x1;
3653         } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
3654                 /* CN5XXX have an errata with channel 3 */
3655                 usb->idle_hardware_channels = 0xf7;
3656         } else {
3657                 usb->idle_hardware_channels = 0xff;
3658         }
3659
3660         status = cvmx_usb_initialize(dev, usb);
3661         if (status) {
3662                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3663                 usb_put_hcd(hcd);
3664                 return -1;
3665         }
3666
3667         status = usb_add_hcd(hcd, irq, 0);
3668         if (status) {
3669                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3670                 usb_put_hcd(hcd);
3671                 return -1;
3672         }
3673         device_wakeup_enable(hcd->self.controller);
3674
3675         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3676
3677         return 0;
3678 }
3679
3680 static int octeon_usb_remove(struct platform_device *pdev)
3681 {
3682         int status;
3683         struct device *dev = &pdev->dev;
3684         struct usb_hcd *hcd = dev_get_drvdata(dev);
3685         struct octeon_hcd *usb = hcd_to_octeon(hcd);
3686         unsigned long flags;
3687
3688         usb_remove_hcd(hcd);
3689         spin_lock_irqsave(&usb->lock, flags);
3690         status = cvmx_usb_shutdown(usb);
3691         spin_unlock_irqrestore(&usb->lock, flags);
3692         if (status)
3693                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3694
3695         usb_put_hcd(hcd);
3696
3697         return 0;
3698 }
3699
3700 static const struct of_device_id octeon_usb_match[] = {
3701         {
3702                 .compatible = "cavium,octeon-5750-usbc",
3703         },
3704         {},
3705 };
3706 MODULE_DEVICE_TABLE(of, octeon_usb_match);
3707
3708 static struct platform_driver octeon_usb_driver = {
3709         .driver = {
3710                 .name           = "octeon-hcd",
3711                 .of_match_table = octeon_usb_match,
3712         },
3713         .probe      = octeon_usb_probe,
3714         .remove     = octeon_usb_remove,
3715 };
3716
3717 static int __init octeon_usb_driver_init(void)
3718 {
3719         if (usb_disabled())
3720                 return 0;
3721
3722         return platform_driver_register(&octeon_usb_driver);
3723 }
3724 module_init(octeon_usb_driver_init);
3725
3726 static void __exit octeon_usb_driver_exit(void)
3727 {
3728         if (usb_disabled())
3729                 return;
3730
3731         platform_driver_unregister(&octeon_usb_driver);
3732 }
3733 module_exit(octeon_usb_driver_exit);
3734
3735 MODULE_LICENSE("GPL");
3736 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3737 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");