drivers/bluetooth/hci_bcm4377.c

   1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
   2 /*
   3  * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
   4  *
   5  * Copyright (C) The Asahi Linux Contributors
   6  */
   7
   8 #include <linux/async.h>
   9 #include <linux/bitfield.h>
  10 #include <linux/completion.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/dmi.h>
  13 #include <linux/firmware.h>
  14 #include <linux/module.h>
  15 #include <linux/msi.h>
  16 #include <linux/of.h>
  17 #include <linux/pci.h>
  18 #include <linux/printk.h>
  19
  20 #include <asm/unaligned.h>
  21
  22 #include <net/bluetooth/bluetooth.h>
  23 #include <net/bluetooth/hci_core.h>
  24
  25 enum bcm4377_chip {
  26         BCM4377 = 0,
  27         BCM4378,
  28         BCM4387,
  29 };
  30
  31 #define BCM4377_DEVICE_ID 0x5fa0
  32 #define BCM4378_DEVICE_ID 0x5f69
  33 #define BCM4387_DEVICE_ID 0x5f71
  34
  35 #define BCM4377_TIMEOUT 1000
  36
  37 /*
  38  * These devices only support DMA transactions inside a 32bit window
  39  * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
  40  * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
  41  * which effectively limits the window to [start, start+0xfffffe00].
  42  * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
  43  * run into this limitation.
  44  */
  45 #define BCM4377_DMA_MASK 0xfffffe00
  46
  47 #define BCM4377_PCIECFG_BAR0_WINDOW1       0x80
  48 #define BCM4377_PCIECFG_BAR0_WINDOW2       0x70
  49 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
  50 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
  51 #define BCM4377_PCIECFG_BAR2_WINDOW        0x84
  52
  53 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
  54 #define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT        0x19000000
  55
  56 #define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
  57
  58 #define BCM4377_BAR0_FW_DOORBELL 0x140
  59 #define BCM4377_BAR0_RTI_CONTROL 0x144
  60
  61 #define BCM4377_BAR0_SLEEP_CONTROL            0x150
  62 #define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE  0
  63 #define BCM4377_BAR0_SLEEP_CONTROL_AWAKE      2
  64 #define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE    3
  65
  66 #define BCM4377_BAR0_DOORBELL       0x174
  67 #define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
  68 #define BCM4377_BAR0_DOORBELL_IDX   GENMASK(15, 8)
  69 #define BCM4377_BAR0_DOORBELL_RING  BIT(5)
  70
  71 #define BCM4377_BAR0_HOST_WINDOW_LO   0x590
  72 #define BCM4377_BAR0_HOST_WINDOW_HI   0x594
  73 #define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
  74
  75 #define BCM4377_BAR2_BOOTSTAGE 0x200454
  76
  77 #define BCM4377_BAR2_FW_LO   0x200478
  78 #define BCM4377_BAR2_FW_HI   0x20047c
  79 #define BCM4377_BAR2_FW_SIZE 0x200480
  80
  81 #define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
  82 #define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
  83
  84 #define BCM4377_BAR2_RTI_STATUS      0x20045c
  85 #define BCM4377_BAR2_RTI_WINDOW_LO   0x200494
  86 #define BCM4377_BAR2_RTI_WINDOW_HI   0x200498
  87 #define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
  88
  89 #define BCM4377_OTP_SIZE          0xe0
  90 #define BCM4377_OTP_SYS_VENDOR    0x15
  91 #define BCM4377_OTP_CIS           0x80
  92 #define BCM4377_OTP_VENDOR_HDR    0x00000008
  93 #define BCM4377_OTP_MAX_PARAM_LEN 16
  94
  95 #define BCM4377_N_TRANSFER_RINGS   9
  96 #define BCM4377_N_COMPLETION_RINGS 6
  97
  98 #define BCM4377_MAX_RING_SIZE 256
  99
 100 #define BCM4377_MSGID_GENERATION GENMASK(15, 8)
 101 #define BCM4377_MSGID_ID         GENMASK(7, 0)
 102
 103 #define BCM4377_RING_N_ENTRIES 128
 104
 105 #define BCM4377_CONTROL_MSG_SIZE                   0x34
 106 #define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
 107
 108 #define MAX_ACL_PAYLOAD_SIZE   (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
 109 #define MAX_SCO_PAYLOAD_SIZE   (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
 110 #define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
 111
 112 enum bcm4377_otp_params_type {
 113         BCM4377_OTP_BOARD_PARAMS,
 114         BCM4377_OTP_CHIP_PARAMS
 115 };
 116
 117 enum bcm4377_transfer_ring_id {
 118         BCM4377_XFER_RING_CONTROL = 0,
 119         BCM4377_XFER_RING_HCI_H2D = 1,
 120         BCM4377_XFER_RING_HCI_D2H = 2,
 121         BCM4377_XFER_RING_SCO_H2D = 3,
 122         BCM4377_XFER_RING_SCO_D2H = 4,
 123         BCM4377_XFER_RING_ACL_H2D = 5,
 124         BCM4377_XFER_RING_ACL_D2H = 6,
 125 };
 126
 127 enum bcm4377_completion_ring_id {
 128         BCM4377_ACK_RING_CONTROL = 0,
 129         BCM4377_ACK_RING_HCI_ACL = 1,
 130         BCM4377_EVENT_RING_HCI_ACL = 2,
 131         BCM4377_ACK_RING_SCO = 3,
 132         BCM4377_EVENT_RING_SCO = 4,
 133 };
 134
 135 enum bcm4377_doorbell {
 136         BCM4377_DOORBELL_CONTROL = 0,
 137         BCM4377_DOORBELL_HCI_H2D = 1,
 138         BCM4377_DOORBELL_HCI_D2H = 2,
 139         BCM4377_DOORBELL_ACL_H2D = 3,
 140         BCM4377_DOORBELL_ACL_D2H = 4,
 141         BCM4377_DOORBELL_SCO = 6,
 142 };
 143
 144 /*
 145  * Transfer ring entry
 146  *
 147  * flags: Flags to indicate if the payload is appended or mapped
 148  * len: Payload length
 149  * payload: Optional payload DMA address
 150  * id: Message id to recognize the answer in the completion ring entry
 151  */
 152 struct bcm4377_xfer_ring_entry {
 153 #define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED    BIT(0)
 154 #define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
 155         u8 flags;
 156         __le16 len;
 157         u8 _unk0;
 158         __le64 payload;
 159         __le16 id;
 160         u8 _unk1[2];
 161 } __packed;
 162 static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
 163
 164 /*
 165  * Completion ring entry
 166  *
 167  * flags: Flags to indicate if the payload is appended or mapped. If the payload
 168  *        is mapped it can be found in the buffer of the corresponding transfer
 169  *        ring message.
 170  * ring_id: Transfer ring ID which required this message
 171  * msg_id: Message ID specified in transfer ring entry
 172  * len: Payload length
 173  */
 174 struct bcm4377_completion_ring_entry {
 175         u8 flags;
 176         u8 _unk0;
 177         __le16 ring_id;
 178         __le16 msg_id;
 179         __le32 len;
 180         u8 _unk1[6];
 181 } __packed;
 182 static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
 183
 184 enum bcm4377_control_message_type {
 185         BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
 186         BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
 187         BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
 188         BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
 189 };
 190
 191 /*
 192  * Control message used to create a completion ring
 193  *
 194  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
 195  * header_size: Unknown, but probably reserved space in front of the entry
 196  * footer_size: Number of 32 bit words reserved for payloads after the entry
 197  * id/id_again: Completion ring index
 198  * ring_iova: DMA address of the ring buffer
 199  * n_elements: Number of elements inside the ring buffer
 200  * msi: MSI index, doesn't work for all rings though and should be zero
 201  * intmod_delay: Unknown delay
 202  * intmod_bytes: Unknown
 203  */
 204 struct bcm4377_create_completion_ring_msg {
 205         u8 msg_type;
 206         u8 header_size;
 207         u8 footer_size;
 208         u8 _unk0;
 209         __le16 id;
 210         __le16 id_again;
 211         __le64 ring_iova;
 212         __le16 n_elements;
 213         __le32 unk;
 214         u8 _unk1[6];
 215         __le16 msi;
 216         __le16 intmod_delay;
 217         __le32 intmod_bytes;
 218         __le16 _unk2;
 219         __le32 _unk3;
 220         u8 _unk4[10];
 221 } __packed;
 222 static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
 223               BCM4377_CONTROL_MSG_SIZE);
 224
 225 /*
 226  * Control ring message used to destroy a completion ring
 227  *
 228  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
 229  * ring_id: Completion ring to be destroyed
 230  */
 231 struct bcm4377_destroy_completion_ring_msg {
 232         u8 msg_type;
 233         u8 _pad0;
 234         __le16 ring_id;
 235         u8 _pad1[48];
 236 } __packed;
 237 static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
 238               BCM4377_CONTROL_MSG_SIZE);
 239
 240 /*
 241  * Control message used to create a transfer ring
 242  *
 243  * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
 244  * header_size: Number of 32 bit words reserved for unknown content before the
 245  *              entry
 246  * footer_size: Number of 32 bit words reserved for payloads after the entry
 247  * ring_id/ring_id_again: Transfer ring index
 248  * ring_iova: DMA address of the ring buffer
 249  * n_elements: Number of elements inside the ring buffer
 250  * completion_ring_id: Completion ring index for acknowledgements and events
 251  * doorbell: Doorbell index used to notify device of new entries
 252  * flags: Transfer ring flags
 253  *          - virtual: set if there is no associated shared memory and only the
 254  *                     corresponding completion ring is used
 255  *          - sync: only set for the SCO rings
 256  */
 257 struct bcm4377_create_transfer_ring_msg {
 258         u8 msg_type;
 259         u8 header_size;
 260         u8 footer_size;
 261         u8 _unk0;
 262         __le16 ring_id;
 263         __le16 ring_id_again;
 264         __le64 ring_iova;
 265         u8 _unk1[8];
 266         __le16 n_elements;
 267         __le16 completion_ring_id;
 268         __le16 doorbell;
 269 #define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
 270 #define BCM4377_XFER_RING_FLAG_SYNC    BIT(8)
 271         __le16 flags;
 272         u8 _unk2[20];
 273 } __packed;
 274 static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
 275               BCM4377_CONTROL_MSG_SIZE);
 276
 277 /*
 278  * Control ring message used to destroy a transfer ring
 279  *
 280  * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
 281  * ring_id: Transfer ring to be destroyed
 282  */
 283 struct bcm4377_destroy_transfer_ring_msg {
 284         u8 msg_type;
 285         u8 _pad0;
 286         __le16 ring_id;
 287         u8 _pad1[48];
 288 } __packed;
 289 static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
 290               BCM4377_CONTROL_MSG_SIZE);
 291
 292 /*
 293  * "Converged IPC" context struct used to make the device aware of all other
 294  * shared memory structures. A pointer to this structure is configured inside a
 295  * MMIO register.
 296  *
 297  * version: Protocol version, must be 2.
 298  * size: Size of this structure, must be 0x68.
 299  * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
 300  * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
 301  *                       write unknown contents
 302  * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
 303  * n_completion_rings: Number of completion rings, the firmware only works if
 304  *                     this is set to BCM4377_N_COMPLETION_RINGS.
 305  * n_xfer_rings: Number of transfer rings, the firmware only works if
 306  *               this is set to BCM4377_N_TRANSFER_RINGS.
 307  * control_completion_ring_addr: Control completion ring buffer DMA address
 308  * control_xfer_ring_addr: Control transfer ring buffer DMA address
 309  * control_xfer_ring_n_entries: Number of control transfer ring entries
 310  * control_completion_ring_n_entries: Number of control completion ring entries
 311  * control_xfer_ring_doorbell: Control transfer ring doorbell
 312  * control_completion_ring_doorbell: Control completion ring doorbell,
 313  *                                   must be set to 0xffff
 314  * control_xfer_ring_msi: Control completion ring MSI index, must be 0
 315  * control_completion_ring_msi: Control completion ring MSI index, must be 0.
 316  * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
 317  *                                every control transfer ring entry
 318  * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
 319  *                                control transfer ring entry
 320  * control_completion_ring_header_size: Number of 32 bit words reserved in front
 321  *                                      of every control completion ring entry
 322  * control_completion_ring_footer_size: Number of 32 bit words reserved after
 323  *                                      every control completion ring entry
 324  * scratch_pad: Optional scratch pad DMA address
 325  * scratch_pad_size: Scratch pad size
 326  */
 327 struct bcm4377_context {
 328         __le16 version;
 329         __le16 size;
 330         __le32 enabled_caps;
 331
 332         __le64 peripheral_info_addr;
 333
 334         /* ring heads and tails */
 335         __le64 completion_ring_heads_addr;
 336         __le64 xfer_ring_tails_addr;
 337         __le64 completion_ring_tails_addr;
 338         __le64 xfer_ring_heads_addr;
 339         __le16 n_completion_rings;
 340         __le16 n_xfer_rings;
 341
 342         /* control ring configuration */
 343         __le64 control_completion_ring_addr;
 344         __le64 control_xfer_ring_addr;
 345         __le16 control_xfer_ring_n_entries;
 346         __le16 control_completion_ring_n_entries;
 347         __le16 control_xfer_ring_doorbell;
 348         __le16 control_completion_ring_doorbell;
 349         __le16 control_xfer_ring_msi;
 350         __le16 control_completion_ring_msi;
 351         u8 control_xfer_ring_header_size;
 352         u8 control_xfer_ring_footer_size;
 353         u8 control_completion_ring_header_size;
 354         u8 control_completion_ring_footer_size;
 355
 356         __le16 _unk0;
 357         __le16 _unk1;
 358
 359         __le64 scratch_pad;
 360         __le32 scratch_pad_size;
 361
 362         __le32 _unk3;
 363 } __packed;
 364 static_assert(sizeof(struct bcm4377_context) == 0x68);
 365
 366 #define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
 367 struct bcm4378_hci_send_calibration_cmd {
 368         u8 unk;
 369         __le16 blocks_left;
 370         u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
 371 } __packed;
 372
 373 #define BCM4378_PTB_CHUNK_SIZE 0xcf
 374 struct bcm4378_hci_send_ptb_cmd {
 375         __le16 blocks_left;
 376         u8 data[BCM4378_PTB_CHUNK_SIZE];
 377 } __packed;
 378
 379 /*
 380  * Shared memory structure used to store the ring head and tail pointers.
 381  */
 382 struct bcm4377_ring_state {
 383         __le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
 384         __le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
 385         __le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
 386         __le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
 387 };
 388
 389 /*
 390  * A transfer ring can be used in two configurations:
 391  *  1) Send control or HCI messages to the device which are then acknowledged
 392  *     in the corresponding completion ring
 393  *  2) Receiving HCI frames from the devices. In this case the transfer ring
 394  *     itself contains empty messages that are acknowledged once data is
 395  *     available from the device. If the payloads fit inside the footers
 396  *     of the completion ring the transfer ring can be configured to be
 397  *     virtual such that it has no ring buffer.
 398  *
 399  * ring_id: ring index hardcoded in the firmware
 400  * doorbell: doorbell index to notify device of new entries
 401  * payload_size: optional in-place payload size
 402  * mapped_payload_size: optional out-of-place payload size
 403  * completion_ring: index of corresponding completion ring
 404  * n_entries: number of entries inside this ring
 405  * generation: ring generation; incremented on hci_open to detect stale messages
 406  * sync: set to true for SCO rings
 407  * virtual: set to true if this ring has no entries and is just required to
 408  *          setup a corresponding completion ring for device->host messages
 409  * d2h_buffers_only: set to true if this ring is only used to provide large
 410  *                   buffers used by device->host messages in the completion
 411  *                   ring
 412  * allow_wait: allow to wait for messages to be acknowledged
 413  * enabled: true once the ring has been created and can be used
 414  * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
 415  * ring_dma: DMA address for ring entry buffer
 416  * payloads: payload buffer for mapped_payload_size payloads
 417  * payloads_dma:DMA address for payload buffer
 418  * events: pointer to array of completions if waiting is allowed
 419  * msgids: bitmap to keep track of used message ids
 420  * lock: Spinlock to protect access to ring structurs used in the irq handler
 421  */
 422 struct bcm4377_transfer_ring {
 423         enum bcm4377_transfer_ring_id ring_id;
 424         enum bcm4377_doorbell doorbell;
 425         size_t payload_size;
 426         size_t mapped_payload_size;
 427         u8 completion_ring;
 428         u16 n_entries;
 429         u8 generation;
 430
 431         bool sync;
 432         bool virtual;
 433         bool d2h_buffers_only;
 434         bool allow_wait;
 435         bool enabled;
 436
 437         void *ring;
 438         dma_addr_t ring_dma;
 439
 440         void *payloads;
 441         dma_addr_t payloads_dma;
 442
 443         struct completion **events;
 444         DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
 445         spinlock_t lock;
 446 };
 447
 448 /*
 449  * A completion ring can be either used to either acknowledge messages sent in
 450  * the corresponding transfer ring or to receive messages associated with the
 451  * transfer ring. When used to receive messages the transfer ring either
 452  * has no ring buffer and is only advanced ("virtual transfer ring") or it
 453  * only contains empty DMA buffers to be used for the payloads.
 454  *
 455  * ring_id: completion ring id, hardcoded in firmware
 456  * payload_size: optional payload size after each entry
 457  * delay: unknown delay
 458  * n_entries: number of entries in this ring
 459  * enabled: true once the ring has been created and can be used
 460  * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
 461  * ring_dma: DMA address of ring buffer
 462  * transfer_rings: bitmap of corresponding transfer ring ids
 463  */
 464 struct bcm4377_completion_ring {
 465         enum bcm4377_completion_ring_id ring_id;
 466         u16 payload_size;
 467         u16 delay;
 468         u16 n_entries;
 469         bool enabled;
 470
 471         void *ring;
 472         dma_addr_t ring_dma;
 473
 474         unsigned long transfer_rings;
 475 };
 476
 477 struct bcm4377_data;
 478
 479 /*
 480  * Chip-specific configuration struct
 481  *
 482  * id: Chip id (e.g. 0x4377 for BCM4377)
 483  * otp_offset: Offset to the start of the OTP inside BAR0
 484  * bar0_window1: Backplane address mapped to the first window in BAR0
 485  * bar0_window2: Backplane address mapped to the second window in BAR0
 486  * bar0_core2_window2: Optional backplane address mapped to the second core's
 487  *                     second window in BAR0
 488  * has_bar0_core2_window2: Set to true if this chip requires the second core's
 489  *                         second window to be configured
 490  * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
 491  *                                     vendor-specific subsystem control
 492  *                                     register has to be cleared
 493  * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
 494  * broken_ext_scan: Set to true if the chip erroneously claims to support
 495  *                  extended scanning
 496  * broken_mws_transport_config: Set to true if the chip erroneously claims to
 497  *                              support MWS Transport Configuration
 498  * send_calibration: Optional callback to send calibration data
 499  * send_ptb: Callback to send "PTB" regulatory/calibration data
 500  */
 501 struct bcm4377_hw {
 502         unsigned int id;
 503
 504         u32 otp_offset;
 505
 506         u32 bar0_window1;
 507         u32 bar0_window2;
 508         u32 bar0_core2_window2;
 509
 510         unsigned long has_bar0_core2_window2 : 1;
 511         unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
 512         unsigned long disable_aspm : 1;
 513         unsigned long broken_ext_scan : 1;
 514         unsigned long broken_mws_transport_config : 1;
 515         unsigned long broken_le_coded : 1;
 516
 517         int (*send_calibration)(struct bcm4377_data *bcm4377);
 518         int (*send_ptb)(struct bcm4377_data *bcm4377,
 519                         const struct firmware *fw);
 520 };
 521
 522 static const struct bcm4377_hw bcm4377_hw_variants[];
 523 static const struct dmi_system_id bcm4377_dmi_board_table[];
 524
 525 /*
 526  * Private struct associated with each device containing global state
 527  *
 528  * pdev: Pointer to associated struct pci_dev
 529  * hdev: Pointer to associated strucy hci_dev
 530  * bar0: iomem pointing to BAR0
 531  * bar1: iomem pointing to BAR2
 532  * bootstage: Current value of the bootstage
 533  * rti_status: Current "RTI" status value
 534  * hw: Pointer to chip-specific struct bcm4377_hw
 535  * taurus_cal_blob: "Taurus" calibration blob used for some chips
 536  * taurus_cal_size: "Taurus" calibration blob size
 537  * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
 538  *                              some chips
 539  * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
 540  * stepping: Chip stepping read from OTP; used for firmware selection
 541  * vendor: Antenna vendor read from OTP; used for firmware selection
 542  * board_type: Board type from FDT or DMI match; used for firmware selection
 543  * event: Event for changed bootstage or rti_status; used for booting firmware
 544  * ctx: "Converged IPC" context
 545  * ctx_dma: "Converged IPC" context DMA address
 546  * ring_state: Shared memory buffer containing ring head and tail indexes
 547  * ring_state_dma: DMA address for ring_state
 548  * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
 549  * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
 550  * control_h2d_ring: Transfer ring used for control messages
 551  * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
 552  * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
 553  *                         corresponding completion ring
 554  */
 555 struct bcm4377_data {
 556         struct pci_dev *pdev;
 557         struct hci_dev *hdev;
 558
 559         void __iomem *bar0;
 560         void __iomem *bar2;
 561
 562         u32 bootstage;
 563         u32 rti_status;
 564
 565         const struct bcm4377_hw *hw;
 566
 567         const void *taurus_cal_blob;
 568         int taurus_cal_size;
 569         const void *taurus_beamforming_cal_blob;
 570         int taurus_beamforming_cal_size;
 571
 572         char stepping[BCM4377_OTP_MAX_PARAM_LEN];
 573         char vendor[BCM4377_OTP_MAX_PARAM_LEN];
 574         const char *board_type;
 575
 576         struct completion event;
 577
 578         struct bcm4377_context *ctx;
 579         dma_addr_t ctx_dma;
 580
 581         struct bcm4377_ring_state *ring_state;
 582         dma_addr_t ring_state_dma;
 583
 584         /*
 585          * The HCI and ACL rings have to be merged because this structure is
 586          * hardcoded in the firmware.
 587          */
 588         struct bcm4377_completion_ring control_ack_ring;
 589         struct bcm4377_completion_ring hci_acl_ack_ring;
 590         struct bcm4377_completion_ring hci_acl_event_ring;
 591         struct bcm4377_completion_ring sco_ack_ring;
 592         struct bcm4377_completion_ring sco_event_ring;
 593
 594         struct bcm4377_transfer_ring control_h2d_ring;
 595         struct bcm4377_transfer_ring hci_h2d_ring;
 596         struct bcm4377_transfer_ring hci_d2h_ring;
 597         struct bcm4377_transfer_ring sco_h2d_ring;
 598         struct bcm4377_transfer_ring sco_d2h_ring;
 599         struct bcm4377_transfer_ring acl_h2d_ring;
 600         struct bcm4377_transfer_ring acl_d2h_ring;
 601 };
 602
 603 static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
 604                                   u16 val)
 605 {
 606         u32 db = 0;
 607
 608         db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
 609         db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
 610         db |= BCM4377_BAR0_DOORBELL_RING;
 611
 612         dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
 613                 doorbell, db);
 614         iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
 615 }
 616
 617 static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
 618                                  struct bcm4377_transfer_ring *ring,
 619                                  u16 raw_msgid, u8 *msgid)
 620 {
 621         u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
 622         *msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
 623
 624         if (generation != ring->generation) {
 625                 dev_warn(
 626                         &bcm4377->pdev->dev,
 627                         "invalid message generation %d should be %d in entry for ring %d\n",
 628                         generation, ring->generation, ring->ring_id);
 629                 return -EINVAL;
 630         }
 631
 632         if (*msgid >= ring->n_entries) {
 633                 dev_warn(&bcm4377->pdev->dev,
 634                          "invalid message id in entry for ring %d: %d > %d\n",
 635                          ring->ring_id, *msgid, ring->n_entries);
 636                 return -EINVAL;
 637         }
 638
 639         return 0;
 640 }
 641
 642 static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
 643                                  struct bcm4377_transfer_ring *ring,
 644                                  u16 raw_msgid, u8 entry_flags, u8 type,
 645                                  void *payload, size_t len)
 646 {
 647         struct sk_buff *skb;
 648         u16 head;
 649         u8 msgid;
 650         unsigned long flags;
 651
 652         spin_lock_irqsave(&ring->lock, flags);
 653         if (!ring->enabled) {
 654                 dev_warn(&bcm4377->pdev->dev,
 655                          "event for disabled transfer ring %d\n",
 656                          ring->ring_id);
 657                 goto out;
 658         }
 659
 660         if (ring->d2h_buffers_only &&
 661             entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
 662                 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
 663                         goto out;
 664
 665                 if (len > ring->mapped_payload_size) {
 666                         dev_warn(
 667                                 &bcm4377->pdev->dev,
 668                                 "invalid payload len in event for ring %d: %zu > %zu\n",
 669                                 ring->ring_id, len, ring->mapped_payload_size);
 670                         goto out;
 671                 }
 672
 673                 payload = ring->payloads + msgid * ring->mapped_payload_size;
 674         }
 675
 676         skb = bt_skb_alloc(len, GFP_ATOMIC);
 677         if (!skb)
 678                 goto out;
 679
 680         memcpy(skb_put(skb, len), payload, len);
 681         hci_skb_pkt_type(skb) = type;
 682         hci_recv_frame(bcm4377->hdev, skb);
 683
 684 out:
 685         head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
 686         head = (head + 1) % ring->n_entries;
 687         bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
 688
 689         bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
 690
 691         spin_unlock_irqrestore(&ring->lock, flags);
 692 }
 693
 694 static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
 695                                struct bcm4377_transfer_ring *ring,
 696                                u16 raw_msgid)
 697 {
 698         unsigned long flags;
 699         u8 msgid;
 700
 701         spin_lock_irqsave(&ring->lock, flags);
 702
 703         if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
 704                 goto unlock;
 705
 706         if (!test_bit(msgid, ring->msgids)) {
 707                 dev_warn(
 708                         &bcm4377->pdev->dev,
 709                         "invalid message id in ack for ring %d: %d is not used\n",
 710                         ring->ring_id, msgid);
 711                 goto unlock;
 712         }
 713
 714         if (ring->allow_wait && ring->events[msgid]) {
 715                 complete(ring->events[msgid]);
 716                 ring->events[msgid] = NULL;
 717         }
 718
 719         bitmap_release_region(ring->msgids, msgid, ring->n_entries);
 720
 721 unlock:
 722         spin_unlock_irqrestore(&ring->lock, flags);
 723 }
 724
 725 static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
 726                                       struct bcm4377_completion_ring *ring,
 727                                       u16 pos)
 728 {
 729         struct bcm4377_completion_ring_entry *entry;
 730         u16 msg_id, transfer_ring;
 731         size_t entry_size, data_len;
 732         void *data;
 733
 734         if (pos >= ring->n_entries) {
 735                 dev_warn(&bcm4377->pdev->dev,
 736                          "invalid offset %d for completion ring %d\n", pos,
 737                          ring->ring_id);
 738                 return;
 739         }
 740
 741         entry_size = sizeof(*entry) + ring->payload_size;
 742         entry = ring->ring + pos * entry_size;
 743         data = ring->ring + pos * entry_size + sizeof(*entry);
 744         data_len = le32_to_cpu(entry->len);
 745         msg_id = le16_to_cpu(entry->msg_id);
 746         transfer_ring = le16_to_cpu(entry->ring_id);
 747
 748         if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
 749                 dev_warn(
 750                         &bcm4377->pdev->dev,
 751                         "invalid entry at offset %d for transfer ring %d in completion ring %d\n",
 752                         pos, transfer_ring, ring->ring_id);
 753                 return;
 754         }
 755
 756         dev_dbg(&bcm4377->pdev->dev,
 757                 "entry in completion ring %d for transfer ring %d with msg_id %d\n",
 758                 ring->ring_id, transfer_ring, msg_id);
 759
 760         switch (transfer_ring) {
 761         case BCM4377_XFER_RING_CONTROL:
 762                 bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
 763                 break;
 764         case BCM4377_XFER_RING_HCI_H2D:
 765                 bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
 766                 break;
 767         case BCM4377_XFER_RING_SCO_H2D:
 768                 bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
 769                 break;
 770         case BCM4377_XFER_RING_ACL_H2D:
 771                 bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
 772                 break;
 773
 774         case BCM4377_XFER_RING_HCI_D2H:
 775                 bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
 776                                      entry->flags, HCI_EVENT_PKT, data,
 777                                      data_len);
 778                 break;
 779         case BCM4377_XFER_RING_SCO_D2H:
 780                 bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
 781                                      entry->flags, HCI_SCODATA_PKT, data,
 782                                      data_len);
 783                 break;
 784         case BCM4377_XFER_RING_ACL_D2H:
 785                 bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
 786                                      entry->flags, HCI_ACLDATA_PKT, data,
 787                                      data_len);
 788                 break;
 789
 790         default:
 791                 dev_warn(
 792                         &bcm4377->pdev->dev,
 793                         "entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
 794                         ring->ring_id, transfer_ring, msg_id);
 795         }
 796 }
 797
 798 static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
 799                                          struct bcm4377_completion_ring *ring)
 800 {
 801         u16 tail;
 802         __le16 *heads = bcm4377->ring_state->completion_ring_head;
 803         __le16 *tails = bcm4377->ring_state->completion_ring_tail;
 804
 805         if (!ring->enabled)
 806                 return;
 807
 808         tail = le16_to_cpu(tails[ring->ring_id]);
 809         dev_dbg(&bcm4377->pdev->dev,
 810                 "completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
 811                 le16_to_cpu(heads[ring->ring_id]), tail);
 812
 813         while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
 814                 /*
 815                  * ensure the CPU doesn't speculate through the comparison.
 816                  * otherwise it might already read the (empty) queue entry
 817                  * before the updated head has been loaded and checked.
 818                  */
 819                 dma_rmb();
 820
 821                 bcm4377_handle_completion(bcm4377, ring, tail);
 822
 823                 tail = (tail + 1) % ring->n_entries;
 824                 tails[ring->ring_id] = cpu_to_le16(tail);
 825         }
 826 }
 827
 828 static irqreturn_t bcm4377_irq(int irq, void *data)
 829 {
 830         struct bcm4377_data *bcm4377 = data;
 831         u32 bootstage, rti_status;
 832
 833         bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
 834         rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
 835
 836         if (bootstage != bcm4377->bootstage ||
 837             rti_status != bcm4377->rti_status) {
 838                 dev_dbg(&bcm4377->pdev->dev,
 839                         "bootstage = %d -> %d, rti state = %d -> %d\n",
 840                         bcm4377->bootstage, bootstage, bcm4377->rti_status,
 841                         rti_status);
 842                 complete(&bcm4377->event);
 843                 bcm4377->bootstage = bootstage;
 844                 bcm4377->rti_status = rti_status;
 845         }
 846
 847         if (rti_status > 2)
 848                 dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
 849
 850         bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
 851         bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
 852         bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
 853         bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
 854         bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
 855
 856         return IRQ_HANDLED;
 857 }
 858
 859 static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
 860                            struct bcm4377_transfer_ring *ring, void *data,
 861                            size_t len, bool wait)
 862 {
 863         unsigned long flags;
 864         struct bcm4377_xfer_ring_entry *entry;
 865         void *payload;
 866         size_t offset;
 867         u16 head, tail, new_head;
 868         u16 raw_msgid;
 869         int ret, msgid;
 870         DECLARE_COMPLETION_ONSTACK(event);
 871
 872         if (len > ring->payload_size && len > ring->mapped_payload_size) {
 873                 dev_warn(
 874                         &bcm4377->pdev->dev,
 875                         "payload len %zu is too large for ring %d (max is %zu or %zu)\n",
 876                         len, ring->ring_id, ring->payload_size,
 877                         ring->mapped_payload_size);
 878                 return -EINVAL;
 879         }
 880         if (wait && !ring->allow_wait)
 881                 return -EINVAL;
 882         if (ring->virtual)
 883                 return -EINVAL;
 884
 885         spin_lock_irqsave(&ring->lock, flags);
 886
 887         head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
 888         tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
 889
 890         new_head = (head + 1) % ring->n_entries;
 891
 892         if (new_head == tail) {
 893                 dev_warn(&bcm4377->pdev->dev,
 894                          "can't send message because ring %d is full\n",
 895                          ring->ring_id);
 896                 ret = -EINVAL;
 897                 goto out;
 898         }
 899
 900         msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
 901         if (msgid < 0) {
 902                 dev_warn(&bcm4377->pdev->dev,
 903                          "can't find message id for ring %d\n", ring->ring_id);
 904                 ret = -EINVAL;
 905                 goto out;
 906         }
 907
 908         raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
 909         raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
 910
 911         offset = head * (sizeof(*entry) + ring->payload_size);
 912         entry = ring->ring + offset;
 913
 914         memset(entry, 0, sizeof(*entry));
 915         entry->id = cpu_to_le16(raw_msgid);
 916         entry->len = cpu_to_le16(len);
 917
 918         if (len <= ring->payload_size) {
 919                 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
 920                 payload = ring->ring + offset + sizeof(*entry);
 921         } else {
 922                 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
 923                 entry->payload = cpu_to_le64(ring->payloads_dma +
 924                                              msgid * ring->mapped_payload_size);
 925                 payload = ring->payloads + msgid * ring->mapped_payload_size;
 926         }
 927
 928         memcpy(payload, data, len);
 929
 930         if (wait)
 931                 ring->events[msgid] = &event;
 932
 933         /*
 934          * The 4377 chips stop responding to any commands as soon as they
 935          * have been idle for a while. Poking the sleep control register here
 936          * makes them come alive again.
 937          */
 938         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
 939                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
 940
 941         dev_dbg(&bcm4377->pdev->dev,
 942                 "updating head for transfer queue #%d to %d\n", ring->ring_id,
 943                 new_head);
 944         bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
 945                 cpu_to_le16(new_head);
 946
 947         if (!ring->sync)
 948                 bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
 949         ret = 0;
 950
 951 out:
 952         spin_unlock_irqrestore(&ring->lock, flags);
 953
 954         if (ret == 0 && wait) {
 955                 ret = wait_for_completion_interruptible_timeout(
 956                         &event, BCM4377_TIMEOUT);
 957                 if (ret == 0)
 958                         ret = -ETIMEDOUT;
 959                 else if (ret > 0)
 960                         ret = 0;
 961
 962                 spin_lock_irqsave(&ring->lock, flags);
 963                 ring->events[msgid] = NULL;
 964                 spin_unlock_irqrestore(&ring->lock, flags);
 965         }
 966
 967         return ret;
 968 }
 969
 970 static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
 971                                           struct bcm4377_completion_ring *ring)
 972 {
 973         struct bcm4377_create_completion_ring_msg msg;
 974         int ret;
 975
 976         if (ring->enabled) {
 977                 dev_warn(&bcm4377->pdev->dev,
 978                          "completion ring %d already enabled\n", ring->ring_id);
 979                 return 0;
 980         }
 981
 982         memset(ring->ring, 0,
 983                ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
 984                                   ring->payload_size));
 985         memset(&msg, 0, sizeof(msg));
 986         msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
 987         msg.id = cpu_to_le16(ring->ring_id);
 988         msg.id_again = cpu_to_le16(ring->ring_id);
 989         msg.ring_iova = cpu_to_le64(ring->ring_dma);
 990         msg.n_elements = cpu_to_le16(ring->n_entries);
 991         msg.intmod_bytes = cpu_to_le32(0xffffffff);
 992         msg.unk = cpu_to_le32(0xffffffff);
 993         msg.intmod_delay = cpu_to_le16(ring->delay);
 994         msg.footer_size = ring->payload_size / 4;
 995
 996         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
 997                               sizeof(msg), true);
 998         if (!ret)
 999                 ring->enabled = true;
1000
1001         return ret;
1002 }
1003
1004 static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1005                                            struct bcm4377_completion_ring *ring)
1006 {
1007         struct bcm4377_destroy_completion_ring_msg msg;
1008         int ret;
1009
1010         memset(&msg, 0, sizeof(msg));
1011         msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1012         msg.ring_id = cpu_to_le16(ring->ring_id);
1013
1014         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1015                               sizeof(msg), true);
1016         if (ret)
1017                 dev_warn(&bcm4377->pdev->dev,
1018                          "failed to destroy completion ring %d\n",
1019                          ring->ring_id);
1020
1021         ring->enabled = false;
1022         return ret;
1023 }
1024
1025 static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1026                                         struct bcm4377_transfer_ring *ring)
1027 {
1028         struct bcm4377_create_transfer_ring_msg msg;
1029         u16 flags = 0;
1030         int ret, i;
1031         unsigned long spinlock_flags;
1032
1033         if (ring->virtual)
1034                 flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1035         if (ring->sync)
1036                 flags |= BCM4377_XFER_RING_FLAG_SYNC;
1037
1038         spin_lock_irqsave(&ring->lock, spinlock_flags);
1039         memset(&msg, 0, sizeof(msg));
1040         msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1041         msg.ring_id = cpu_to_le16(ring->ring_id);
1042         msg.ring_id_again = cpu_to_le16(ring->ring_id);
1043         msg.ring_iova = cpu_to_le64(ring->ring_dma);
1044         msg.n_elements = cpu_to_le16(ring->n_entries);
1045         msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1046         msg.doorbell = cpu_to_le16(ring->doorbell);
1047         msg.flags = cpu_to_le16(flags);
1048         msg.footer_size = ring->payload_size / 4;
1049
1050         bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1051         bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1052         ring->generation++;
1053         spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1054
1055         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1056                               sizeof(msg), true);
1057
1058         spin_lock_irqsave(&ring->lock, spinlock_flags);
1059
1060         if (ring->d2h_buffers_only) {
1061                 for (i = 0; i < ring->n_entries; ++i) {
1062                         struct bcm4377_xfer_ring_entry *entry =
1063                                 ring->ring + i * sizeof(*entry);
1064                         u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1065                                                    ring->generation);
1066                         raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1067
1068                         memset(entry, 0, sizeof(*entry));
1069                         entry->id = cpu_to_le16(raw_msgid);
1070                         entry->len = cpu_to_le16(ring->mapped_payload_size);
1071                         entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1072                         entry->payload =
1073                                 cpu_to_le64(ring->payloads_dma +
1074                                             i * ring->mapped_payload_size);
1075                 }
1076         }
1077
1078         /*
1079          * send some messages if this is a device->host ring to allow the device
1080          * to reply by acknowledging them in the completion ring
1081          */
1082         if (ring->virtual || ring->d2h_buffers_only) {
1083                 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1084                         cpu_to_le16(0xf);
1085                 bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1086         }
1087
1088         ring->enabled = true;
1089         spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1090
1091         return ret;
1092 }
1093
1094 static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1095                                          struct bcm4377_transfer_ring *ring)
1096 {
1097         struct bcm4377_destroy_transfer_ring_msg msg;
1098         int ret;
1099
1100         memset(&msg, 0, sizeof(msg));
1101         msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1102         msg.ring_id = cpu_to_le16(ring->ring_id);
1103
1104         ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1105                               sizeof(msg), true);
1106         if (ret)
1107                 dev_warn(&bcm4377->pdev->dev,
1108                          "failed to destroy transfer ring %d\n", ring->ring_id);
1109
1110         ring->enabled = false;
1111         return ret;
1112 }
1113
1114 static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1115                                             const void *data, size_t data_len,
1116                                             u16 blocks_left)
1117 {
1118         struct bcm4378_hci_send_calibration_cmd cmd;
1119         struct sk_buff *skb;
1120
1121         if (data_len > sizeof(cmd.data))
1122                 return -EINVAL;
1123
1124         memset(&cmd, 0, sizeof(cmd));
1125         cmd.unk = 0x03;
1126         cmd.blocks_left = cpu_to_le16(blocks_left);
1127         memcpy(cmd.data, data, data_len);
1128
1129         skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1130                              HCI_INIT_TIMEOUT);
1131         if (IS_ERR(skb))
1132                 return PTR_ERR(skb);
1133
1134         kfree_skb(skb);
1135         return 0;
1136 }
1137
1138 static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1139                                       const void *data, size_t data_size)
1140 {
1141         int ret;
1142         size_t i, left, transfer_len;
1143         size_t blocks =
1144                 DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1145
1146         if (!data) {
1147                 dev_err(&bcm4377->pdev->dev,
1148                         "no calibration data available.\n");
1149                 return -ENOENT;
1150         }
1151
1152         for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1153                 transfer_len =
1154                         min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1155
1156                 ret = __bcm4378_send_calibration_chunk(
1157                         bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1158                         transfer_len, blocks - i - 1);
1159                 if (ret) {
1160                         dev_err(&bcm4377->pdev->dev,
1161                                 "send calibration chunk failed with %d\n", ret);
1162                         return ret;
1163                 }
1164         }
1165
1166         return 0;
1167 }
1168
1169 static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1170 {
1171         if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1172             strcmp(bcm4377->stepping, "b3") == 0)
1173                 return __bcm4378_send_calibration(
1174                         bcm4377, bcm4377->taurus_beamforming_cal_blob,
1175                         bcm4377->taurus_beamforming_cal_size);
1176         else
1177                 return __bcm4378_send_calibration(bcm4377,
1178                                                   bcm4377->taurus_cal_blob,
1179                                                   bcm4377->taurus_cal_size);
1180 }
1181
1182 static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1183 {
1184         if (strcmp(bcm4377->stepping, "c2") == 0)
1185                 return __bcm4378_send_calibration(
1186                         bcm4377, bcm4377->taurus_beamforming_cal_blob,
1187                         bcm4377->taurus_beamforming_cal_size);
1188         else
1189                 return __bcm4378_send_calibration(bcm4377,
1190                                                   bcm4377->taurus_cal_blob,
1191                                                   bcm4377->taurus_cal_size);
1192 }
1193
1194 static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1195                                                    const char *suffix)
1196 {
1197         const struct firmware *fw;
1198         char name0[64], name1[64];
1199         int ret;
1200
1201         snprintf(name0, sizeof(name0), "/*(DEBLOBBED)*/",
1202                  bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1203                  bcm4377->vendor, suffix);
1204         snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1205                  bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1206                  suffix);
1207         dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1208                 name0, name1);
1209
1210         ret = firmware_reject_nowarn(&fw, name0, &bcm4377->pdev->dev);
1211         if (!ret)
1212                 return fw;
1213         ret = firmware_reject_nowarn(&fw, name1, &bcm4377->pdev->dev);
1214         if (!ret)
1215                 return fw;
1216
1217         dev_err(&bcm4377->pdev->dev,
1218                 "Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1219         return NULL;
1220 }
1221
1222 static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1223                             const struct firmware *fw)
1224 {
1225         struct sk_buff *skb;
1226
1227         skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1228                              HCI_INIT_TIMEOUT);
1229         /*
1230          * This command seems to always fail on more recent firmware versions
1231          * (even in traces taken from the macOS driver). It's unclear why this
1232          * happens but because the PTB file contains calibration and/or
1233          * regulatory data and may be required on older firmware we still try to
1234          * send it here just in case and just ignore if it fails.
1235          */
1236         if (!IS_ERR(skb))
1237                 kfree_skb(skb);
1238         return 0;
1239 }
1240
1241 static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1242                                   const void *data, size_t data_len,
1243                                   u16 blocks_left)
1244 {
1245         struct bcm4378_hci_send_ptb_cmd cmd;
1246         struct sk_buff *skb;
1247
1248         if (data_len > BCM4378_PTB_CHUNK_SIZE)
1249                 return -EINVAL;
1250
1251         memset(&cmd, 0, sizeof(cmd));
1252         cmd.blocks_left = cpu_to_le16(blocks_left);
1253         memcpy(cmd.data, data, data_len);
1254
1255         skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1256                              HCI_INIT_TIMEOUT);
1257         if (IS_ERR(skb))
1258                 return PTR_ERR(skb);
1259
1260         kfree_skb(skb);
1261         return 0;
1262 }
1263
1264 static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1265                             const struct firmware *fw)
1266 {
1267         size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1268         size_t i, left, transfer_len;
1269         int ret;
1270
1271         for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1272                 transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1273
1274                 dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1275                         i + 1, chunks);
1276                 ret = bcm4378_send_ptb_chunk(
1277                         bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1278                         transfer_len, chunks - i - 1);
1279                 if (ret) {
1280                         dev_err(&bcm4377->pdev->dev,
1281                                 "sending ptb chunk %zu failed (%d)", i, ret);
1282                         return ret;
1283                 }
1284         }
1285
1286         return 0;
1287 }
1288
1289 static int bcm4377_hci_open(struct hci_dev *hdev)
1290 {
1291         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1292         int ret;
1293
1294         dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1295
1296         ret = bcm4377_create_completion_ring(bcm4377,
1297                                              &bcm4377->hci_acl_ack_ring);
1298         if (ret)
1299                 return ret;
1300         ret = bcm4377_create_completion_ring(bcm4377,
1301                                              &bcm4377->hci_acl_event_ring);
1302         if (ret)
1303                 goto destroy_hci_acl_ack;
1304         ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1305         if (ret)
1306                 goto destroy_hci_acl_event;
1307         ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1308         if (ret)
1309                 goto destroy_sco_ack;
1310         dev_dbg(&bcm4377->pdev->dev,
1311                 "all completion rings successfully created!\n");
1312
1313         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1314         if (ret)
1315                 goto destroy_sco_event;
1316         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1317         if (ret)
1318                 goto destroy_hci_h2d;
1319         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1320         if (ret)
1321                 goto destroy_hci_d2h;
1322         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1323         if (ret)
1324                 goto destroy_sco_h2d;
1325         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1326         if (ret)
1327                 goto destroy_sco_d2h;
1328         ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1329         if (ret)
1330                 goto destroy_acl_h2d;
1331         dev_dbg(&bcm4377->pdev->dev,
1332                 "all transfer rings successfully created!\n");
1333
1334         return 0;
1335
1336 destroy_acl_h2d:
1337         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1338 destroy_sco_d2h:
1339         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1340 destroy_sco_h2d:
1341         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1342 destroy_hci_d2h:
1343         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1344 destroy_hci_h2d:
1345         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1346 destroy_sco_event:
1347         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1348 destroy_sco_ack:
1349         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1350 destroy_hci_acl_event:
1351         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1352 destroy_hci_acl_ack:
1353         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1354
1355         dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1356         return ret;
1357 }
1358
1359 static int bcm4377_hci_close(struct hci_dev *hdev)
1360 {
1361         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1362
1363         dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1364
1365         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1366         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1367         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1368         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1369         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1370         bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1371
1372         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1373         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1374         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1375         bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1376
1377         return 0;
1378 }
1379
1380 static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1381                                     bdaddr_t *addr)
1382 {
1383         if (addr->b[0] != 0x93)
1384                 return true;
1385         if (addr->b[1] != 0x76)
1386                 return true;
1387         if (addr->b[2] != 0x00)
1388                 return true;
1389         if (addr->b[4] != (bcm4377->hw->id & 0xff))
1390                 return true;
1391         if (addr->b[5] != (bcm4377->hw->id >> 8))
1392                 return true;
1393         return false;
1394 }
1395
1396 static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1397 {
1398         struct hci_rp_read_bd_addr *bda;
1399         struct sk_buff *skb;
1400
1401         skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1402                              HCI_INIT_TIMEOUT);
1403         if (IS_ERR(skb)) {
1404                 int err = PTR_ERR(skb);
1405
1406                 dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1407                         err);
1408                 return err;
1409         }
1410
1411         if (skb->len != sizeof(*bda)) {
1412                 dev_err(&bcm4377->pdev->dev,
1413                         "HCI_OP_READ_BD_ADDR reply length invalid");
1414                 kfree_skb(skb);
1415                 return -EIO;
1416         }
1417
1418         bda = (struct hci_rp_read_bd_addr *)skb->data;
1419         if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1420                 set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &bcm4377->hdev->quirks);
1421
1422         kfree_skb(skb);
1423         return 0;
1424 }
1425
1426 static int bcm4377_hci_setup(struct hci_dev *hdev)
1427 {
1428         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1429         const struct firmware *fw;
1430         int ret;
1431
1432         if (bcm4377->hw->send_calibration) {
1433                 ret = bcm4377->hw->send_calibration(bcm4377);
1434                 if (ret)
1435                         return ret;
1436         }
1437
1438         fw = bcm4377_request_blob(bcm4377, "ptb");
1439         if (!fw) {
1440                 dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1441                 return -ENOENT;
1442         }
1443
1444         ret = bcm4377->hw->send_ptb(bcm4377, fw);
1445         release_firmware(fw);
1446         if (ret)
1447                 return ret;
1448
1449         return bcm4377_check_bdaddr(bcm4377);
1450 }
1451
1452 static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1453 {
1454         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1455         struct bcm4377_transfer_ring *ring;
1456         int ret;
1457
1458         switch (hci_skb_pkt_type(skb)) {
1459         case HCI_COMMAND_PKT:
1460                 hdev->stat.cmd_tx++;
1461                 ring = &bcm4377->hci_h2d_ring;
1462                 break;
1463
1464         case HCI_ACLDATA_PKT:
1465                 hdev->stat.acl_tx++;
1466                 ring = &bcm4377->acl_h2d_ring;
1467                 break;
1468
1469         case HCI_SCODATA_PKT:
1470                 hdev->stat.sco_tx++;
1471                 ring = &bcm4377->sco_h2d_ring;
1472                 break;
1473
1474         default:
1475                 return -EILSEQ;
1476         }
1477
1478         ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1479         if (ret < 0) {
1480                 hdev->stat.err_tx++;
1481                 return ret;
1482         }
1483
1484         hdev->stat.byte_tx += skb->len;
1485         kfree_skb(skb);
1486         return ret;
1487 }
1488
1489 static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1490 {
1491         struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1492         struct sk_buff *skb;
1493         int err;
1494
1495         skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1496         if (IS_ERR(skb)) {
1497                 err = PTR_ERR(skb);
1498                 dev_err(&bcm4377->pdev->dev,
1499                         "Change address command failed (%d)", err);
1500                 return err;
1501         }
1502         kfree_skb(skb);
1503
1504         return 0;
1505 }
1506
1507 static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1508                                        struct bcm4377_transfer_ring *ring)
1509 {
1510         size_t entry_size;
1511
1512         spin_lock_init(&ring->lock);
1513         ring->payload_size = ALIGN(ring->payload_size, 4);
1514         ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1515
1516         if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1517                 return -EINVAL;
1518         if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1519                 return -EINVAL;
1520         if (ring->virtual && ring->allow_wait)
1521                 return -EINVAL;
1522
1523         if (ring->d2h_buffers_only) {
1524                 if (ring->virtual)
1525                         return -EINVAL;
1526                 if (ring->payload_size)
1527                         return -EINVAL;
1528                 if (!ring->mapped_payload_size)
1529                         return -EINVAL;
1530         }
1531         if (ring->virtual)
1532                 return 0;
1533
1534         entry_size =
1535                 ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1536         ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1537                                          ring->n_entries * entry_size,
1538                                          &ring->ring_dma, GFP_KERNEL);
1539         if (!ring->ring)
1540                 return -ENOMEM;
1541
1542         if (ring->allow_wait) {
1543                 ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1544                                             ring->n_entries,
1545                                             sizeof(*ring->events), GFP_KERNEL);
1546                 if (!ring->events)
1547                         return -ENOMEM;
1548         }
1549
1550         if (ring->mapped_payload_size) {
1551                 ring->payloads = dmam_alloc_coherent(
1552                         &bcm4377->pdev->dev,
1553                         ring->n_entries * ring->mapped_payload_size,
1554                         &ring->payloads_dma, GFP_KERNEL);
1555                 if (!ring->payloads)
1556                         return -ENOMEM;
1557         }
1558
1559         return 0;
1560 }
1561
1562 static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1563                                          struct bcm4377_completion_ring *ring)
1564 {
1565         size_t entry_size;
1566
1567         ring->payload_size = ALIGN(ring->payload_size, 4);
1568         if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1569                 return -EINVAL;
1570         if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1571                 return -EINVAL;
1572
1573         entry_size = ring->payload_size +
1574                      sizeof(struct bcm4377_completion_ring_entry);
1575
1576         ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1577                                          ring->n_entries * entry_size,
1578                                          &ring->ring_dma, GFP_KERNEL);
1579         if (!ring->ring)
1580                 return -ENOMEM;
1581         return 0;
1582 }
1583
1584 static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1585 {
1586         struct device *dev = &bcm4377->pdev->dev;
1587         dma_addr_t peripheral_info_dma;
1588
1589         bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1590                                            &bcm4377->ctx_dma, GFP_KERNEL);
1591         if (!bcm4377->ctx)
1592                 return -ENOMEM;
1593         memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1594
1595         bcm4377->ring_state =
1596                 dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1597                                     &bcm4377->ring_state_dma, GFP_KERNEL);
1598         if (!bcm4377->ring_state)
1599                 return -ENOMEM;
1600         memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1601
1602         bcm4377->ctx->version = cpu_to_le16(1);
1603         bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1604         bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1605
1606         /*
1607          * The BT device will write 0x20 bytes of data to this buffer but
1608          * the exact contents are unknown. It only needs to exist for BT
1609          * to work such that we can just allocate and then ignore it.
1610          */
1611         if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1612                                  &peripheral_info_dma, GFP_KERNEL))
1613                 return -ENOMEM;
1614         bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1615
1616         bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1617                 bcm4377->ring_state_dma +
1618                 offsetof(struct bcm4377_ring_state, xfer_ring_head));
1619         bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1620                 bcm4377->ring_state_dma +
1621                 offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1622         bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1623                 bcm4377->ring_state_dma +
1624                 offsetof(struct bcm4377_ring_state, completion_ring_head));
1625         bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1626                 bcm4377->ring_state_dma +
1627                 offsetof(struct bcm4377_ring_state, completion_ring_tail));
1628
1629         bcm4377->ctx->n_completion_rings =
1630                 cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1631         bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1632
1633         bcm4377->ctx->control_completion_ring_addr =
1634                 cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1635         bcm4377->ctx->control_completion_ring_n_entries =
1636                 cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1637         bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1638         bcm4377->ctx->control_completion_ring_msi = 0;
1639         bcm4377->ctx->control_completion_ring_header_size = 0;
1640         bcm4377->ctx->control_completion_ring_footer_size = 0;
1641
1642         bcm4377->ctx->control_xfer_ring_addr =
1643                 cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1644         bcm4377->ctx->control_xfer_ring_n_entries =
1645                 cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1646         bcm4377->ctx->control_xfer_ring_doorbell =
1647                 cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1648         bcm4377->ctx->control_xfer_ring_msi = 0;
1649         bcm4377->ctx->control_xfer_ring_header_size = 0;
1650         bcm4377->ctx->control_xfer_ring_footer_size =
1651                 bcm4377->control_h2d_ring.payload_size / 4;
1652
1653         dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1654                 &bcm4377->ctx_dma);
1655
1656         return 0;
1657 }
1658
1659 static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1660 {
1661         int ret;
1662
1663         /*
1664          * Even though many of these settings appear to be configurable
1665          * when sending the "create ring" messages most of these are
1666          * actually hardcoded in some (and quite possibly all) firmware versions
1667          * and changing them on the host has no effect.
1668          * Specifically, this applies to at least the doorbells, the transfer
1669          * and completion ring ids and their mapping (e.g. both HCI and ACL
1670          * entries will always be queued in completion rings 1 and 2 no matter
1671          * what we configure here).
1672          */
1673         bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1674         bcm4377->control_ack_ring.n_entries = 32;
1675         bcm4377->control_ack_ring.transfer_rings =
1676                 BIT(BCM4377_XFER_RING_CONTROL);
1677
1678         bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1679         bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1680         bcm4377->hci_acl_ack_ring.transfer_rings =
1681                 BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1682         bcm4377->hci_acl_ack_ring.delay = 1000;
1683
1684         /*
1685          * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1686          * ACL packets will be transmitted inside buffers mapped via
1687          * acl_d2h_ring anyway.
1688          */
1689         bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1690         bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1691         bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1692         bcm4377->hci_acl_event_ring.transfer_rings =
1693                 BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1694         bcm4377->hci_acl_event_ring.delay = 1000;
1695
1696         bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1697         bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1698         bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1699
1700         bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1701         bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1702         bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1703         bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1704
1705         bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1706         bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1707         bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1708         bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1709         bcm4377->control_h2d_ring.allow_wait = true;
1710         bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1711
1712         bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1713         bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1714         bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1715         bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1716         bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1717
1718         bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1719         bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1720         bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1721         bcm4377->hci_d2h_ring.virtual = true;
1722         bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1723
1724         bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1725         bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1726         bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1727         bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1728         bcm4377->sco_h2d_ring.sync = true;
1729         bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1730
1731         bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1732         bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1733         bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1734         bcm4377->sco_d2h_ring.virtual = true;
1735         bcm4377->sco_d2h_ring.sync = true;
1736         bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1737
1738         /*
1739          * This ring has to use mapped_payload_size because the largest ACL
1740          * packet doesn't fit inside the largest possible footer
1741          */
1742         bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1743         bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1744         bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1745         bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1746         bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1747
1748         /*
1749          * This ring only contains empty buffers to be used by incoming
1750          * ACL packets that do not fit inside the footer of hci_acl_event_ring
1751          */
1752         bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1753         bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1754         bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1755         bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1756         bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1757         bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1758
1759         /*
1760          * no need for any cleanup since this is only called from _probe
1761          * and only devres-managed allocations are used
1762          */
1763         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1764         if (ret)
1765                 return ret;
1766         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1767         if (ret)
1768                 return ret;
1769         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1770         if (ret)
1771                 return ret;
1772         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1773         if (ret)
1774                 return ret;
1775         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1776         if (ret)
1777                 return ret;
1778         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1779         if (ret)
1780                 return ret;
1781         ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1782         if (ret)
1783                 return ret;
1784
1785         ret = bcm4377_alloc_completion_ring(bcm4377,
1786                                             &bcm4377->control_ack_ring);
1787         if (ret)
1788                 return ret;
1789         ret = bcm4377_alloc_completion_ring(bcm4377,
1790                                             &bcm4377->hci_acl_ack_ring);
1791         if (ret)
1792                 return ret;
1793         ret = bcm4377_alloc_completion_ring(bcm4377,
1794                                             &bcm4377->hci_acl_event_ring);
1795         if (ret)
1796                 return ret;
1797         ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1798         if (ret)
1799                 return ret;
1800         ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1801         if (ret)
1802                 return ret;
1803
1804         dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1805
1806         return 0;
1807 }
1808
1809 static int bcm4377_boot(struct bcm4377_data *bcm4377)
1810 {
1811         const struct firmware *fw;
1812         void *bfr;
1813         dma_addr_t fw_dma;
1814         int ret = 0;
1815         u32 bootstage, rti_status;
1816
1817         bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1818         rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1819
1820         if (bootstage != 0) {
1821                 dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1822                         bootstage);
1823                 return -EINVAL;
1824         }
1825
1826         if (rti_status != 0) {
1827                 dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1828                         rti_status);
1829                 return -EINVAL;
1830         }
1831
1832         fw = bcm4377_request_blob(bcm4377, "bin");
1833         if (!fw) {
1834                 dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1835                 return -ENOENT;
1836         }
1837
1838         bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1839                                  GFP_KERNEL);
1840         if (!bfr) {
1841                 ret = -ENOMEM;
1842                 goto out_release_fw;
1843         }
1844
1845         memcpy(bfr, fw->data, fw->size);
1846
1847         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1848         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1849         iowrite32(BCM4377_DMA_MASK,
1850                   bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1851
1852         iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1853         iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1854         iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1855         iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1856
1857         dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1858
1859         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1860                                                         BCM4377_TIMEOUT);
1861         if (ret == 0) {
1862                 ret = -ETIMEDOUT;
1863                 goto out_dma_free;
1864         } else if (ret < 0) {
1865                 goto out_dma_free;
1866         }
1867
1868         if (bcm4377->bootstage != 2) {
1869                 dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1870                         bcm4377->bootstage);
1871                 ret = -ENXIO;
1872                 goto out_dma_free;
1873         }
1874
1875         dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1876                 bcm4377->bootstage);
1877         ret = 0;
1878
1879 out_dma_free:
1880         dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1881 out_release_fw:
1882         release_firmware(fw);
1883         return ret;
1884 }
1885
1886 static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1887 {
1888         int ret;
1889
1890         dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1891         iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1892
1893         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1894                                                         BCM4377_TIMEOUT);
1895         if (ret == 0) {
1896                 dev_err(&bcm4377->pdev->dev,
1897                         "timed out while waiting for RTI to transition to state 1");
1898                 return -ETIMEDOUT;
1899         } else if (ret < 0) {
1900                 return ret;
1901         }
1902
1903         if (bcm4377->rti_status != 1) {
1904                 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1905                         bcm4377->rti_status);
1906                 return -ENODEV;
1907         }
1908         dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1909
1910         /* allow access to the entire IOVA space again */
1911         iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1912         iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1913         iowrite32(BCM4377_DMA_MASK,
1914                   bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1915
1916         /* setup "Converged IPC" context */
1917         iowrite32(lower_32_bits(bcm4377->ctx_dma),
1918                   bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1919         iowrite32(upper_32_bits(bcm4377->ctx_dma),
1920                   bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1921         iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1922
1923         ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1924                                                         BCM4377_TIMEOUT);
1925         if (ret == 0) {
1926                 dev_err(&bcm4377->pdev->dev,
1927                         "timed out while waiting for RTI to transition to state 2");
1928                 return -ETIMEDOUT;
1929         } else if (ret < 0) {
1930                 return ret;
1931         }
1932
1933         if (bcm4377->rti_status != 2) {
1934                 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1935                         bcm4377->rti_status);
1936                 return -ENODEV;
1937         }
1938
1939         dev_dbg(&bcm4377->pdev->dev,
1940                 "RTI is in state 2; control ring is ready\n");
1941         bcm4377->control_ack_ring.enabled = true;
1942
1943         return 0;
1944 }
1945
1946 static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1947                                           char tag, const char *val, size_t len)
1948 {
1949         if (tag != 'V')
1950                 return 0;
1951         if (len >= sizeof(bcm4377->vendor))
1952                 return -EINVAL;
1953
1954         strscpy(bcm4377->vendor, val, len + 1);
1955         return 0;
1956 }
1957
1958 static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1959                                          const char *val, size_t len)
1960 {
1961         size_t idx = 0;
1962
1963         if (tag != 's')
1964                 return 0;
1965         if (len >= sizeof(bcm4377->stepping))
1966                 return -EINVAL;
1967
1968         while (len != 0) {
1969                 bcm4377->stepping[idx] = tolower(val[idx]);
1970                 if (val[idx] == '\0')
1971                         return 0;
1972
1973                 idx++;
1974                 len--;
1975         }
1976
1977         bcm4377->stepping[idx] = '\0';
1978         return 0;
1979 }
1980
1981 static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1982                                  enum bcm4377_otp_params_type type)
1983 {
1984         const char *p;
1985         int ret;
1986
1987         p = skip_spaces(str);
1988         while (*p) {
1989                 char tag = *p++;
1990                 const char *end;
1991                 size_t len;
1992
1993                 if (*p++ != '=') /* implicit NUL check */
1994                         return -EINVAL;
1995
1996                 /* *p might be NUL here, if so end == p and len == 0 */
1997                 end = strchrnul(p, ' ');
1998                 len = end - p;
1999
2000                 /* leave 1 byte for NUL in destination string */
2001                 if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2002                         return -EINVAL;
2003
2004                 switch (type) {
2005                 case BCM4377_OTP_BOARD_PARAMS:
2006                         ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2007                                                              len);
2008                         break;
2009                 case BCM4377_OTP_CHIP_PARAMS:
2010                         ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2011                                                             len);
2012                         break;
2013                 default:
2014                         ret = -EINVAL;
2015                         break;
2016                 }
2017
2018                 if (ret)
2019                         return ret;
2020
2021                 /* Skip to next arg, if any */
2022                 p = skip_spaces(end);
2023         }
2024
2025         return 0;
2026 }
2027
2028 static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2029                                         size_t size)
2030 {
2031         int idx = 4;
2032         const char *chip_params;
2033         const char *board_params;
2034         int ret;
2035
2036         /* 4-byte header and two empty strings */
2037         if (size < 6)
2038                 return -EINVAL;
2039
2040         if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2041                 return -EINVAL;
2042
2043         chip_params = &otp[idx];
2044
2045         /* Skip first string, including terminator */
2046         idx += strnlen(chip_params, size - idx) + 1;
2047         if (idx >= size)
2048                 return -EINVAL;
2049
2050         board_params = &otp[idx];
2051
2052         /* Skip to terminator of second string */
2053         idx += strnlen(board_params, size - idx);
2054         if (idx >= size)
2055                 return -EINVAL;
2056
2057         /* At this point both strings are guaranteed NUL-terminated */
2058         dev_dbg(&bcm4377->pdev->dev,
2059                 "OTP: chip_params='%s' board_params='%s'\n", chip_params,
2060                 board_params);
2061
2062         ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2063                                     BCM4377_OTP_CHIP_PARAMS);
2064         if (ret)
2065                 return ret;
2066
2067         ret = bcm4377_parse_otp_str(bcm4377, board_params,
2068                                     BCM4377_OTP_BOARD_PARAMS);
2069         if (ret)
2070                 return ret;
2071
2072         if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2073                 return -EINVAL;
2074
2075         dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2076                 bcm4377->stepping, bcm4377->vendor);
2077         return 0;
2078 }
2079
2080 static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2081 {
2082         u8 *otp;
2083         int i;
2084         int ret = -ENOENT;
2085
2086         otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2087         if (!otp)
2088                 return -ENOMEM;
2089
2090         for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2091                 otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2092
2093         i = 0;
2094         while (i < (BCM4377_OTP_SIZE - 1)) {
2095                 u8 type = otp[i];
2096                 u8 length = otp[i + 1];
2097
2098                 if (type == 0)
2099                         break;
2100
2101                 if ((i + 2 + length) > BCM4377_OTP_SIZE)
2102                         break;
2103
2104                 switch (type) {
2105                 case BCM4377_OTP_SYS_VENDOR:
2106                         dev_dbg(&bcm4377->pdev->dev,
2107                                 "OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2108                         ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2109                                                            length);
2110                         break;
2111                 case BCM4377_OTP_CIS:
2112                         dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2113                                 length);
2114                         break;
2115                 default:
2116                         dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2117                                 i, length);
2118                         break;
2119                 }
2120
2121                 i += 2 + length;
2122         }
2123
2124         kfree(otp);
2125         return ret;
2126 }
2127
2128 static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2129 {
2130         int ret;
2131         u32 ctrl;
2132
2133         ret = pci_write_config_dword(bcm4377->pdev,
2134                                      BCM4377_PCIECFG_BAR0_WINDOW1,
2135                                      bcm4377->hw->bar0_window1);
2136         if (ret)
2137                 return ret;
2138
2139         ret = pci_write_config_dword(bcm4377->pdev,
2140                                      BCM4377_PCIECFG_BAR0_WINDOW2,
2141                                      bcm4377->hw->bar0_window2);
2142         if (ret)
2143                 return ret;
2144
2145         ret = pci_write_config_dword(
2146                 bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2147                 BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2148         if (ret)
2149                 return ret;
2150
2151         if (bcm4377->hw->has_bar0_core2_window2) {
2152                 ret = pci_write_config_dword(bcm4377->pdev,
2153                                              BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2154                                              bcm4377->hw->bar0_core2_window2);
2155                 if (ret)
2156                         return ret;
2157         }
2158
2159         ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2160                                      BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2161         if (ret)
2162                 return ret;
2163
2164         ret = pci_read_config_dword(bcm4377->pdev,
2165                                     BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2166         if (ret)
2167                 return ret;
2168
2169         if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2170                 ctrl &= ~BIT(19);
2171         ctrl |= BIT(16);
2172
2173         return pci_write_config_dword(bcm4377->pdev,
2174                                       BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2175 }
2176
2177 static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2178 {
2179         const struct dmi_system_id *board_type_dmi_id;
2180
2181         board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2182         if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2183                 bcm4377->board_type = board_type_dmi_id->driver_data;
2184                 dev_dbg(&bcm4377->pdev->dev,
2185                         "found board type via DMI match: %s\n",
2186                         bcm4377->board_type);
2187         }
2188
2189         return 0;
2190 }
2191
2192 static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2193 {
2194         struct device_node *np = bcm4377->pdev->dev.of_node;
2195         int ret;
2196
2197         if (!np)
2198                 return 0;
2199
2200         ret = of_property_read_string(np, "brcm,board-type",
2201                                       &bcm4377->board_type);
2202         if (ret) {
2203                 dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2204                 return ret;
2205         }
2206
2207         bcm4377->taurus_beamforming_cal_blob =
2208                 of_get_property(np, "brcm,taurus-bf-cal-blob",
2209                                 &bcm4377->taurus_beamforming_cal_size);
2210         if (!bcm4377->taurus_beamforming_cal_blob) {
2211                 dev_err(&bcm4377->pdev->dev,
2212                         "no brcm,taurus-bf-cal-blob property\n");
2213                 return -ENOENT;
2214         }
2215         bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2216                                                    &bcm4377->taurus_cal_size);
2217         if (!bcm4377->taurus_cal_blob) {
2218                 dev_err(&bcm4377->pdev->dev,
2219                         "no brcm,taurus-cal-blob property\n");
2220                 return -ENOENT;
2221         }
2222
2223         return 0;
2224 }
2225
2226 static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2227 {
2228         pci_disable_link_state(bcm4377->pdev,
2229                                PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2230
2231         /*
2232          * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2233          * or if the BIOS hasn't handed over control to us. We must *always*
2234          * disable ASPM for this device due to hardware errata though.
2235          */
2236         pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2237                                    PCI_EXP_LNKCTL_ASPMC);
2238 }
2239
2240 static void bcm4377_pci_free_irq_vectors(void *data)
2241 {
2242         pci_free_irq_vectors(data);
2243 }
2244
2245 static void bcm4377_hci_free_dev(void *data)
2246 {
2247         hci_free_dev(data);
2248 }
2249
2250 static void bcm4377_hci_unregister_dev(void *data)
2251 {
2252         hci_unregister_dev(data);
2253 }
2254
2255 static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2256 {
2257         struct bcm4377_data *bcm4377;
2258         struct hci_dev *hdev;
2259         int ret, irq;
2260
2261         ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2262         if (ret)
2263                 return ret;
2264
2265         bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2266         if (!bcm4377)
2267                 return -ENOMEM;
2268
2269         bcm4377->pdev = pdev;
2270         bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2271         init_completion(&bcm4377->event);
2272
2273         ret = bcm4377_prepare_rings(bcm4377);
2274         if (ret)
2275                 return ret;
2276
2277         ret = bcm4377_init_context(bcm4377);
2278         if (ret)
2279                 return ret;
2280
2281         ret = bcm4377_probe_dmi(bcm4377);
2282         if (ret)
2283                 return ret;
2284         ret = bcm4377_probe_of(bcm4377);
2285         if (ret)
2286                 return ret;
2287         if (!bcm4377->board_type) {
2288                 dev_err(&pdev->dev, "unable to determine board type\n");
2289                 return -ENODEV;
2290         }
2291
2292         if (bcm4377->hw->disable_aspm)
2293                 bcm4377_disable_aspm(bcm4377);
2294
2295         ret = pci_reset_function_locked(pdev);
2296         if (ret)
2297                 dev_warn(
2298                         &pdev->dev,
2299                         "function level reset failed with %d; trying to continue anyway\n",
2300                         ret);
2301
2302         /*
2303          * If this number is too low and we try to access any BAR too
2304          * early the device will crash. Experiments have shown that
2305          * approximately 50 msec is the minimum amount we have to wait.
2306          * Let's double that to be safe.
2307          */
2308         msleep(100);
2309
2310         ret = pcim_enable_device(pdev);
2311         if (ret)
2312                 return ret;
2313         pci_set_master(pdev);
2314
2315         ret = bcm4377_init_cfg(bcm4377);
2316         if (ret)
2317                 return ret;
2318
2319         bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2320         if (!bcm4377->bar0)
2321                 return -EBUSY;
2322         bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2323         if (!bcm4377->bar2)
2324                 return -EBUSY;
2325
2326         ret = bcm4377_parse_otp(bcm4377);
2327         if (ret) {
2328                 dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2329                 return ret;
2330         }
2331
2332         /*
2333          * Legacy interrupts result in an IRQ storm because we don't know where
2334          * the interrupt mask and status registers for these chips are.
2335          * MSIs are acked automatically instead.
2336          */
2337         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2338         if (ret < 0)
2339                 return -ENODEV;
2340         ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2341                                        pdev);
2342         if (ret)
2343                 return ret;
2344
2345         irq = pci_irq_vector(pdev, 0);
2346         if (irq <= 0)
2347                 return -ENODEV;
2348
2349         ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2350                                bcm4377);
2351         if (ret)
2352                 return ret;
2353
2354         hdev = hci_alloc_dev();
2355         if (!hdev)
2356                 return -ENOMEM;
2357         ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2358         if (ret)
2359                 return ret;
2360
2361         bcm4377->hdev = hdev;
2362
2363         hdev->bus = HCI_PCI;
2364         hdev->dev_type = HCI_PRIMARY;
2365         hdev->open = bcm4377_hci_open;
2366         hdev->close = bcm4377_hci_close;
2367         hdev->send = bcm4377_hci_send_frame;
2368         hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2369         hdev->setup = bcm4377_hci_setup;
2370
2371         if (bcm4377->hw->broken_mws_transport_config)
2372                 set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2373         if (bcm4377->hw->broken_ext_scan)
2374                 set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2375         if (bcm4377->hw->broken_le_coded)
2376                 set_bit(HCI_QUIRK_BROKEN_LE_CODED, &hdev->quirks);
2377
2378         pci_set_drvdata(pdev, bcm4377);
2379         hci_set_drvdata(hdev, bcm4377);
2380         SET_HCIDEV_DEV(hdev, &pdev->dev);
2381
2382         ret = bcm4377_boot(bcm4377);
2383         if (ret)
2384                 return ret;
2385
2386         ret = bcm4377_setup_rti(bcm4377);
2387         if (ret)
2388                 return ret;
2389
2390         ret = hci_register_dev(hdev);
2391         if (ret)
2392                 return ret;
2393         return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2394                                         hdev);
2395 }
2396
2397 static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2398 {
2399         struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2400         int ret;
2401
2402         ret = hci_suspend_dev(bcm4377->hdev);
2403         if (ret)
2404                 return ret;
2405
2406         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2407                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2408
2409         return 0;
2410 }
2411
2412 static int bcm4377_resume(struct pci_dev *pdev)
2413 {
2414         struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2415
2416         iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2417                   bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2418
2419         return hci_resume_dev(bcm4377->hdev);
2420 }
2421
2422 static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2423         {
2424                 .matches = {
2425                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2426                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2427                 },
2428                 .driver_data = "apple,formosa",
2429         },
2430         {
2431                 .matches = {
2432                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2433                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2434                 },
2435                 .driver_data = "apple,formosa",
2436         },
2437         {
2438                 .matches = {
2439                         DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2440                         DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2441                 },
2442                 .driver_data = "apple,formosa",
2443         },
2444         {}
2445 };
2446
2447 static const struct bcm4377_hw bcm4377_hw_variants[] = {
2448         [BCM4377] = {
2449                 .id = 0x4377,
2450                 .otp_offset = 0x4120,
2451                 .bar0_window1 = 0x1800b000,
2452                 .bar0_window2 = 0x1810c000,
2453                 .disable_aspm = true,
2454                 .broken_ext_scan = true,
2455                 .send_ptb = bcm4377_send_ptb,
2456         },
2457
2458         [BCM4378] = {
2459                 .id = 0x4378,
2460                 .otp_offset = 0x4120,
2461                 .bar0_window1 = 0x18002000,
2462                 .bar0_window2 = 0x1810a000,
2463                 .bar0_core2_window2 = 0x18107000,
2464                 .has_bar0_core2_window2 = true,
2465                 .broken_mws_transport_config = true,
2466                 .broken_le_coded = true,
2467                 .send_calibration = bcm4378_send_calibration,
2468                 .send_ptb = bcm4378_send_ptb,
2469         },
2470
2471         [BCM4387] = {
2472                 .id = 0x4387,
2473                 .otp_offset = 0x413c,
2474                 .bar0_window1 = 0x18002000,
2475                 .bar0_window2 = 0x18109000,
2476                 .bar0_core2_window2 = 0x18106000,
2477                 .has_bar0_core2_window2 = true,
2478                 .clear_pciecfg_subsystem_ctrl_bit19 = true,
2479                 .broken_mws_transport_config = true,
2480                 .broken_le_coded = true,
2481                 .send_calibration = bcm4387_send_calibration,
2482                 .send_ptb = bcm4378_send_ptb,
2483         },
2484 };
2485
2486 #define BCM4377_DEVID_ENTRY(id)                                             \
2487         {                                                                   \
2488                 PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID,    \
2489                         PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2490                         BCM##id                                             \
2491         }
2492
2493 static const struct pci_device_id bcm4377_devid_table[] = {
2494         BCM4377_DEVID_ENTRY(4377),
2495         BCM4377_DEVID_ENTRY(4378),
2496         BCM4377_DEVID_ENTRY(4387),
2497         {},
2498 };
2499 MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2500
2501 static struct pci_driver bcm4377_pci_driver = {
2502         .name = "hci_bcm4377",
2503         .id_table = bcm4377_devid_table,
2504         .probe = bcm4377_probe,
2505         .suspend = bcm4377_suspend,
2506         .resume = bcm4377_resume,
2507 };
2508 module_pci_driver(bcm4377_pci_driver);
2509
2510 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2511 MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2512 MODULE_LICENSE("Dual MIT/GPL");
2513 /*(DEBLOBBED)*/