GNU Linux-libre 5.19-rc6-gnu
[releases.git] / drivers / mailbox / bcm-flexrm-mailbox.c
1 /*
2  * Copyright (C) 2017 Broadcom
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License as
6  * published by the Free Software Foundation version 2.
7  *
8  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9  * kind, whether express or implied; without even the implied warranty
10  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  */
13
14 /*
15  * Broadcom FlexRM Mailbox Driver
16  *
17  * Each Broadcom FlexSparx4 offload engine is implemented as an
18  * extension to Broadcom FlexRM ring manager. The FlexRM ring
19  * manager provides a set of rings which can be used to submit
20  * work to a FlexSparx4 offload engine.
21  *
22  * This driver creates a mailbox controller using a set of FlexRM
23  * rings where each mailbox channel represents a separate FlexRM ring.
24  */
25
26 #include <asm/barrier.h>
27 #include <asm/byteorder.h>
28 #include <linux/atomic.h>
29 #include <linux/bitmap.h>
30 #include <linux/debugfs.h>
31 #include <linux/delay.h>
32 #include <linux/device.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/dmapool.h>
35 #include <linux/err.h>
36 #include <linux/interrupt.h>
37 #include <linux/kernel.h>
38 #include <linux/mailbox_controller.h>
39 #include <linux/mailbox_client.h>
40 #include <linux/mailbox/brcm-message.h>
41 #include <linux/module.h>
42 #include <linux/msi.h>
43 #include <linux/of_address.h>
44 #include <linux/of_irq.h>
45 #include <linux/platform_device.h>
46 #include <linux/spinlock.h>
47
48 /* ====== FlexRM register defines ===== */
49
50 /* FlexRM configuration */
51 #define RING_REGS_SIZE                                  0x10000
52 #define RING_DESC_SIZE                                  8
53 #define RING_DESC_INDEX(offset)                         \
54                         ((offset) / RING_DESC_SIZE)
55 #define RING_DESC_OFFSET(index)                         \
56                         ((index) * RING_DESC_SIZE)
57 #define RING_MAX_REQ_COUNT                              1024
58 #define RING_BD_ALIGN_ORDER                             12
59 #define RING_BD_ALIGN_CHECK(addr)                       \
60                         (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1)))
61 #define RING_BD_TOGGLE_INVALID(offset)                  \
62                         (((offset) >> RING_BD_ALIGN_ORDER) & 0x1)
63 #define RING_BD_TOGGLE_VALID(offset)                    \
64                         (!RING_BD_TOGGLE_INVALID(offset))
65 #define RING_BD_DESC_PER_REQ                            32
66 #define RING_BD_DESC_COUNT                              \
67                         (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ)
68 #define RING_BD_SIZE                                    \
69                         (RING_BD_DESC_COUNT * RING_DESC_SIZE)
70 #define RING_CMPL_ALIGN_ORDER                           13
71 #define RING_CMPL_DESC_COUNT                            RING_MAX_REQ_COUNT
72 #define RING_CMPL_SIZE                                  \
73                         (RING_CMPL_DESC_COUNT * RING_DESC_SIZE)
74 #define RING_VER_MAGIC                                  0x76303031
75
76 /* Per-Ring register offsets */
77 #define RING_VER                                        0x000
78 #define RING_BD_START_ADDR                              0x004
79 #define RING_BD_READ_PTR                                0x008
80 #define RING_BD_WRITE_PTR                               0x00c
81 #define RING_BD_READ_PTR_DDR_LS                         0x010
82 #define RING_BD_READ_PTR_DDR_MS                         0x014
83 #define RING_CMPL_START_ADDR                            0x018
84 #define RING_CMPL_WRITE_PTR                             0x01c
85 #define RING_NUM_REQ_RECV_LS                            0x020
86 #define RING_NUM_REQ_RECV_MS                            0x024
87 #define RING_NUM_REQ_TRANS_LS                           0x028
88 #define RING_NUM_REQ_TRANS_MS                           0x02c
89 #define RING_NUM_REQ_OUTSTAND                           0x030
90 #define RING_CONTROL                                    0x034
91 #define RING_FLUSH_DONE                                 0x038
92 #define RING_MSI_ADDR_LS                                0x03c
93 #define RING_MSI_ADDR_MS                                0x040
94 #define RING_MSI_CONTROL                                0x048
95 #define RING_BD_READ_PTR_DDR_CONTROL                    0x04c
96 #define RING_MSI_DATA_VALUE                             0x064
97
98 /* Register RING_BD_START_ADDR fields */
99 #define BD_LAST_UPDATE_HW_SHIFT                         28
100 #define BD_LAST_UPDATE_HW_MASK                          0x1
101 #define BD_START_ADDR_VALUE(pa)                         \
102         ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff))
103 #define BD_START_ADDR_DECODE(val)                       \
104         ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER)
105
106 /* Register RING_CMPL_START_ADDR fields */
107 #define CMPL_START_ADDR_VALUE(pa)                       \
108         ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff))
109
110 /* Register RING_CONTROL fields */
111 #define CONTROL_MASK_DISABLE_CONTROL                    12
112 #define CONTROL_FLUSH_SHIFT                             5
113 #define CONTROL_ACTIVE_SHIFT                            4
114 #define CONTROL_RATE_ADAPT_MASK                         0xf
115 #define CONTROL_RATE_DYNAMIC                            0x0
116 #define CONTROL_RATE_FAST                               0x8
117 #define CONTROL_RATE_MEDIUM                             0x9
118 #define CONTROL_RATE_SLOW                               0xa
119 #define CONTROL_RATE_IDLE                               0xb
120
121 /* Register RING_FLUSH_DONE fields */
122 #define FLUSH_DONE_MASK                                 0x1
123
124 /* Register RING_MSI_CONTROL fields */
125 #define MSI_TIMER_VAL_SHIFT                             16
126 #define MSI_TIMER_VAL_MASK                              0xffff
127 #define MSI_ENABLE_SHIFT                                15
128 #define MSI_ENABLE_MASK                                 0x1
129 #define MSI_COUNT_SHIFT                                 0
130 #define MSI_COUNT_MASK                                  0x3ff
131
132 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */
133 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT                 16
134 #define BD_READ_PTR_DDR_TIMER_VAL_MASK                  0xffff
135 #define BD_READ_PTR_DDR_ENABLE_SHIFT                    15
136 #define BD_READ_PTR_DDR_ENABLE_MASK                     0x1
137
138 /* ====== FlexRM ring descriptor defines ===== */
139
140 /* Completion descriptor format */
141 #define CMPL_OPAQUE_SHIFT                       0
142 #define CMPL_OPAQUE_MASK                        0xffff
143 #define CMPL_ENGINE_STATUS_SHIFT                16
144 #define CMPL_ENGINE_STATUS_MASK                 0xffff
145 #define CMPL_DME_STATUS_SHIFT                   32
146 #define CMPL_DME_STATUS_MASK                    0xffff
147 #define CMPL_RM_STATUS_SHIFT                    48
148 #define CMPL_RM_STATUS_MASK                     0xffff
149
150 /* Completion DME status code */
151 #define DME_STATUS_MEM_COR_ERR                  BIT(0)
152 #define DME_STATUS_MEM_UCOR_ERR                 BIT(1)
153 #define DME_STATUS_FIFO_UNDERFLOW               BIT(2)
154 #define DME_STATUS_FIFO_OVERFLOW                BIT(3)
155 #define DME_STATUS_RRESP_ERR                    BIT(4)
156 #define DME_STATUS_BRESP_ERR                    BIT(5)
157 #define DME_STATUS_ERROR_MASK                   (DME_STATUS_MEM_COR_ERR | \
158                                                  DME_STATUS_MEM_UCOR_ERR | \
159                                                  DME_STATUS_FIFO_UNDERFLOW | \
160                                                  DME_STATUS_FIFO_OVERFLOW | \
161                                                  DME_STATUS_RRESP_ERR | \
162                                                  DME_STATUS_BRESP_ERR)
163
164 /* Completion RM status code */
165 #define RM_STATUS_CODE_SHIFT                    0
166 #define RM_STATUS_CODE_MASK                     0x3ff
167 #define RM_STATUS_CODE_GOOD                     0x0
168 #define RM_STATUS_CODE_AE_TIMEOUT               0x3ff
169
170 /* General descriptor format */
171 #define DESC_TYPE_SHIFT                         60
172 #define DESC_TYPE_MASK                          0xf
173 #define DESC_PAYLOAD_SHIFT                      0
174 #define DESC_PAYLOAD_MASK                       0x0fffffffffffffff
175
176 /* Null descriptor format  */
177 #define NULL_TYPE                               0
178 #define NULL_TOGGLE_SHIFT                       58
179 #define NULL_TOGGLE_MASK                        0x1
180
181 /* Header descriptor format */
182 #define HEADER_TYPE                             1
183 #define HEADER_TOGGLE_SHIFT                     58
184 #define HEADER_TOGGLE_MASK                      0x1
185 #define HEADER_ENDPKT_SHIFT                     57
186 #define HEADER_ENDPKT_MASK                      0x1
187 #define HEADER_STARTPKT_SHIFT                   56
188 #define HEADER_STARTPKT_MASK                    0x1
189 #define HEADER_BDCOUNT_SHIFT                    36
190 #define HEADER_BDCOUNT_MASK                     0x1f
191 #define HEADER_BDCOUNT_MAX                      HEADER_BDCOUNT_MASK
192 #define HEADER_FLAGS_SHIFT                      16
193 #define HEADER_FLAGS_MASK                       0xffff
194 #define HEADER_OPAQUE_SHIFT                     0
195 #define HEADER_OPAQUE_MASK                      0xffff
196
197 /* Source (SRC) descriptor format */
198 #define SRC_TYPE                                2
199 #define SRC_LENGTH_SHIFT                        44
200 #define SRC_LENGTH_MASK                         0xffff
201 #define SRC_ADDR_SHIFT                          0
202 #define SRC_ADDR_MASK                           0x00000fffffffffff
203
204 /* Destination (DST) descriptor format */
205 #define DST_TYPE                                3
206 #define DST_LENGTH_SHIFT                        44
207 #define DST_LENGTH_MASK                         0xffff
208 #define DST_ADDR_SHIFT                          0
209 #define DST_ADDR_MASK                           0x00000fffffffffff
210
211 /* Immediate (IMM) descriptor format */
212 #define IMM_TYPE                                4
213 #define IMM_DATA_SHIFT                          0
214 #define IMM_DATA_MASK                           0x0fffffffffffffff
215
216 /* Next pointer (NPTR) descriptor format */
217 #define NPTR_TYPE                               5
218 #define NPTR_TOGGLE_SHIFT                       58
219 #define NPTR_TOGGLE_MASK                        0x1
220 #define NPTR_ADDR_SHIFT                         0
221 #define NPTR_ADDR_MASK                          0x00000fffffffffff
222
223 /* Mega source (MSRC) descriptor format */
224 #define MSRC_TYPE                               6
225 #define MSRC_LENGTH_SHIFT                       44
226 #define MSRC_LENGTH_MASK                        0xffff
227 #define MSRC_ADDR_SHIFT                         0
228 #define MSRC_ADDR_MASK                          0x00000fffffffffff
229
230 /* Mega destination (MDST) descriptor format */
231 #define MDST_TYPE                               7
232 #define MDST_LENGTH_SHIFT                       44
233 #define MDST_LENGTH_MASK                        0xffff
234 #define MDST_ADDR_SHIFT                         0
235 #define MDST_ADDR_MASK                          0x00000fffffffffff
236
237 /* Source with tlast (SRCT) descriptor format */
238 #define SRCT_TYPE                               8
239 #define SRCT_LENGTH_SHIFT                       44
240 #define SRCT_LENGTH_MASK                        0xffff
241 #define SRCT_ADDR_SHIFT                         0
242 #define SRCT_ADDR_MASK                          0x00000fffffffffff
243
244 /* Destination with tlast (DSTT) descriptor format */
245 #define DSTT_TYPE                               9
246 #define DSTT_LENGTH_SHIFT                       44
247 #define DSTT_LENGTH_MASK                        0xffff
248 #define DSTT_ADDR_SHIFT                         0
249 #define DSTT_ADDR_MASK                          0x00000fffffffffff
250
251 /* Immediate with tlast (IMMT) descriptor format */
252 #define IMMT_TYPE                               10
253 #define IMMT_DATA_SHIFT                         0
254 #define IMMT_DATA_MASK                          0x0fffffffffffffff
255
256 /* Descriptor helper macros */
257 #define DESC_DEC(_d, _s, _m)                    (((_d) >> (_s)) & (_m))
258 #define DESC_ENC(_d, _v, _s, _m)                \
259                         do { \
260                                 (_d) &= ~((u64)(_m) << (_s)); \
261                                 (_d) |= (((u64)(_v) & (_m)) << (_s)); \
262                         } while (0)
263
264 /* ====== FlexRM data structures ===== */
265
266 struct flexrm_ring {
267         /* Unprotected members */
268         int num;
269         struct flexrm_mbox *mbox;
270         void __iomem *regs;
271         bool irq_requested;
272         unsigned int irq;
273         cpumask_t irq_aff_hint;
274         unsigned int msi_timer_val;
275         unsigned int msi_count_threshold;
276         struct brcm_message *requests[RING_MAX_REQ_COUNT];
277         void *bd_base;
278         dma_addr_t bd_dma_base;
279         u32 bd_write_offset;
280         void *cmpl_base;
281         dma_addr_t cmpl_dma_base;
282         /* Atomic stats */
283         atomic_t msg_send_count;
284         atomic_t msg_cmpl_count;
285         /* Protected members */
286         spinlock_t lock;
287         DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT);
288         u32 cmpl_read_offset;
289 };
290
291 struct flexrm_mbox {
292         struct device *dev;
293         void __iomem *regs;
294         u32 num_rings;
295         struct flexrm_ring *rings;
296         struct dma_pool *bd_pool;
297         struct dma_pool *cmpl_pool;
298         struct dentry *root;
299         struct mbox_controller controller;
300 };
301
302 /* ====== FlexRM ring descriptor helper routines ===== */
303
304 static u64 flexrm_read_desc(void *desc_ptr)
305 {
306         return le64_to_cpu(*((u64 *)desc_ptr));
307 }
308
309 static void flexrm_write_desc(void *desc_ptr, u64 desc)
310 {
311         *((u64 *)desc_ptr) = cpu_to_le64(desc);
312 }
313
314 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc)
315 {
316         return (u32)(cmpl_desc & CMPL_OPAQUE_MASK);
317 }
318
319 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc)
320 {
321         u32 status;
322
323         status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT,
324                           CMPL_DME_STATUS_MASK);
325         if (status & DME_STATUS_ERROR_MASK)
326                 return -EIO;
327
328         status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT,
329                           CMPL_RM_STATUS_MASK);
330         status &= RM_STATUS_CODE_MASK;
331         if (status == RM_STATUS_CODE_AE_TIMEOUT)
332                 return -ETIMEDOUT;
333
334         return 0;
335 }
336
337 static bool flexrm_is_next_table_desc(void *desc_ptr)
338 {
339         u64 desc = flexrm_read_desc(desc_ptr);
340         u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
341
342         return (type == NPTR_TYPE) ? true : false;
343 }
344
345 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr)
346 {
347         u64 desc = 0;
348
349         DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
350         DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK);
351         DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK);
352
353         return desc;
354 }
355
356 static u64 flexrm_null_desc(u32 toggle)
357 {
358         u64 desc = 0;
359
360         DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
361         DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK);
362
363         return desc;
364 }
365
366 static u32 flexrm_estimate_header_desc_count(u32 nhcnt)
367 {
368         u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX;
369
370         if (!(nhcnt % HEADER_BDCOUNT_MAX))
371                 hcnt += 1;
372
373         return hcnt;
374 }
375
376 static void flexrm_flip_header_toggle(void *desc_ptr)
377 {
378         u64 desc = flexrm_read_desc(desc_ptr);
379
380         if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT))
381                 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT);
382         else
383                 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT);
384
385         flexrm_write_desc(desc_ptr, desc);
386 }
387
388 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt,
389                                u32 bdcount, u32 flags, u32 opaque)
390 {
391         u64 desc = 0;
392
393         DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
394         DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK);
395         DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK);
396         DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK);
397         DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK);
398         DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK);
399         DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK);
400
401         return desc;
402 }
403
404 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid,
405                                  u64 desc, void **desc_ptr, u32 *toggle,
406                                  void *start_desc, void *end_desc)
407 {
408         u64 d;
409         u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount;
410
411         /* Sanity check */
412         if (nhcnt <= nhpos)
413                 return;
414
415         /*
416          * Each request or packet start with a HEADER descriptor followed
417          * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST,
418          * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors
419          * following a HEADER descriptor is represented by BDCOUNT field
420          * of HEADER descriptor. The max value of BDCOUNT field is 31 which
421          * means we can only have 31 non-HEADER descriptors following one
422          * HEADER descriptor.
423          *
424          * In general use, number of non-HEADER descriptors can easily go
425          * beyond 31. To tackle this situation, we have packet (or request)
426          * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor.
427          *
428          * To use packet extension, the first HEADER descriptor of request
429          * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate
430          * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last
431          * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the
432          * TOGGLE bit of the first HEADER will be set to invalid state to
433          * ensure that FlexRM does not start fetching descriptors till all
434          * descriptors are enqueued. The user of this function will flip
435          * the TOGGLE bit of first HEADER after all descriptors are
436          * enqueued.
437          */
438
439         if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) {
440                 /* Prepare the header descriptor */
441                 nhavail = (nhcnt - nhpos);
442                 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle);
443                 _startpkt = (nhpos == 0) ? 0x1 : 0x0;
444                 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0;
445                 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ?
446                                 nhavail : HEADER_BDCOUNT_MAX;
447                 if (nhavail <= HEADER_BDCOUNT_MAX)
448                         _bdcount = nhavail;
449                 else
450                         _bdcount = HEADER_BDCOUNT_MAX;
451                 d = flexrm_header_desc(_toggle, _startpkt, _endpkt,
452                                         _bdcount, 0x0, reqid);
453
454                 /* Write header descriptor */
455                 flexrm_write_desc(*desc_ptr, d);
456
457                 /* Point to next descriptor */
458                 *desc_ptr += sizeof(desc);
459                 if (*desc_ptr == end_desc)
460                         *desc_ptr = start_desc;
461
462                 /* Skip next pointer descriptors */
463                 while (flexrm_is_next_table_desc(*desc_ptr)) {
464                         *toggle = (*toggle) ? 0 : 1;
465                         *desc_ptr += sizeof(desc);
466                         if (*desc_ptr == end_desc)
467                                 *desc_ptr = start_desc;
468                 }
469         }
470
471         /* Write desired descriptor */
472         flexrm_write_desc(*desc_ptr, desc);
473
474         /* Point to next descriptor */
475         *desc_ptr += sizeof(desc);
476         if (*desc_ptr == end_desc)
477                 *desc_ptr = start_desc;
478
479         /* Skip next pointer descriptors */
480         while (flexrm_is_next_table_desc(*desc_ptr)) {
481                 *toggle = (*toggle) ? 0 : 1;
482                 *desc_ptr += sizeof(desc);
483                 if (*desc_ptr == end_desc)
484                         *desc_ptr = start_desc;
485         }
486 }
487
488 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length)
489 {
490         u64 desc = 0;
491
492         DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
493         DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK);
494         DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK);
495
496         return desc;
497 }
498
499 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16)
500 {
501         u64 desc = 0;
502
503         DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
504         DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK);
505         DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK);
506
507         return desc;
508 }
509
510 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length)
511 {
512         u64 desc = 0;
513
514         DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
515         DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK);
516         DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK);
517
518         return desc;
519 }
520
521 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16)
522 {
523         u64 desc = 0;
524
525         DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
526         DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK);
527         DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK);
528
529         return desc;
530 }
531
532 static u64 flexrm_imm_desc(u64 data)
533 {
534         u64 desc = 0;
535
536         DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
537         DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK);
538
539         return desc;
540 }
541
542 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length)
543 {
544         u64 desc = 0;
545
546         DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
547         DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK);
548         DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK);
549
550         return desc;
551 }
552
553 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length)
554 {
555         u64 desc = 0;
556
557         DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
558         DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK);
559         DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK);
560
561         return desc;
562 }
563
564 static u64 flexrm_immt_desc(u64 data)
565 {
566         u64 desc = 0;
567
568         DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK);
569         DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK);
570
571         return desc;
572 }
573
574 static bool flexrm_spu_sanity_check(struct brcm_message *msg)
575 {
576         struct scatterlist *sg;
577
578         if (!msg->spu.src || !msg->spu.dst)
579                 return false;
580         for (sg = msg->spu.src; sg; sg = sg_next(sg)) {
581                 if (sg->length & 0xf) {
582                         if (sg->length > SRC_LENGTH_MASK)
583                                 return false;
584                 } else {
585                         if (sg->length > (MSRC_LENGTH_MASK * 16))
586                                 return false;
587                 }
588         }
589         for (sg = msg->spu.dst; sg; sg = sg_next(sg)) {
590                 if (sg->length & 0xf) {
591                         if (sg->length > DST_LENGTH_MASK)
592                                 return false;
593                 } else {
594                         if (sg->length > (MDST_LENGTH_MASK * 16))
595                                 return false;
596                 }
597         }
598
599         return true;
600 }
601
602 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg)
603 {
604         u32 cnt = 0;
605         unsigned int dst_target = 0;
606         struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
607
608         while (src_sg || dst_sg) {
609                 if (src_sg) {
610                         cnt++;
611                         dst_target = src_sg->length;
612                         src_sg = sg_next(src_sg);
613                 } else
614                         dst_target = UINT_MAX;
615
616                 while (dst_target && dst_sg) {
617                         cnt++;
618                         if (dst_sg->length < dst_target)
619                                 dst_target -= dst_sg->length;
620                         else
621                                 dst_target = 0;
622                         dst_sg = sg_next(dst_sg);
623                 }
624         }
625
626         return cnt;
627 }
628
629 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg)
630 {
631         int rc;
632
633         rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
634                         DMA_TO_DEVICE);
635         if (rc < 0)
636                 return rc;
637
638         rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
639                         DMA_FROM_DEVICE);
640         if (rc < 0) {
641                 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
642                              DMA_TO_DEVICE);
643                 return rc;
644         }
645
646         return 0;
647 }
648
649 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg)
650 {
651         dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst),
652                      DMA_FROM_DEVICE);
653         dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src),
654                      DMA_TO_DEVICE);
655 }
656
657 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt,
658                                      u32 reqid, void *desc_ptr, u32 toggle,
659                                      void *start_desc, void *end_desc)
660 {
661         u64 d;
662         u32 nhpos = 0;
663         void *orig_desc_ptr = desc_ptr;
664         unsigned int dst_target = 0;
665         struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst;
666
667         while (src_sg || dst_sg) {
668                 if (src_sg) {
669                         if (sg_dma_len(src_sg) & 0xf)
670                                 d = flexrm_src_desc(sg_dma_address(src_sg),
671                                                      sg_dma_len(src_sg));
672                         else
673                                 d = flexrm_msrc_desc(sg_dma_address(src_sg),
674                                                       sg_dma_len(src_sg)/16);
675                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
676                                              d, &desc_ptr, &toggle,
677                                              start_desc, end_desc);
678                         nhpos++;
679                         dst_target = sg_dma_len(src_sg);
680                         src_sg = sg_next(src_sg);
681                 } else
682                         dst_target = UINT_MAX;
683
684                 while (dst_target && dst_sg) {
685                         if (sg_dma_len(dst_sg) & 0xf)
686                                 d = flexrm_dst_desc(sg_dma_address(dst_sg),
687                                                      sg_dma_len(dst_sg));
688                         else
689                                 d = flexrm_mdst_desc(sg_dma_address(dst_sg),
690                                                       sg_dma_len(dst_sg)/16);
691                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
692                                              d, &desc_ptr, &toggle,
693                                              start_desc, end_desc);
694                         nhpos++;
695                         if (sg_dma_len(dst_sg) < dst_target)
696                                 dst_target -= sg_dma_len(dst_sg);
697                         else
698                                 dst_target = 0;
699                         dst_sg = sg_next(dst_sg);
700                 }
701         }
702
703         /* Null descriptor with invalid toggle bit */
704         flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
705
706         /* Ensure that descriptors have been written to memory */
707         wmb();
708
709         /* Flip toggle bit in header */
710         flexrm_flip_header_toggle(orig_desc_ptr);
711
712         return desc_ptr;
713 }
714
715 static bool flexrm_sba_sanity_check(struct brcm_message *msg)
716 {
717         u32 i;
718
719         if (!msg->sba.cmds || !msg->sba.cmds_count)
720                 return false;
721
722         for (i = 0; i < msg->sba.cmds_count; i++) {
723                 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
724                      (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) &&
725                     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT))
726                         return false;
727                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) &&
728                     (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
729                         return false;
730                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) &&
731                     (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK))
732                         return false;
733                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) &&
734                     (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK))
735                         return false;
736                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) &&
737                     (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK))
738                         return false;
739         }
740
741         return true;
742 }
743
744 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg)
745 {
746         u32 i, cnt;
747
748         cnt = 0;
749         for (i = 0; i < msg->sba.cmds_count; i++) {
750                 cnt++;
751
752                 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) ||
753                     (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C))
754                         cnt++;
755
756                 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP)
757                         cnt++;
758
759                 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)
760                         cnt++;
761         }
762
763         return cnt;
764 }
765
766 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt,
767                                      u32 reqid, void *desc_ptr, u32 toggle,
768                                      void *start_desc, void *end_desc)
769 {
770         u64 d;
771         u32 i, nhpos = 0;
772         struct brcm_sba_command *c;
773         void *orig_desc_ptr = desc_ptr;
774
775         /* Convert SBA commands into descriptors */
776         for (i = 0; i < msg->sba.cmds_count; i++) {
777                 c = &msg->sba.cmds[i];
778
779                 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) &&
780                     (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) {
781                         /* Destination response descriptor */
782                         d = flexrm_dst_desc(c->resp, c->resp_len);
783                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
784                                              d, &desc_ptr, &toggle,
785                                              start_desc, end_desc);
786                         nhpos++;
787                 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) {
788                         /* Destination response with tlast descriptor */
789                         d = flexrm_dstt_desc(c->resp, c->resp_len);
790                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
791                                              d, &desc_ptr, &toggle,
792                                              start_desc, end_desc);
793                         nhpos++;
794                 }
795
796                 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) {
797                         /* Destination with tlast descriptor */
798                         d = flexrm_dstt_desc(c->data, c->data_len);
799                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
800                                              d, &desc_ptr, &toggle,
801                                              start_desc, end_desc);
802                         nhpos++;
803                 }
804
805                 if (c->flags & BRCM_SBA_CMD_TYPE_B) {
806                         /* Command as immediate descriptor */
807                         d = flexrm_imm_desc(c->cmd);
808                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
809                                              d, &desc_ptr, &toggle,
810                                              start_desc, end_desc);
811                         nhpos++;
812                 } else {
813                         /* Command as immediate descriptor with tlast */
814                         d = flexrm_immt_desc(c->cmd);
815                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
816                                              d, &desc_ptr, &toggle,
817                                              start_desc, end_desc);
818                         nhpos++;
819                 }
820
821                 if ((c->flags & BRCM_SBA_CMD_TYPE_B) ||
822                     (c->flags & BRCM_SBA_CMD_TYPE_C)) {
823                         /* Source with tlast descriptor */
824                         d = flexrm_srct_desc(c->data, c->data_len);
825                         flexrm_enqueue_desc(nhpos, nhcnt, reqid,
826                                              d, &desc_ptr, &toggle,
827                                              start_desc, end_desc);
828                         nhpos++;
829                 }
830         }
831
832         /* Null descriptor with invalid toggle bit */
833         flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle));
834
835         /* Ensure that descriptors have been written to memory */
836         wmb();
837
838         /* Flip toggle bit in header */
839         flexrm_flip_header_toggle(orig_desc_ptr);
840
841         return desc_ptr;
842 }
843
844 static bool flexrm_sanity_check(struct brcm_message *msg)
845 {
846         if (!msg)
847                 return false;
848
849         switch (msg->type) {
850         case BRCM_MESSAGE_SPU:
851                 return flexrm_spu_sanity_check(msg);
852         case BRCM_MESSAGE_SBA:
853                 return flexrm_sba_sanity_check(msg);
854         default:
855                 return false;
856         };
857 }
858
859 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg)
860 {
861         if (!msg)
862                 return 0;
863
864         switch (msg->type) {
865         case BRCM_MESSAGE_SPU:
866                 return flexrm_spu_estimate_nonheader_desc_count(msg);
867         case BRCM_MESSAGE_SBA:
868                 return flexrm_sba_estimate_nonheader_desc_count(msg);
869         default:
870                 return 0;
871         };
872 }
873
874 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg)
875 {
876         if (!dev || !msg)
877                 return -EINVAL;
878
879         switch (msg->type) {
880         case BRCM_MESSAGE_SPU:
881                 return flexrm_spu_dma_map(dev, msg);
882         default:
883                 break;
884         }
885
886         return 0;
887 }
888
889 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg)
890 {
891         if (!dev || !msg)
892                 return;
893
894         switch (msg->type) {
895         case BRCM_MESSAGE_SPU:
896                 flexrm_spu_dma_unmap(dev, msg);
897                 break;
898         default:
899                 break;
900         }
901 }
902
903 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt,
904                                 u32 reqid, void *desc_ptr, u32 toggle,
905                                 void *start_desc, void *end_desc)
906 {
907         if (!msg || !desc_ptr || !start_desc || !end_desc)
908                 return ERR_PTR(-ENOTSUPP);
909
910         if ((desc_ptr < start_desc) || (end_desc <= desc_ptr))
911                 return ERR_PTR(-ERANGE);
912
913         switch (msg->type) {
914         case BRCM_MESSAGE_SPU:
915                 return flexrm_spu_write_descs(msg, nhcnt, reqid,
916                                                desc_ptr, toggle,
917                                                start_desc, end_desc);
918         case BRCM_MESSAGE_SBA:
919                 return flexrm_sba_write_descs(msg, nhcnt, reqid,
920                                                desc_ptr, toggle,
921                                                start_desc, end_desc);
922         default:
923                 return ERR_PTR(-ENOTSUPP);
924         };
925 }
926
927 /* ====== FlexRM driver helper routines ===== */
928
929 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox,
930                                            struct seq_file *file)
931 {
932         int i;
933         const char *state;
934         struct flexrm_ring *ring;
935
936         seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n",
937                    "Ring#", "State", "BD_Addr", "BD_Size",
938                    "Cmpl_Addr", "Cmpl_Size");
939
940         for (i = 0; i < mbox->num_rings; i++) {
941                 ring = &mbox->rings[i];
942                 if (readl(ring->regs + RING_CONTROL) &
943                     BIT(CONTROL_ACTIVE_SHIFT))
944                         state = "active";
945                 else
946                         state = "inactive";
947                 seq_printf(file,
948                            "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n",
949                            ring->num, state,
950                            (unsigned long long)ring->bd_dma_base,
951                            (u32)RING_BD_SIZE,
952                            (unsigned long long)ring->cmpl_dma_base,
953                            (u32)RING_CMPL_SIZE);
954         }
955 }
956
957 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox,
958                                           struct seq_file *file)
959 {
960         int i;
961         u32 val, bd_read_offset;
962         struct flexrm_ring *ring;
963
964         seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n",
965                    "Ring#", "BD_Read", "BD_Write",
966                    "Cmpl_Read", "Submitted", "Completed");
967
968         for (i = 0; i < mbox->num_rings; i++) {
969                 ring = &mbox->rings[i];
970                 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
971                 val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
972                 bd_read_offset *= RING_DESC_SIZE;
973                 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) -
974                                         ring->bd_dma_base);
975                 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n",
976                            ring->num,
977                            (u32)bd_read_offset,
978                            (u32)ring->bd_write_offset,
979                            (u32)ring->cmpl_read_offset,
980                            (u32)atomic_read(&ring->msg_send_count),
981                            (u32)atomic_read(&ring->msg_cmpl_count));
982         }
983 }
984
985 static int flexrm_new_request(struct flexrm_ring *ring,
986                                 struct brcm_message *batch_msg,
987                                 struct brcm_message *msg)
988 {
989         void *next;
990         unsigned long flags;
991         u32 val, count, nhcnt;
992         u32 read_offset, write_offset;
993         bool exit_cleanup = false;
994         int ret = 0, reqid;
995
996         /* Do sanity check on message */
997         if (!flexrm_sanity_check(msg))
998                 return -EIO;
999         msg->error = 0;
1000
1001         /* If no requests possible then save data pointer and goto done. */
1002         spin_lock_irqsave(&ring->lock, flags);
1003         reqid = bitmap_find_free_region(ring->requests_bmap,
1004                                         RING_MAX_REQ_COUNT, 0);
1005         spin_unlock_irqrestore(&ring->lock, flags);
1006         if (reqid < 0)
1007                 return -ENOSPC;
1008         ring->requests[reqid] = msg;
1009
1010         /* Do DMA mappings for the message */
1011         ret = flexrm_dma_map(ring->mbox->dev, msg);
1012         if (ret < 0) {
1013                 ring->requests[reqid] = NULL;
1014                 spin_lock_irqsave(&ring->lock, flags);
1015                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1016                 spin_unlock_irqrestore(&ring->lock, flags);
1017                 return ret;
1018         }
1019
1020         /* Determine current HW BD read offset */
1021         read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR);
1022         val = readl_relaxed(ring->regs + RING_BD_START_ADDR);
1023         read_offset *= RING_DESC_SIZE;
1024         read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base);
1025
1026         /*
1027          * Number required descriptors = number of non-header descriptors +
1028          *                               number of header descriptors +
1029          *                               1x null descriptor
1030          */
1031         nhcnt = flexrm_estimate_nonheader_desc_count(msg);
1032         count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1;
1033
1034         /* Check for available descriptor space. */
1035         write_offset = ring->bd_write_offset;
1036         while (count) {
1037                 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset))
1038                         count--;
1039                 write_offset += RING_DESC_SIZE;
1040                 if (write_offset == RING_BD_SIZE)
1041                         write_offset = 0x0;
1042                 if (write_offset == read_offset)
1043                         break;
1044         }
1045         if (count) {
1046                 ret = -ENOSPC;
1047                 exit_cleanup = true;
1048                 goto exit;
1049         }
1050
1051         /* Write descriptors to ring */
1052         next = flexrm_write_descs(msg, nhcnt, reqid,
1053                         ring->bd_base + ring->bd_write_offset,
1054                         RING_BD_TOGGLE_VALID(ring->bd_write_offset),
1055                         ring->bd_base, ring->bd_base + RING_BD_SIZE);
1056         if (IS_ERR(next)) {
1057                 ret = PTR_ERR(next);
1058                 exit_cleanup = true;
1059                 goto exit;
1060         }
1061
1062         /* Save ring BD write offset */
1063         ring->bd_write_offset = (unsigned long)(next - ring->bd_base);
1064
1065         /* Increment number of messages sent */
1066         atomic_inc_return(&ring->msg_send_count);
1067
1068 exit:
1069         /* Update error status in message */
1070         msg->error = ret;
1071
1072         /* Cleanup if we failed */
1073         if (exit_cleanup) {
1074                 flexrm_dma_unmap(ring->mbox->dev, msg);
1075                 ring->requests[reqid] = NULL;
1076                 spin_lock_irqsave(&ring->lock, flags);
1077                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1078                 spin_unlock_irqrestore(&ring->lock, flags);
1079         }
1080
1081         return ret;
1082 }
1083
1084 static int flexrm_process_completions(struct flexrm_ring *ring)
1085 {
1086         u64 desc;
1087         int err, count = 0;
1088         unsigned long flags;
1089         struct brcm_message *msg = NULL;
1090         u32 reqid, cmpl_read_offset, cmpl_write_offset;
1091         struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num];
1092
1093         spin_lock_irqsave(&ring->lock, flags);
1094
1095         /*
1096          * Get current completion read and write offset
1097          *
1098          * Note: We should read completion write pointer at least once
1099          * after we get a MSI interrupt because HW maintains internal
1100          * MSI status which will allow next MSI interrupt only after
1101          * completion write pointer is read.
1102          */
1103         cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1104         cmpl_write_offset *= RING_DESC_SIZE;
1105         cmpl_read_offset = ring->cmpl_read_offset;
1106         ring->cmpl_read_offset = cmpl_write_offset;
1107
1108         spin_unlock_irqrestore(&ring->lock, flags);
1109
1110         /* For each completed request notify mailbox clients */
1111         reqid = 0;
1112         while (cmpl_read_offset != cmpl_write_offset) {
1113                 /* Dequeue next completion descriptor */
1114                 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset));
1115
1116                 /* Next read offset */
1117                 cmpl_read_offset += RING_DESC_SIZE;
1118                 if (cmpl_read_offset == RING_CMPL_SIZE)
1119                         cmpl_read_offset = 0;
1120
1121                 /* Decode error from completion descriptor */
1122                 err = flexrm_cmpl_desc_to_error(desc);
1123                 if (err < 0) {
1124                         dev_warn(ring->mbox->dev,
1125                         "ring%d got completion desc=0x%lx with error %d\n",
1126                         ring->num, (unsigned long)desc, err);
1127                 }
1128
1129                 /* Determine request id from completion descriptor */
1130                 reqid = flexrm_cmpl_desc_to_reqid(desc);
1131
1132                 /* Determine message pointer based on reqid */
1133                 msg = ring->requests[reqid];
1134                 if (!msg) {
1135                         dev_warn(ring->mbox->dev,
1136                         "ring%d null msg pointer for completion desc=0x%lx\n",
1137                         ring->num, (unsigned long)desc);
1138                         continue;
1139                 }
1140
1141                 /* Release reqid for recycling */
1142                 ring->requests[reqid] = NULL;
1143                 spin_lock_irqsave(&ring->lock, flags);
1144                 bitmap_release_region(ring->requests_bmap, reqid, 0);
1145                 spin_unlock_irqrestore(&ring->lock, flags);
1146
1147                 /* Unmap DMA mappings */
1148                 flexrm_dma_unmap(ring->mbox->dev, msg);
1149
1150                 /* Give-back message to mailbox client */
1151                 msg->error = err;
1152                 mbox_chan_received_data(chan, msg);
1153
1154                 /* Increment number of completions processed */
1155                 atomic_inc_return(&ring->msg_cmpl_count);
1156                 count++;
1157         }
1158
1159         return count;
1160 }
1161
1162 /* ====== FlexRM Debugfs callbacks ====== */
1163
1164 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset)
1165 {
1166         struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1167
1168         /* Write config in file */
1169         flexrm_write_config_in_seqfile(mbox, file);
1170
1171         return 0;
1172 }
1173
1174 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset)
1175 {
1176         struct flexrm_mbox *mbox = dev_get_drvdata(file->private);
1177
1178         /* Write stats in file */
1179         flexrm_write_stats_in_seqfile(mbox, file);
1180
1181         return 0;
1182 }
1183
1184 /* ====== FlexRM interrupt handler ===== */
1185
1186 static irqreturn_t flexrm_irq_event(int irq, void *dev_id)
1187 {
1188         /* We only have MSI for completions so just wakeup IRQ thread */
1189         /* Ring related errors will be informed via completion descriptors */
1190
1191         return IRQ_WAKE_THREAD;
1192 }
1193
1194 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id)
1195 {
1196         flexrm_process_completions(dev_id);
1197
1198         return IRQ_HANDLED;
1199 }
1200
1201 /* ====== FlexRM mailbox callbacks ===== */
1202
1203 static int flexrm_send_data(struct mbox_chan *chan, void *data)
1204 {
1205         int i, rc;
1206         struct flexrm_ring *ring = chan->con_priv;
1207         struct brcm_message *msg = data;
1208
1209         if (msg->type == BRCM_MESSAGE_BATCH) {
1210                 for (i = msg->batch.msgs_queued;
1211                      i < msg->batch.msgs_count; i++) {
1212                         rc = flexrm_new_request(ring, msg,
1213                                                  &msg->batch.msgs[i]);
1214                         if (rc) {
1215                                 msg->error = rc;
1216                                 return rc;
1217                         }
1218                         msg->batch.msgs_queued++;
1219                 }
1220                 return 0;
1221         }
1222
1223         return flexrm_new_request(ring, NULL, data);
1224 }
1225
1226 static bool flexrm_peek_data(struct mbox_chan *chan)
1227 {
1228         int cnt = flexrm_process_completions(chan->con_priv);
1229
1230         return (cnt > 0) ? true : false;
1231 }
1232
1233 static int flexrm_startup(struct mbox_chan *chan)
1234 {
1235         u64 d;
1236         u32 val, off;
1237         int ret = 0;
1238         dma_addr_t next_addr;
1239         struct flexrm_ring *ring = chan->con_priv;
1240
1241         /* Allocate BD memory */
1242         ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool,
1243                                        GFP_KERNEL, &ring->bd_dma_base);
1244         if (!ring->bd_base) {
1245                 dev_err(ring->mbox->dev,
1246                         "can't allocate BD memory for ring%d\n",
1247                         ring->num);
1248                 ret = -ENOMEM;
1249                 goto fail;
1250         }
1251
1252         /* Configure next table pointer entries in BD memory */
1253         for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) {
1254                 next_addr = off + RING_DESC_SIZE;
1255                 if (next_addr == RING_BD_SIZE)
1256                         next_addr = 0;
1257                 next_addr += ring->bd_dma_base;
1258                 if (RING_BD_ALIGN_CHECK(next_addr))
1259                         d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off),
1260                                                     next_addr);
1261                 else
1262                         d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off));
1263                 flexrm_write_desc(ring->bd_base + off, d);
1264         }
1265
1266         /* Allocate completion memory */
1267         ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool,
1268                                          GFP_KERNEL, &ring->cmpl_dma_base);
1269         if (!ring->cmpl_base) {
1270                 dev_err(ring->mbox->dev,
1271                         "can't allocate completion memory for ring%d\n",
1272                         ring->num);
1273                 ret = -ENOMEM;
1274                 goto fail_free_bd_memory;
1275         }
1276
1277         /* Request IRQ */
1278         if (ring->irq == UINT_MAX) {
1279                 dev_err(ring->mbox->dev,
1280                         "ring%d IRQ not available\n", ring->num);
1281                 ret = -ENODEV;
1282                 goto fail_free_cmpl_memory;
1283         }
1284         ret = request_threaded_irq(ring->irq,
1285                                    flexrm_irq_event,
1286                                    flexrm_irq_thread,
1287                                    0, dev_name(ring->mbox->dev), ring);
1288         if (ret) {
1289                 dev_err(ring->mbox->dev,
1290                         "failed to request ring%d IRQ\n", ring->num);
1291                 goto fail_free_cmpl_memory;
1292         }
1293         ring->irq_requested = true;
1294
1295         /* Set IRQ affinity hint */
1296         ring->irq_aff_hint = CPU_MASK_NONE;
1297         val = ring->mbox->num_rings;
1298         val = (num_online_cpus() < val) ? val / num_online_cpus() : 1;
1299         cpumask_set_cpu((ring->num / val) % num_online_cpus(),
1300                         &ring->irq_aff_hint);
1301         ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint);
1302         if (ret) {
1303                 dev_err(ring->mbox->dev,
1304                         "failed to set IRQ affinity hint for ring%d\n",
1305                         ring->num);
1306                 goto fail_free_irq;
1307         }
1308
1309         /* Disable/inactivate ring */
1310         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1311
1312         /* Program BD start address */
1313         val = BD_START_ADDR_VALUE(ring->bd_dma_base);
1314         writel_relaxed(val, ring->regs + RING_BD_START_ADDR);
1315
1316         /* BD write pointer will be same as HW write pointer */
1317         ring->bd_write_offset =
1318                         readl_relaxed(ring->regs + RING_BD_WRITE_PTR);
1319         ring->bd_write_offset *= RING_DESC_SIZE;
1320
1321         /* Program completion start address */
1322         val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base);
1323         writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR);
1324
1325         /* Completion read pointer will be same as HW write pointer */
1326         ring->cmpl_read_offset =
1327                         readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR);
1328         ring->cmpl_read_offset *= RING_DESC_SIZE;
1329
1330         /* Read ring Tx, Rx, and Outstanding counts to clear */
1331         readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS);
1332         readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS);
1333         readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS);
1334         readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS);
1335         readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND);
1336
1337         /* Configure RING_MSI_CONTROL */
1338         val = 0;
1339         val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT);
1340         val |= BIT(MSI_ENABLE_SHIFT);
1341         val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT;
1342         writel_relaxed(val, ring->regs + RING_MSI_CONTROL);
1343
1344         /* Enable/activate ring */
1345         val = BIT(CONTROL_ACTIVE_SHIFT);
1346         writel_relaxed(val, ring->regs + RING_CONTROL);
1347
1348         /* Reset stats to zero */
1349         atomic_set(&ring->msg_send_count, 0);
1350         atomic_set(&ring->msg_cmpl_count, 0);
1351
1352         return 0;
1353
1354 fail_free_irq:
1355         free_irq(ring->irq, ring);
1356         ring->irq_requested = false;
1357 fail_free_cmpl_memory:
1358         dma_pool_free(ring->mbox->cmpl_pool,
1359                       ring->cmpl_base, ring->cmpl_dma_base);
1360         ring->cmpl_base = NULL;
1361 fail_free_bd_memory:
1362         dma_pool_free(ring->mbox->bd_pool,
1363                       ring->bd_base, ring->bd_dma_base);
1364         ring->bd_base = NULL;
1365 fail:
1366         return ret;
1367 }
1368
1369 static void flexrm_shutdown(struct mbox_chan *chan)
1370 {
1371         u32 reqid;
1372         unsigned int timeout;
1373         struct brcm_message *msg;
1374         struct flexrm_ring *ring = chan->con_priv;
1375
1376         /* Disable/inactivate ring */
1377         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1378
1379         /* Set ring flush state */
1380         timeout = 1000; /* timeout of 1s */
1381         writel_relaxed(BIT(CONTROL_FLUSH_SHIFT),
1382                         ring->regs + RING_CONTROL);
1383         do {
1384                 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1385                     FLUSH_DONE_MASK)
1386                         break;
1387                 mdelay(1);
1388         } while (--timeout);
1389         if (!timeout)
1390                 dev_err(ring->mbox->dev,
1391                         "setting ring%d flush state timedout\n", ring->num);
1392
1393         /* Clear ring flush state */
1394         timeout = 1000; /* timeout of 1s */
1395         writel_relaxed(0x0, ring->regs + RING_CONTROL);
1396         do {
1397                 if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) &
1398                       FLUSH_DONE_MASK))
1399                         break;
1400                 mdelay(1);
1401         } while (--timeout);
1402         if (!timeout)
1403                 dev_err(ring->mbox->dev,
1404                         "clearing ring%d flush state timedout\n", ring->num);
1405
1406         /* Abort all in-flight requests */
1407         for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) {
1408                 msg = ring->requests[reqid];
1409                 if (!msg)
1410                         continue;
1411
1412                 /* Release reqid for recycling */
1413                 ring->requests[reqid] = NULL;
1414
1415                 /* Unmap DMA mappings */
1416                 flexrm_dma_unmap(ring->mbox->dev, msg);
1417
1418                 /* Give-back message to mailbox client */
1419                 msg->error = -EIO;
1420                 mbox_chan_received_data(chan, msg);
1421         }
1422
1423         /* Clear requests bitmap */
1424         bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1425
1426         /* Release IRQ */
1427         if (ring->irq_requested) {
1428                 irq_update_affinity_hint(ring->irq, NULL);
1429                 free_irq(ring->irq, ring);
1430                 ring->irq_requested = false;
1431         }
1432
1433         /* Free-up completion descriptor ring */
1434         if (ring->cmpl_base) {
1435                 dma_pool_free(ring->mbox->cmpl_pool,
1436                               ring->cmpl_base, ring->cmpl_dma_base);
1437                 ring->cmpl_base = NULL;
1438         }
1439
1440         /* Free-up BD descriptor ring */
1441         if (ring->bd_base) {
1442                 dma_pool_free(ring->mbox->bd_pool,
1443                               ring->bd_base, ring->bd_dma_base);
1444                 ring->bd_base = NULL;
1445         }
1446 }
1447
1448 static const struct mbox_chan_ops flexrm_mbox_chan_ops = {
1449         .send_data      = flexrm_send_data,
1450         .startup        = flexrm_startup,
1451         .shutdown       = flexrm_shutdown,
1452         .peek_data      = flexrm_peek_data,
1453 };
1454
1455 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr,
1456                                         const struct of_phandle_args *pa)
1457 {
1458         struct mbox_chan *chan;
1459         struct flexrm_ring *ring;
1460
1461         if (pa->args_count < 3)
1462                 return ERR_PTR(-EINVAL);
1463
1464         if (pa->args[0] >= cntlr->num_chans)
1465                 return ERR_PTR(-ENOENT);
1466
1467         if (pa->args[1] > MSI_COUNT_MASK)
1468                 return ERR_PTR(-EINVAL);
1469
1470         if (pa->args[2] > MSI_TIMER_VAL_MASK)
1471                 return ERR_PTR(-EINVAL);
1472
1473         chan = &cntlr->chans[pa->args[0]];
1474         ring = chan->con_priv;
1475         ring->msi_count_threshold = pa->args[1];
1476         ring->msi_timer_val = pa->args[2];
1477
1478         return chan;
1479 }
1480
1481 /* ====== FlexRM platform driver ===== */
1482
1483 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg)
1484 {
1485         struct device *dev = msi_desc_to_dev(desc);
1486         struct flexrm_mbox *mbox = dev_get_drvdata(dev);
1487         struct flexrm_ring *ring = &mbox->rings[desc->msi_index];
1488
1489         /* Configure per-Ring MSI registers */
1490         writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS);
1491         writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS);
1492         writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE);
1493 }
1494
1495 static int flexrm_mbox_probe(struct platform_device *pdev)
1496 {
1497         int index, ret = 0;
1498         void __iomem *regs;
1499         void __iomem *regs_end;
1500         struct resource *iomem;
1501         struct flexrm_ring *ring;
1502         struct flexrm_mbox *mbox;
1503         struct device *dev = &pdev->dev;
1504
1505         /* Allocate driver mailbox struct */
1506         mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL);
1507         if (!mbox) {
1508                 ret = -ENOMEM;
1509                 goto fail;
1510         }
1511         mbox->dev = dev;
1512         platform_set_drvdata(pdev, mbox);
1513
1514         /* Get resource for registers */
1515         iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1516         if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) {
1517                 ret = -ENODEV;
1518                 goto fail;
1519         }
1520
1521         /* Map registers of all rings */
1522         mbox->regs = devm_ioremap_resource(&pdev->dev, iomem);
1523         if (IS_ERR(mbox->regs)) {
1524                 ret = PTR_ERR(mbox->regs);
1525                 goto fail;
1526         }
1527         regs_end = mbox->regs + resource_size(iomem);
1528
1529         /* Scan and count available rings */
1530         mbox->num_rings = 0;
1531         for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) {
1532                 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC)
1533                         mbox->num_rings++;
1534         }
1535         if (!mbox->num_rings) {
1536                 ret = -ENODEV;
1537                 goto fail;
1538         }
1539
1540         /* Allocate driver ring structs */
1541         ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL);
1542         if (!ring) {
1543                 ret = -ENOMEM;
1544                 goto fail;
1545         }
1546         mbox->rings = ring;
1547
1548         /* Initialize members of driver ring structs */
1549         regs = mbox->regs;
1550         for (index = 0; index < mbox->num_rings; index++) {
1551                 ring = &mbox->rings[index];
1552                 ring->num = index;
1553                 ring->mbox = mbox;
1554                 while ((regs < regs_end) &&
1555                        (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC))
1556                         regs += RING_REGS_SIZE;
1557                 if (regs_end <= regs) {
1558                         ret = -ENODEV;
1559                         goto fail;
1560                 }
1561                 ring->regs = regs;
1562                 regs += RING_REGS_SIZE;
1563                 ring->irq = UINT_MAX;
1564                 ring->irq_requested = false;
1565                 ring->msi_timer_val = MSI_TIMER_VAL_MASK;
1566                 ring->msi_count_threshold = 0x1;
1567                 memset(ring->requests, 0, sizeof(ring->requests));
1568                 ring->bd_base = NULL;
1569                 ring->bd_dma_base = 0;
1570                 ring->cmpl_base = NULL;
1571                 ring->cmpl_dma_base = 0;
1572                 atomic_set(&ring->msg_send_count, 0);
1573                 atomic_set(&ring->msg_cmpl_count, 0);
1574                 spin_lock_init(&ring->lock);
1575                 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT);
1576                 ring->cmpl_read_offset = 0;
1577         }
1578
1579         /* FlexRM is capable of 40-bit physical addresses only */
1580         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40));
1581         if (ret) {
1582                 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1583                 if (ret)
1584                         goto fail;
1585         }
1586
1587         /* Create DMA pool for ring BD memory */
1588         mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE,
1589                                         1 << RING_BD_ALIGN_ORDER, 0);
1590         if (!mbox->bd_pool) {
1591                 ret = -ENOMEM;
1592                 goto fail;
1593         }
1594
1595         /* Create DMA pool for ring completion memory */
1596         mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE,
1597                                           1 << RING_CMPL_ALIGN_ORDER, 0);
1598         if (!mbox->cmpl_pool) {
1599                 ret = -ENOMEM;
1600                 goto fail_destroy_bd_pool;
1601         }
1602
1603         /* Allocate platform MSIs for each ring */
1604         ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings,
1605                                                 flexrm_mbox_msi_write);
1606         if (ret)
1607                 goto fail_destroy_cmpl_pool;
1608
1609         /* Save alloced IRQ numbers for each ring */
1610         for (index = 0; index < mbox->num_rings; index++)
1611                 mbox->rings[index].irq = msi_get_virq(dev, index);
1612
1613         /* Check availability of debugfs */
1614         if (!debugfs_initialized())
1615                 goto skip_debugfs;
1616
1617         /* Create debugfs root entry */
1618         mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL);
1619
1620         /* Create debugfs config entry */
1621         debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root,
1622                                     flexrm_debugfs_conf_show);
1623
1624         /* Create debugfs stats entry */
1625         debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root,
1626                                     flexrm_debugfs_stats_show);
1627
1628 skip_debugfs:
1629
1630         /* Initialize mailbox controller */
1631         mbox->controller.txdone_irq = false;
1632         mbox->controller.txdone_poll = false;
1633         mbox->controller.ops = &flexrm_mbox_chan_ops;
1634         mbox->controller.dev = dev;
1635         mbox->controller.num_chans = mbox->num_rings;
1636         mbox->controller.of_xlate = flexrm_mbox_of_xlate;
1637         mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings,
1638                                 sizeof(*mbox->controller.chans), GFP_KERNEL);
1639         if (!mbox->controller.chans) {
1640                 ret = -ENOMEM;
1641                 goto fail_free_debugfs_root;
1642         }
1643         for (index = 0; index < mbox->num_rings; index++)
1644                 mbox->controller.chans[index].con_priv = &mbox->rings[index];
1645
1646         /* Register mailbox controller */
1647         ret = devm_mbox_controller_register(dev, &mbox->controller);
1648         if (ret)
1649                 goto fail_free_debugfs_root;
1650
1651         dev_info(dev, "registered flexrm mailbox with %d channels\n",
1652                         mbox->controller.num_chans);
1653
1654         return 0;
1655
1656 fail_free_debugfs_root:
1657         debugfs_remove_recursive(mbox->root);
1658         platform_msi_domain_free_irqs(dev);
1659 fail_destroy_cmpl_pool:
1660         dma_pool_destroy(mbox->cmpl_pool);
1661 fail_destroy_bd_pool:
1662         dma_pool_destroy(mbox->bd_pool);
1663 fail:
1664         return ret;
1665 }
1666
1667 static int flexrm_mbox_remove(struct platform_device *pdev)
1668 {
1669         struct device *dev = &pdev->dev;
1670         struct flexrm_mbox *mbox = platform_get_drvdata(pdev);
1671
1672         debugfs_remove_recursive(mbox->root);
1673
1674         platform_msi_domain_free_irqs(dev);
1675
1676         dma_pool_destroy(mbox->cmpl_pool);
1677         dma_pool_destroy(mbox->bd_pool);
1678
1679         return 0;
1680 }
1681
1682 static const struct of_device_id flexrm_mbox_of_match[] = {
1683         { .compatible = "brcm,iproc-flexrm-mbox", },
1684         {},
1685 };
1686 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match);
1687
1688 static struct platform_driver flexrm_mbox_driver = {
1689         .driver = {
1690                 .name = "brcm-flexrm-mbox",
1691                 .of_match_table = flexrm_mbox_of_match,
1692         },
1693         .probe          = flexrm_mbox_probe,
1694         .remove         = flexrm_mbox_remove,
1695 };
1696 module_platform_driver(flexrm_mbox_driver);
1697
1698 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
1699 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver");
1700 MODULE_LICENSE("GPL v2");