drivers/gpu/host1x/hw/cdma_hw.c

   1 /*
   2  * Tegra host1x Command DMA
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/slab.h>
  20 #include <linux/scatterlist.h>
  21 #include <linux/dma-mapping.h>
  22
  23 #include "../cdma.h"
  24 #include "../channel.h"
  25 #include "../dev.h"
  26 #include "../debug.h"
  27
  28 /*
  29  * Put the restart at the end of pushbuffer memory
  30  */
  31 static void push_buffer_init(struct push_buffer *pb)
  32 {
  33         *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  34 }
  35
  36 /*
  37  * Increment timedout buffer's syncpt via CPU.
  38  */
  39 static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41 {
  42         struct host1x *host1x = cdma_to_host1x(cdma);
  43         struct push_buffer *pb = &cdma->push_buffer;
  44         unsigned int i;
  45
  46         for (i = 0; i < syncpt_incrs; i++)
  47                 host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49         /* after CPU incr, ensure shadow is up to date */
  50         host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52         /* NOP all the PB slots */
  53         while (nr_slots--) {
  54                 u32 *p = (u32 *)(pb->mapped + getptr);
  55                 *(p++) = HOST1X_OPCODE_NOP;
  56                 *(p++) = HOST1X_OPCODE_NOP;
  57                 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                         &pb->dma, getptr);
  59                 getptr = (getptr + 8) & (pb->size - 1);
  60         }
  61
  62         wmb();
  63 }
  64
  65 /*
  66  * Start channel DMA
  67  */
  68 static void cdma_start(struct host1x_cdma *cdma)
  69 {
  70         struct host1x_channel *ch = cdma_to_channel(cdma);
  71
  72         if (cdma->running)
  73                 return;
  74
  75         cdma->last_pos = cdma->push_buffer.pos;
  76
  77         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  78                          HOST1X_CHANNEL_DMACTRL);
  79
  80         /* set base, put and end pointer */
  81         host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
  82         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  83         host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size + 4,
  84                          HOST1X_CHANNEL_DMAEND);
  85
  86         /* reset GET */
  87         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  88                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  89                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  90                          HOST1X_CHANNEL_DMACTRL);
  91
  92         /* start the command DMA */
  93         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  94
  95         cdma->running = true;
  96 }
  97
  98 /*
  99  * Similar to cdma_start(), but rather than starting from an idle
 100  * state (where DMA GET is set to DMA PUT), on a timeout we restore
 101  * DMA GET from an explicit value (so DMA may again be pending).
 102  */
 103 static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 104 {
 105         struct host1x *host1x = cdma_to_host1x(cdma);
 106         struct host1x_channel *ch = cdma_to_channel(cdma);
 107
 108         if (cdma->running)
 109                 return;
 110
 111         cdma->last_pos = cdma->push_buffer.pos;
 112
 113         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 114                          HOST1X_CHANNEL_DMACTRL);
 115
 116         /* set base, end pointer (all of memory) */
 117         host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
 118         host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size,
 119                          HOST1X_CHANNEL_DMAEND);
 120
 121         /* set GET, by loading the value in PUT (then reset GET) */
 122         host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 123         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 124                          HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 125                          HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 126                          HOST1X_CHANNEL_DMACTRL);
 127
 128         dev_dbg(host1x->dev,
 129                 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 130                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 131                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 132                 cdma->last_pos);
 133
 134         /* deassert GET reset and set PUT */
 135         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 136                          HOST1X_CHANNEL_DMACTRL);
 137         host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 138
 139         /* start the command DMA */
 140         host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 141
 142         cdma->running = true;
 143 }
 144
 145 /*
 146  * Kick channel DMA into action by writing its PUT offset (if it has changed)
 147  */
 148 static void cdma_flush(struct host1x_cdma *cdma)
 149 {
 150         struct host1x_channel *ch = cdma_to_channel(cdma);
 151
 152         if (cdma->push_buffer.pos != cdma->last_pos) {
 153                 host1x_ch_writel(ch, cdma->push_buffer.pos,
 154                                  HOST1X_CHANNEL_DMAPUT);
 155                 cdma->last_pos = cdma->push_buffer.pos;
 156         }
 157 }
 158
 159 static void cdma_stop(struct host1x_cdma *cdma)
 160 {
 161         struct host1x_channel *ch = cdma_to_channel(cdma);
 162
 163         mutex_lock(&cdma->lock);
 164
 165         if (cdma->running) {
 166                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 167                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 168                                  HOST1X_CHANNEL_DMACTRL);
 169                 cdma->running = false;
 170         }
 171
 172         mutex_unlock(&cdma->lock);
 173 }
 174
 175 /*
 176  * Stops both channel's command processor and CDMA immediately.
 177  * Also, tears down the channel and resets corresponding module.
 178  */
 179 static void cdma_freeze(struct host1x_cdma *cdma)
 180 {
 181         struct host1x *host = cdma_to_host1x(cdma);
 182         struct host1x_channel *ch = cdma_to_channel(cdma);
 183         u32 cmdproc_stop;
 184
 185         if (cdma->torndown && !cdma->running) {
 186                 dev_warn(host->dev, "Already torn down\n");
 187                 return;
 188         }
 189
 190         dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 191
 192         cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 193         cmdproc_stop |= BIT(ch->id);
 194         host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 195
 196         dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 197                 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 198                 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 199                 cdma->last_pos);
 200
 201         host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 202                          HOST1X_CHANNEL_DMACTRL);
 203
 204         host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 205
 206         cdma->running = false;
 207         cdma->torndown = true;
 208 }
 209
 210 static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 211 {
 212         struct host1x *host1x = cdma_to_host1x(cdma);
 213         struct host1x_channel *ch = cdma_to_channel(cdma);
 214         u32 cmdproc_stop;
 215
 216         dev_dbg(host1x->dev,
 217                 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 218                 ch->id, getptr);
 219
 220         cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 221         cmdproc_stop &= ~BIT(ch->id);
 222         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 223
 224         cdma->torndown = false;
 225         cdma_timeout_restart(cdma, getptr);
 226 }
 227
 228 /*
 229  * If this timeout fires, it indicates the current sync_queue entry has
 230  * exceeded its TTL and the userctx should be timed out and remaining
 231  * submits already issued cleaned up (future submits return an error).
 232  */
 233 static void cdma_timeout_handler(struct work_struct *work)
 234 {
 235         u32 prev_cmdproc, cmdproc_stop, syncpt_val;
 236         struct host1x_cdma *cdma;
 237         struct host1x *host1x;
 238         struct host1x_channel *ch;
 239
 240         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 241                             timeout.wq);
 242         host1x = cdma_to_host1x(cdma);
 243         ch = cdma_to_channel(cdma);
 244
 245         host1x_debug_dump(cdma_to_host1x(cdma));
 246
 247         mutex_lock(&cdma->lock);
 248
 249         if (!cdma->timeout.client) {
 250                 dev_dbg(host1x->dev,
 251                         "cdma_timeout: expired, but has no clientid\n");
 252                 mutex_unlock(&cdma->lock);
 253                 return;
 254         }
 255
 256         /* stop processing to get a clean snapshot */
 257         prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 258         cmdproc_stop = prev_cmdproc | BIT(ch->id);
 259         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 260
 261         dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
 262                 prev_cmdproc, cmdproc_stop);
 263
 264         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 265
 266         /* has buffer actually completed? */
 267         if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 268                 dev_dbg(host1x->dev,
 269                         "cdma_timeout: expired, but buffer had completed\n");
 270                 /* restore */
 271                 cmdproc_stop = prev_cmdproc & ~(BIT(ch->id));
 272                 host1x_sync_writel(host1x, cmdproc_stop,
 273                                    HOST1X_SYNC_CMDPROC_STOP);
 274                 mutex_unlock(&cdma->lock);
 275                 return;
 276         }
 277
 278         dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 279                  __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 280                  syncpt_val, cdma->timeout.syncpt_val);
 281
 282         /* stop HW, resetting channel/module */
 283         host1x_hw_cdma_freeze(host1x, cdma);
 284
 285         host1x_cdma_update_sync_queue(cdma, ch->dev);
 286         mutex_unlock(&cdma->lock);
 287 }
 288
 289 /*
 290  * Init timeout resources
 291  */
 292 static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 293 {
 294         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 295         cdma->timeout.initialized = true;
 296
 297         return 0;
 298 }
 299
 300 /*
 301  * Clean up timeout resources
 302  */
 303 static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 304 {
 305         if (cdma->timeout.initialized)
 306                 cancel_delayed_work(&cdma->timeout.wq);
 307
 308         cdma->timeout.initialized = false;
 309 }
 310
 311 static const struct host1x_cdma_ops host1x_cdma_ops = {
 312         .start = cdma_start,
 313         .stop = cdma_stop,
 314         .flush = cdma_flush,
 315
 316         .timeout_init = cdma_timeout_init,
 317         .timeout_destroy = cdma_timeout_destroy,
 318         .freeze = cdma_freeze,
 319         .resume = cdma_resume,
 320         .timeout_cpu_incr = cdma_timeout_cpu_incr,
 321 };
 322
 323 static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 324         .init = push_buffer_init,
 325 };