2 * Copyright(c) 2015, 2016 Intel Corporation.
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
50 /* additive distance between non-SOP and SOP space */
51 #define SOP_DISTANCE (TXE_PIO_SIZE / 2)
52 #define PIO_BLOCK_MASK (PIO_BLOCK_SIZE - 1)
53 /* number of QUADWORDs in a block */
54 #define PIO_BLOCK_QWS (PIO_BLOCK_SIZE / sizeof(u64))
57 * pio_copy - copy data block to MMIO space
58 * @pbuf: a number of blocks allocated within a PIO send context
60 * @from: source, must be 8 byte aligned
61 * @count: number of DWORD (32-bit) quantities to copy from source
63 * Copy data from source to PIO Send Buffer memory, 8 bytes at a time.
64 * Must always write full BLOCK_SIZE bytes blocks. The first block must
65 * be written to the corresponding SOP=1 address.
68 * o pbuf->start always starts on a block boundary
69 * o pbuf can wrap only at a block boundary
71 void pio_copy(struct hfi1_devdata *dd, struct pio_buf *pbuf, u64 pbc,
72 const void *from, size_t count)
74 void __iomem *dest = pbuf->start + SOP_DISTANCE;
75 void __iomem *send = dest + PIO_BLOCK_SIZE;
76 void __iomem *dend; /* 8-byte data end */
82 /* calculate where the QWORD data ends - in SOP=1 space */
83 dend = dest + ((count >> 1) * sizeof(u64));
87 * all QWORD data is within the SOP block, does *not*
88 * reach the end of the SOP block
92 writeq(*(u64 *)from, dest);
97 * No boundary checks are needed here:
98 * 0. We're not on the SOP block boundary
99 * 1. The possible DWORD dangle will still be within
101 * 2. We cannot wrap except on a block boundary.
104 /* QWORD data extends _to_ or beyond the SOP block */
106 /* write 8-byte SOP chunk data */
107 while (dest < send) {
108 writeq(*(u64 *)from, dest);
112 /* drop out of the SOP range */
113 dest -= SOP_DISTANCE;
114 dend -= SOP_DISTANCE;
117 * If the wrap comes before or matches the data end,
118 * copy until until the wrap, then wrap.
120 * If the data ends at the end of the SOP above and
121 * the buffer wraps, then pbuf->end == dend == dest
122 * and nothing will get written, but we will wrap in
123 * case there is a dangling DWORD.
125 if (pbuf->end <= dend) {
126 while (dest < pbuf->end) {
127 writeq(*(u64 *)from, dest);
136 /* write 8-byte non-SOP, non-wrap chunk data */
137 while (dest < dend) {
138 writeq(*(u64 *)from, dest);
143 /* at this point we have wrapped if we are going to wrap */
145 /* write dangling u32, if any */
150 val.val32[0] = *(u32 *)from;
151 writeq(val.val64, dest);
155 * fill in rest of block, no need to check pbuf->end
156 * as we only wrap on a block boundary
158 while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
163 /* finished with this buffer */
164 this_cpu_dec(*pbuf->sc->buffers_allocated);
169 * Handle carry bytes using shifts and masks.
171 * NOTE: the value the unused portion of carry is expected to always be zero.
175 * "zero" shift - bit shift used to zero out upper bytes. Input is
176 * the count of LSB bytes to preserve.
178 #define zshift(x) (8 * (8 - (x)))
181 * "merge" shift - bit shift used to merge with carry bytes. Input is
182 * the LSB byte count to move beyond.
184 #define mshift(x) (8 * (x))
187 * Jump copy - no-loop copy for < 8 bytes.
189 static inline void jcopy(u8 *dest, const u8 *src, u32 n)
217 * Read nbytes from "from" and and place them in the low bytes
218 * of pbuf->carry. Other bytes are left as-is. Any previous
219 * value in pbuf->carry is lost.
222 * o do not read from from if nbytes is zero
223 * o from may _not_ be u64 aligned.
225 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,
228 pbuf->carry.val64 = 0;
229 jcopy(&pbuf->carry.val8[0], from, nbytes);
230 pbuf->carry_bytes = nbytes;
234 * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
235 * It is expected that the extra read does not overfill carry.
238 * o from may _not_ be u64 aligned
239 * o nbytes may span a QW boundary
241 static inline void read_extra_bytes(struct pio_buf *pbuf,
242 const void *from, unsigned int nbytes)
244 jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);
245 pbuf->carry_bytes += nbytes;
249 * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
250 * Put the unused part of the next 8 bytes of src into the LSB bytes of
251 * pbuf->carry with the upper bytes zeroed..
254 * o result must keep unused bytes zeroed
255 * o src must be u64 aligned
257 static inline void merge_write8(
258 struct pio_buf *pbuf,
265 temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));
267 pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);
271 * Write a quad word using all bytes of carry.
273 static inline void carry8_write8(union mix carry, void __iomem *dest)
275 writeq(carry.val64, dest);
279 * Write a quad word using all the valid bytes of carry. If carry
280 * has zero valid bytes, nothing is written.
281 * Returns 0 on nothing written, non-zero on quad word written.
283 static inline int carry_write8(struct pio_buf *pbuf, void __iomem *dest)
285 if (pbuf->carry_bytes) {
286 /* unused bytes are always kept zeroed, so just write */
287 writeq(pbuf->carry.val64, dest);
295 * Segmented PIO Copy - start
299 * @pbuf: destination buffer
300 * @pbc: the PBC for the PIO buffer
301 * @from: data source, QWORD aligned
302 * @nbytes: bytes to copy
304 void seg_pio_copy_start(struct pio_buf *pbuf, u64 pbc,
305 const void *from, size_t nbytes)
307 void __iomem *dest = pbuf->start + SOP_DISTANCE;
308 void __iomem *send = dest + PIO_BLOCK_SIZE;
309 void __iomem *dend; /* 8-byte data end */
314 /* calculate where the QWORD data ends - in SOP=1 space */
315 dend = dest + ((nbytes >> 3) * sizeof(u64));
319 * all QWORD data is within the SOP block, does *not*
320 * reach the end of the SOP block
323 while (dest < dend) {
324 writeq(*(u64 *)from, dest);
329 * No boundary checks are needed here:
330 * 0. We're not on the SOP block boundary
331 * 1. The possible DWORD dangle will still be within
333 * 2. We cannot wrap except on a block boundary.
336 /* QWORD data extends _to_ or beyond the SOP block */
338 /* write 8-byte SOP chunk data */
339 while (dest < send) {
340 writeq(*(u64 *)from, dest);
344 /* drop out of the SOP range */
345 dest -= SOP_DISTANCE;
346 dend -= SOP_DISTANCE;
349 * If the wrap comes before or matches the data end,
350 * copy until until the wrap, then wrap.
352 * If the data ends at the end of the SOP above and
353 * the buffer wraps, then pbuf->end == dend == dest
354 * and nothing will get written, but we will wrap in
355 * case there is a dangling DWORD.
357 if (pbuf->end <= dend) {
358 while (dest < pbuf->end) {
359 writeq(*(u64 *)from, dest);
368 /* write 8-byte non-SOP, non-wrap chunk data */
369 while (dest < dend) {
370 writeq(*(u64 *)from, dest);
375 /* at this point we have wrapped if we are going to wrap */
377 /* ...but it doesn't matter as we're done writing */
379 /* save dangling bytes, if any */
380 read_low_bytes(pbuf, from, nbytes & 0x7);
382 pbuf->qw_written = 1 /*PBC*/ + (nbytes >> 3);
386 * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
387 * bytes are non-zero.
389 * Whole u64s must be written to the chip, so bytes must be manually merged.
391 * @pbuf: destination buffer
392 * @from: data source, is QWORD aligned.
393 * @nbytes: bytes to copy
395 * Must handle nbytes < 8.
397 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)
399 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
400 void __iomem *dend; /* 8-byte data end */
401 unsigned long qw_to_write = nbytes >> 3;
402 unsigned long bytes_left = nbytes & 0x7;
404 /* calculate 8-byte data end */
405 dend = dest + (qw_to_write * sizeof(u64));
407 if (pbuf->qw_written < PIO_BLOCK_QWS) {
409 * Still within SOP block. We don't need to check for
410 * wrap because we are still in the first block and
411 * can only wrap on block boundaries.
413 void __iomem *send; /* SOP end */
417 * calculate the end of data or end of block, whichever
420 send = pbuf->start + PIO_BLOCK_SIZE;
421 xend = min(send, dend);
423 /* shift up to SOP=1 space */
424 dest += SOP_DISTANCE;
425 xend += SOP_DISTANCE;
427 /* write 8-byte chunk data */
428 while (dest < xend) {
429 merge_write8(pbuf, dest, from);
434 /* shift down to SOP=0 space */
435 dest -= SOP_DISTANCE;
438 * At this point dest could be (either, both, or neither):
444 * If the wrap comes before or matches the data end,
445 * copy until until the wrap, then wrap.
447 * If dest is at the wrap, we will fall into the if,
448 * not do the loop, when wrap.
450 * If the data ends at the end of the SOP above and
451 * the buffer wraps, then pbuf->end == dend == dest
452 * and nothing will get written.
454 if (pbuf->end <= dend) {
455 while (dest < pbuf->end) {
456 merge_write8(pbuf, dest, from);
465 /* write 8-byte non-SOP, non-wrap chunk data */
466 while (dest < dend) {
467 merge_write8(pbuf, dest, from);
472 pbuf->qw_written += qw_to_write;
474 /* handle carry and left-over bytes */
475 if (pbuf->carry_bytes + bytes_left >= 8) {
478 /* there is enough to fill another qw - fill carry */
479 nread = 8 - pbuf->carry_bytes;
480 read_extra_bytes(pbuf, from, nread);
483 * One more write - but need to make sure dest is correct.
484 * Check for wrap and the possibility the write
485 * should be in SOP space.
487 * The two checks immediately below cannot both be true, hence
488 * the else. If we have wrapped, we cannot still be within the
489 * first block. Conversely, if we are still in the first block,
490 * we cannot have wrapped. We do the wrap check first as that
493 /* adjust if we have wrapped */
494 if (dest >= pbuf->end)
496 /* jump to the SOP range if within the first block */
497 else if (pbuf->qw_written < PIO_BLOCK_QWS)
498 dest += SOP_DISTANCE;
500 /* flush out full carry */
501 carry8_write8(pbuf->carry, dest);
504 /* now adjust and read the rest of the bytes into carry */
506 from += nread; /* from is now not aligned */
507 read_low_bytes(pbuf, from, bytes_left);
509 /* not enough to fill another qw, append the rest to carry */
510 read_extra_bytes(pbuf, from, bytes_left);
515 * Mid copy helper, "straight case" - source pointer is 64-bit aligned
516 * with no carry bytes.
518 * @pbuf: destination buffer
519 * @from: data source, is QWORD aligned
520 * @nbytes: bytes to copy
522 * Must handle nbytes < 8.
524 static void mid_copy_straight(struct pio_buf *pbuf,
525 const void *from, size_t nbytes)
527 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
528 void __iomem *dend; /* 8-byte data end */
530 /* calculate 8-byte data end */
531 dend = dest + ((nbytes >> 3) * sizeof(u64));
533 if (pbuf->qw_written < PIO_BLOCK_QWS) {
535 * Still within SOP block. We don't need to check for
536 * wrap because we are still in the first block and
537 * can only wrap on block boundaries.
539 void __iomem *send; /* SOP end */
543 * calculate the end of data or end of block, whichever
546 send = pbuf->start + PIO_BLOCK_SIZE;
547 xend = min(send, dend);
549 /* shift up to SOP=1 space */
550 dest += SOP_DISTANCE;
551 xend += SOP_DISTANCE;
553 /* write 8-byte chunk data */
554 while (dest < xend) {
555 writeq(*(u64 *)from, dest);
560 /* shift down to SOP=0 space */
561 dest -= SOP_DISTANCE;
564 * At this point dest could be (either, both, or neither):
570 * If the wrap comes before or matches the data end,
571 * copy until until the wrap, then wrap.
573 * If dest is at the wrap, we will fall into the if,
574 * not do the loop, when wrap.
576 * If the data ends at the end of the SOP above and
577 * the buffer wraps, then pbuf->end == dend == dest
578 * and nothing will get written.
580 if (pbuf->end <= dend) {
581 while (dest < pbuf->end) {
582 writeq(*(u64 *)from, dest);
591 /* write 8-byte non-SOP, non-wrap chunk data */
592 while (dest < dend) {
593 writeq(*(u64 *)from, dest);
598 /* we know carry_bytes was zero on entry to this routine */
599 read_low_bytes(pbuf, from, nbytes & 0x7);
601 pbuf->qw_written += nbytes >> 3;
605 * Segmented PIO Copy - middle
607 * Must handle any aligned tail and any aligned source with any byte count.
609 * @pbuf: a number of blocks allocated within a PIO send context
611 * @nbytes: number of bytes to copy
613 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)
615 unsigned long from_align = (unsigned long)from & 0x7;
617 if (pbuf->carry_bytes + nbytes < 8) {
618 /* not enough bytes to fill a QW */
619 read_extra_bytes(pbuf, from, nbytes);
624 /* misaligned source pointer - align it */
625 unsigned long to_align;
627 /* bytes to read to align "from" */
628 to_align = 8 - from_align;
631 * In the advance-to-alignment logic below, we do not need
632 * to check if we are using more than nbytes. This is because
633 * if we are here, we already know that carry+nbytes will
634 * fill at least one QW.
636 if (pbuf->carry_bytes + to_align < 8) {
637 /* not enough align bytes to fill a QW */
638 read_extra_bytes(pbuf, from, to_align);
642 /* bytes to fill carry */
643 unsigned long to_fill = 8 - pbuf->carry_bytes;
644 /* bytes left over to be read */
645 unsigned long extra = to_align - to_fill;
649 read_extra_bytes(pbuf, from, to_fill);
652 /* may not be enough valid bytes left to align */
656 /* ...now write carry */
657 dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
660 * The two checks immediately below cannot both be
661 * true, hence the else. If we have wrapped, we
662 * cannot still be within the first block.
663 * Conversely, if we are still in the first block, we
664 * cannot have wrapped. We do the wrap check first
665 * as that is more likely.
667 /* adjust if we've wrapped */
668 if (dest >= pbuf->end)
670 /* jump to SOP range if within the first block */
671 else if (pbuf->qw_written < PIO_BLOCK_QWS)
672 dest += SOP_DISTANCE;
674 carry8_write8(pbuf->carry, dest);
677 /* read any extra bytes to do final alignment */
678 /* this will overwrite anything in pbuf->carry */
679 read_low_bytes(pbuf, from, extra);
683 * If no bytes are left, return early - we are done.
684 * NOTE: This short-circuit is *required* because
685 * "extra" may have been reduced in size and "from"
686 * is not aligned, as required when leaving this
693 /* at this point, from is QW aligned */
696 if (pbuf->carry_bytes)
697 mid_copy_mix(pbuf, from, nbytes);
699 mid_copy_straight(pbuf, from, nbytes);
703 * Segmented PIO Copy - end
705 * Write any remainder (in pbuf->carry) and finish writing the whole block.
707 * @pbuf: a number of blocks allocated within a PIO send context
709 void seg_pio_copy_end(struct pio_buf *pbuf)
711 void __iomem *dest = pbuf->start + (pbuf->qw_written * sizeof(u64));
714 * The two checks immediately below cannot both be true, hence the
715 * else. If we have wrapped, we cannot still be within the first
716 * block. Conversely, if we are still in the first block, we
717 * cannot have wrapped. We do the wrap check first as that is
720 /* adjust if we have wrapped */
721 if (dest >= pbuf->end)
723 /* jump to the SOP range if within the first block */
724 else if (pbuf->qw_written < PIO_BLOCK_QWS)
725 dest += SOP_DISTANCE;
727 /* write final bytes, if any */
728 if (carry_write8(pbuf, dest)) {
731 * NOTE: We do not need to recalculate whether dest needs
732 * SOP_DISTANCE or not.
734 * If we are in the first block and the dangle write
735 * keeps us in the same block, dest will need
736 * to retain SOP_DISTANCE in the loop below.
738 * If we are in the first block and the dangle write pushes
739 * us to the next block, then loop below will not run
740 * and dest is not used. Hence we do not need to update
743 * If we are past the first block, then SOP_DISTANCE
744 * was never added, so there is nothing to do.
748 /* fill in rest of block */
749 while (((unsigned long)dest & PIO_BLOCK_MASK) != 0) {
754 /* finished with this buffer */
755 this_cpu_dec(*pbuf->sc->buffers_allocated);