drivers/mtd/nand/gpmi-nand/gpmi-nand.c

   1 /*
   2  * Freescale GPMI NAND Flash Driver
   3  *
   4  * Copyright (C) 2010-2011 Freescale Semiconductor, Inc.
   5  * Copyright (C) 2008 Embedded Alley Solutions, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  20  */
  21 #include <linux/clk.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/module.h>
  25 #include <linux/mtd/partitions.h>
  26 #include <linux/of.h>
  27 #include <linux/of_device.h>
  28 #include <linux/of_mtd.h>
  29 #include "gpmi-nand.h"
  30 #include "bch-regs.h"
  31
  32 /* Resource names for the GPMI NAND driver. */
  33 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME  "gpmi-nand"
  34 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME   "bch"
  35 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME   "bch"
  36
  37 /* add our owner bbt descriptor */
  38 static uint8_t scan_ff_pattern[] = { 0xff };
  39 static struct nand_bbt_descr gpmi_bbt_descr = {
  40         .options        = 0,
  41         .offs           = 0,
  42         .len            = 1,
  43         .pattern        = scan_ff_pattern
  44 };
  45
  46 /*
  47  * We may change the layout if we can get the ECC info from the datasheet,
  48  * else we will use all the (page + OOB).
  49  */
  50 static struct nand_ecclayout gpmi_hw_ecclayout = {
  51         .eccbytes = 0,
  52         .eccpos = { 0, },
  53         .oobfree = { {.offset = 0, .length = 0} }
  54 };
  55
  56 static const struct gpmi_devdata gpmi_devdata_imx23 = {
  57         .type = IS_MX23,
  58         .bch_max_ecc_strength = 20,
  59         .max_chain_delay = 16,
  60 };
  61
  62 static const struct gpmi_devdata gpmi_devdata_imx28 = {
  63         .type = IS_MX28,
  64         .bch_max_ecc_strength = 20,
  65         .max_chain_delay = 16,
  66 };
  67
  68 static const struct gpmi_devdata gpmi_devdata_imx6q = {
  69         .type = IS_MX6Q,
  70         .bch_max_ecc_strength = 40,
  71         .max_chain_delay = 12,
  72 };
  73
  74 static const struct gpmi_devdata gpmi_devdata_imx6sx = {
  75         .type = IS_MX6SX,
  76         .bch_max_ecc_strength = 62,
  77         .max_chain_delay = 12,
  78 };
  79
  80 static irqreturn_t bch_irq(int irq, void *cookie)
  81 {
  82         struct gpmi_nand_data *this = cookie;
  83
  84         gpmi_clear_bch(this);
  85         complete(&this->bch_done);
  86         return IRQ_HANDLED;
  87 }
  88
  89 /*
  90  *  Calculate the ECC strength by hand:
  91  *      E : The ECC strength.
  92  *      G : the length of Galois Field.
  93  *      N : The chunk count of per page.
  94  *      O : the oobsize of the NAND chip.
  95  *      M : the metasize of per page.
  96  *
  97  *      The formula is :
  98  *              E * G * N
  99  *            ------------ <= (O - M)
 100  *                  8
 101  *
 102  *      So, we get E by:
 103  *                    (O - M) * 8
 104  *              E <= -------------
 105  *                       G * N
 106  */
 107 static inline int get_ecc_strength(struct gpmi_nand_data *this)
 108 {
 109         struct bch_geometry *geo = &this->bch_geometry;
 110         struct mtd_info *mtd = &this->mtd;
 111         int ecc_strength;
 112
 113         ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)
 114                         / (geo->gf_len * geo->ecc_chunk_count);
 115
 116         /* We need the minor even number. */
 117         return round_down(ecc_strength, 2);
 118 }
 119
 120 static inline bool gpmi_check_ecc(struct gpmi_nand_data *this)
 121 {
 122         struct bch_geometry *geo = &this->bch_geometry;
 123
 124         /* Do the sanity check. */
 125         if (GPMI_IS_MX23(this) || GPMI_IS_MX28(this)) {
 126                 /* The mx23/mx28 only support the GF13. */
 127                 if (geo->gf_len == 14)
 128                         return false;
 129         }
 130         return geo->ecc_strength <= this->devdata->bch_max_ecc_strength;
 131 }
 132
 133 /*
 134  * If we can get the ECC information from the nand chip, we do not
 135  * need to calculate them ourselves.
 136  *
 137  * We may have available oob space in this case.
 138  */
 139 static bool set_geometry_by_ecc_info(struct gpmi_nand_data *this)
 140 {
 141         struct bch_geometry *geo = &this->bch_geometry;
 142         struct mtd_info *mtd = &this->mtd;
 143         struct nand_chip *chip = mtd->priv;
 144         struct nand_oobfree *of = gpmi_hw_ecclayout.oobfree;
 145         unsigned int block_mark_bit_offset;
 146
 147         if (!(chip->ecc_strength_ds > 0 && chip->ecc_step_ds > 0))
 148                 return false;
 149
 150         switch (chip->ecc_step_ds) {
 151         case SZ_512:
 152                 geo->gf_len = 13;
 153                 break;
 154         case SZ_1K:
 155                 geo->gf_len = 14;
 156                 break;
 157         default:
 158                 dev_err(this->dev,
 159                         "unsupported nand chip. ecc bits : %d, ecc size : %d\n",
 160                         chip->ecc_strength_ds, chip->ecc_step_ds);
 161                 return false;
 162         }
 163         geo->ecc_chunk_size = chip->ecc_step_ds;
 164         geo->ecc_strength = round_up(chip->ecc_strength_ds, 2);
 165         if (!gpmi_check_ecc(this))
 166                 return false;
 167
 168         /* Keep the C >= O */
 169         if (geo->ecc_chunk_size < mtd->oobsize) {
 170                 dev_err(this->dev,
 171                         "unsupported nand chip. ecc size: %d, oob size : %d\n",
 172                         chip->ecc_step_ds, mtd->oobsize);
 173                 return false;
 174         }
 175
 176         /* The default value, see comment in the legacy_set_geometry(). */
 177         geo->metadata_size = 10;
 178
 179         geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
 180
 181         /*
 182          * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:
 183          *
 184          *    |                          P                            |
 185          *    |<----------------------------------------------------->|
 186          *    |                                                       |
 187          *    |                                        (Block Mark)   |
 188          *    |                      P'                      |      | |     |
 189          *    |<-------------------------------------------->|  D   | |  O' |
 190          *    |                                              |<---->| |<--->|
 191          *    V                                              V      V V     V
 192          *    +---+----------+-+----------+-+----------+-+----------+-+-----+
 193          *    | M |   data   |E|   data   |E|   data   |E|   data   |E|     |
 194          *    +---+----------+-+----------+-+----------+-+----------+-+-----+
 195          *                                                   ^              ^
 196          *                                                   |      O       |
 197          *                                                   |<------------>|
 198          *                                                   |              |
 199          *
 200          *      P : the page size for BCH module.
 201          *      E : The ECC strength.
 202          *      G : the length of Galois Field.
 203          *      N : The chunk count of per page.
 204          *      M : the metasize of per page.
 205          *      C : the ecc chunk size, aka the "data" above.
 206          *      P': the nand chip's page size.
 207          *      O : the nand chip's oob size.
 208          *      O': the free oob.
 209          *
 210          *      The formula for P is :
 211          *
 212          *                  E * G * N
 213          *             P = ------------ + P' + M
 214          *                      8
 215          *
 216          * The position of block mark moves forward in the ECC-based view
 217          * of page, and the delta is:
 218          *
 219          *                   E * G * (N - 1)
 220          *             D = (---------------- + M)
 221          *                          8
 222          *
 223          * Please see the comment in legacy_set_geometry().
 224          * With the condition C >= O , we still can get same result.
 225          * So the bit position of the physical block mark within the ECC-based
 226          * view of the page is :
 227          *             (P' - D) * 8
 228          */
 229         geo->page_size = mtd->writesize + geo->metadata_size +
 230                 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
 231
 232         /* The available oob size we have. */
 233         if (geo->page_size < mtd->writesize + mtd->oobsize) {
 234                 of->offset = geo->page_size - mtd->writesize;
 235                 of->length = mtd->oobsize - of->offset;
 236         }
 237
 238         geo->payload_size = mtd->writesize;
 239
 240         geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
 241         geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
 242                                 + ALIGN(geo->ecc_chunk_count, 4);
 243
 244         if (!this->swap_block_mark)
 245                 return true;
 246
 247         /* For bit swap. */
 248         block_mark_bit_offset = mtd->writesize * 8 -
 249                 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
 250                                 + geo->metadata_size * 8);
 251
 252         geo->block_mark_byte_offset = block_mark_bit_offset / 8;
 253         geo->block_mark_bit_offset  = block_mark_bit_offset % 8;
 254         return true;
 255 }
 256
 257 static int legacy_set_geometry(struct gpmi_nand_data *this)
 258 {
 259         struct bch_geometry *geo = &this->bch_geometry;
 260         struct mtd_info *mtd = &this->mtd;
 261         unsigned int metadata_size;
 262         unsigned int status_size;
 263         unsigned int block_mark_bit_offset;
 264
 265         /*
 266          * The size of the metadata can be changed, though we set it to 10
 267          * bytes now. But it can't be too large, because we have to save
 268          * enough space for BCH.
 269          */
 270         geo->metadata_size = 10;
 271
 272         /* The default for the length of Galois Field. */
 273         geo->gf_len = 13;
 274
 275         /* The default for chunk size. */
 276         geo->ecc_chunk_size = 512;
 277         while (geo->ecc_chunk_size < mtd->oobsize) {
 278                 geo->ecc_chunk_size *= 2; /* keep C >= O */
 279                 geo->gf_len = 14;
 280         }
 281
 282         geo->ecc_chunk_count = mtd->writesize / geo->ecc_chunk_size;
 283
 284         /* We use the same ECC strength for all chunks. */
 285         geo->ecc_strength = get_ecc_strength(this);
 286         if (!gpmi_check_ecc(this)) {
 287                 dev_err(this->dev,
 288                         "required ecc strength of the NAND chip: %d is not supported by the GPMI controller (%d)\n",
 289                         geo->ecc_strength,
 290                         this->devdata->bch_max_ecc_strength);
 291                 return -EINVAL;
 292         }
 293
 294         geo->page_size = mtd->writesize + mtd->oobsize;
 295         geo->payload_size = mtd->writesize;
 296
 297         /*
 298          * The auxiliary buffer contains the metadata and the ECC status. The
 299          * metadata is padded to the nearest 32-bit boundary. The ECC status
 300          * contains one byte for every ECC chunk, and is also padded to the
 301          * nearest 32-bit boundary.
 302          */
 303         metadata_size = ALIGN(geo->metadata_size, 4);
 304         status_size   = ALIGN(geo->ecc_chunk_count, 4);
 305
 306         geo->auxiliary_size = metadata_size + status_size;
 307         geo->auxiliary_status_offset = metadata_size;
 308
 309         if (!this->swap_block_mark)
 310                 return 0;
 311
 312         /*
 313          * We need to compute the byte and bit offsets of
 314          * the physical block mark within the ECC-based view of the page.
 315          *
 316          * NAND chip with 2K page shows below:
 317          *                                             (Block Mark)
 318          *                                                   |      |
 319          *                                                   |  D   |
 320          *                                                   |<---->|
 321          *                                                   V      V
 322          *    +---+----------+-+----------+-+----------+-+----------+-+
 323          *    | M |   data   |E|   data   |E|   data   |E|   data   |E|
 324          *    +---+----------+-+----------+-+----------+-+----------+-+
 325          *
 326          * The position of block mark moves forward in the ECC-based view
 327          * of page, and the delta is:
 328          *
 329          *                   E * G * (N - 1)
 330          *             D = (---------------- + M)
 331          *                          8
 332          *
 333          * With the formula to compute the ECC strength, and the condition
 334          *       : C >= O         (C is the ecc chunk size)
 335          *
 336          * It's easy to deduce to the following result:
 337          *
 338          *         E * G       (O - M)      C - M         C - M
 339          *      ----------- <= ------- <=  --------  <  ---------
 340          *           8            N           N          (N - 1)
 341          *
 342          *  So, we get:
 343          *
 344          *                   E * G * (N - 1)
 345          *             D = (---------------- + M) < C
 346          *                          8
 347          *
 348          *  The above inequality means the position of block mark
 349          *  within the ECC-based view of the page is still in the data chunk,
 350          *  and it's NOT in the ECC bits of the chunk.
 351          *
 352          *  Use the following to compute the bit position of the
 353          *  physical block mark within the ECC-based view of the page:
 354          *          (page_size - D) * 8
 355          *
 356          *  --Huang Shijie
 357          */
 358         block_mark_bit_offset = mtd->writesize * 8 -
 359                 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
 360                                 + geo->metadata_size * 8);
 361
 362         geo->block_mark_byte_offset = block_mark_bit_offset / 8;
 363         geo->block_mark_bit_offset  = block_mark_bit_offset % 8;
 364         return 0;
 365 }
 366
 367 int common_nfc_set_geometry(struct gpmi_nand_data *this)
 368 {
 369         if (of_property_read_bool(this->dev->of_node, "fsl,use-minimum-ecc")
 370                 && set_geometry_by_ecc_info(this))
 371                 return 0;
 372         return legacy_set_geometry(this);
 373 }
 374
 375 struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
 376 {
 377         /* We use the DMA channel 0 to access all the nand chips. */
 378         return this->dma_chans[0];
 379 }
 380
 381 /* Can we use the upper's buffer directly for DMA? */
 382 void prepare_data_dma(struct gpmi_nand_data *this, enum dma_data_direction dr)
 383 {
 384         struct scatterlist *sgl = &this->data_sgl;
 385         int ret;
 386
 387         /* first try to map the upper buffer directly */
 388         if (virt_addr_valid(this->upper_buf) &&
 389                 !object_is_on_stack(this->upper_buf)) {
 390                 sg_init_one(sgl, this->upper_buf, this->upper_len);
 391                 ret = dma_map_sg(this->dev, sgl, 1, dr);
 392                 if (ret == 0)
 393                         goto map_fail;
 394
 395                 this->direct_dma_map_ok = true;
 396                 return;
 397         }
 398
 399 map_fail:
 400         /* We have to use our own DMA buffer. */
 401         sg_init_one(sgl, this->data_buffer_dma, this->upper_len);
 402
 403         if (dr == DMA_TO_DEVICE)
 404                 memcpy(this->data_buffer_dma, this->upper_buf, this->upper_len);
 405
 406         dma_map_sg(this->dev, sgl, 1, dr);
 407
 408         this->direct_dma_map_ok = false;
 409 }
 410
 411 /* This will be called after the DMA operation is finished. */
 412 static void dma_irq_callback(void *param)
 413 {
 414         struct gpmi_nand_data *this = param;
 415         struct completion *dma_c = &this->dma_done;
 416
 417         switch (this->dma_type) {
 418         case DMA_FOR_COMMAND:
 419                 dma_unmap_sg(this->dev, &this->cmd_sgl, 1, DMA_TO_DEVICE);
 420                 break;
 421
 422         case DMA_FOR_READ_DATA:
 423                 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_FROM_DEVICE);
 424                 if (this->direct_dma_map_ok == false)
 425                         memcpy(this->upper_buf, this->data_buffer_dma,
 426                                 this->upper_len);
 427                 break;
 428
 429         case DMA_FOR_WRITE_DATA:
 430                 dma_unmap_sg(this->dev, &this->data_sgl, 1, DMA_TO_DEVICE);
 431                 break;
 432
 433         case DMA_FOR_READ_ECC_PAGE:
 434         case DMA_FOR_WRITE_ECC_PAGE:
 435                 /* We have to wait the BCH interrupt to finish. */
 436                 break;
 437
 438         default:
 439                 dev_err(this->dev, "in wrong DMA operation.\n");
 440         }
 441
 442         complete(dma_c);
 443 }
 444
 445 int start_dma_without_bch_irq(struct gpmi_nand_data *this,
 446                                 struct dma_async_tx_descriptor *desc)
 447 {
 448         struct completion *dma_c = &this->dma_done;
 449         unsigned long timeout;
 450
 451         init_completion(dma_c);
 452
 453         desc->callback          = dma_irq_callback;
 454         desc->callback_param    = this;
 455         dmaengine_submit(desc);
 456         dma_async_issue_pending(get_dma_chan(this));
 457
 458         /* Wait for the interrupt from the DMA block. */
 459         timeout = wait_for_completion_timeout(dma_c, msecs_to_jiffies(1000));
 460         if (!timeout) {
 461                 dev_err(this->dev, "DMA timeout, last DMA :%d\n",
 462                         this->last_dma_type);
 463                 gpmi_dump_info(this);
 464                 return -ETIMEDOUT;
 465         }
 466         return 0;
 467 }
 468
 469 /*
 470  * This function is used in BCH reading or BCH writing pages.
 471  * It will wait for the BCH interrupt as long as ONE second.
 472  * Actually, we must wait for two interrupts :
 473  *      [1] firstly the DMA interrupt and
 474  *      [2] secondly the BCH interrupt.
 475  */
 476 int start_dma_with_bch_irq(struct gpmi_nand_data *this,
 477                         struct dma_async_tx_descriptor *desc)
 478 {
 479         struct completion *bch_c = &this->bch_done;
 480         unsigned long timeout;
 481
 482         /* Prepare to receive an interrupt from the BCH block. */
 483         init_completion(bch_c);
 484
 485         /* start the DMA */
 486         start_dma_without_bch_irq(this, desc);
 487
 488         /* Wait for the interrupt from the BCH block. */
 489         timeout = wait_for_completion_timeout(bch_c, msecs_to_jiffies(1000));
 490         if (!timeout) {
 491                 dev_err(this->dev, "BCH timeout, last DMA :%d\n",
 492                         this->last_dma_type);
 493                 gpmi_dump_info(this);
 494                 return -ETIMEDOUT;
 495         }
 496         return 0;
 497 }
 498
 499 static int acquire_register_block(struct gpmi_nand_data *this,
 500                                   const char *res_name)
 501 {
 502         struct platform_device *pdev = this->pdev;
 503         struct resources *res = &this->resources;
 504         struct resource *r;
 505         void __iomem *p;
 506
 507         r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res_name);
 508         p = devm_ioremap_resource(&pdev->dev, r);
 509         if (IS_ERR(p))
 510                 return PTR_ERR(p);
 511
 512         if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))
 513                 res->gpmi_regs = p;
 514         else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))
 515                 res->bch_regs = p;
 516         else
 517                 dev_err(this->dev, "unknown resource name : %s\n", res_name);
 518
 519         return 0;
 520 }
 521
 522 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)
 523 {
 524         struct platform_device *pdev = this->pdev;
 525         const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;
 526         struct resource *r;
 527         int err;
 528
 529         r = platform_get_resource_byname(pdev, IORESOURCE_IRQ, res_name);
 530         if (!r) {
 531                 dev_err(this->dev, "Can't get resource for %s\n", res_name);
 532                 return -ENODEV;
 533         }
 534
 535         err = devm_request_irq(this->dev, r->start, irq_h, 0, res_name, this);
 536         if (err)
 537                 dev_err(this->dev, "error requesting BCH IRQ\n");
 538
 539         return err;
 540 }
 541
 542 static void release_dma_channels(struct gpmi_nand_data *this)
 543 {
 544         unsigned int i;
 545         for (i = 0; i < DMA_CHANS; i++)
 546                 if (this->dma_chans[i]) {
 547                         dma_release_channel(this->dma_chans[i]);
 548                         this->dma_chans[i] = NULL;
 549                 }
 550 }
 551
 552 static int acquire_dma_channels(struct gpmi_nand_data *this)
 553 {
 554         struct platform_device *pdev = this->pdev;
 555         struct dma_chan *dma_chan;
 556
 557         /* request dma channel */
 558         dma_chan = dma_request_slave_channel(&pdev->dev, "rx-tx");
 559         if (!dma_chan) {
 560                 dev_err(this->dev, "Failed to request DMA channel.\n");
 561                 goto acquire_err;
 562         }
 563
 564         this->dma_chans[0] = dma_chan;
 565         return 0;
 566
 567 acquire_err:
 568         release_dma_channels(this);
 569         return -EINVAL;
 570 }
 571
 572 static char *extra_clks_for_mx6q[GPMI_CLK_MAX] = {
 573         "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",
 574 };
 575
 576 static int gpmi_get_clks(struct gpmi_nand_data *this)
 577 {
 578         struct resources *r = &this->resources;
 579         char **extra_clks = NULL;
 580         struct clk *clk;
 581         int err, i;
 582
 583         /* The main clock is stored in the first. */
 584         r->clock[0] = devm_clk_get(this->dev, "gpmi_io");
 585         if (IS_ERR(r->clock[0])) {
 586                 err = PTR_ERR(r->clock[0]);
 587                 goto err_clock;
 588         }
 589
 590         /* Get extra clocks */
 591         if (GPMI_IS_MX6(this))
 592                 extra_clks = extra_clks_for_mx6q;
 593         if (!extra_clks)
 594                 return 0;
 595
 596         for (i = 1; i < GPMI_CLK_MAX; i++) {
 597                 if (extra_clks[i - 1] == NULL)
 598                         break;
 599
 600                 clk = devm_clk_get(this->dev, extra_clks[i - 1]);
 601                 if (IS_ERR(clk)) {
 602                         err = PTR_ERR(clk);
 603                         goto err_clock;
 604                 }
 605
 606                 r->clock[i] = clk;
 607         }
 608
 609         if (GPMI_IS_MX6(this))
 610                 /*
 611                  * Set the default value for the gpmi clock.
 612                  *
 613                  * If you want to use the ONFI nand which is in the
 614                  * Synchronous Mode, you should change the clock as you need.
 615                  */
 616                 clk_set_rate(r->clock[0], 22000000);
 617
 618         return 0;
 619
 620 err_clock:
 621         dev_dbg(this->dev, "failed in finding the clocks.\n");
 622         return err;
 623 }
 624
 625 static int acquire_resources(struct gpmi_nand_data *this)
 626 {
 627         int ret;
 628
 629         ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);
 630         if (ret)
 631                 goto exit_regs;
 632
 633         ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);
 634         if (ret)
 635                 goto exit_regs;
 636
 637         ret = acquire_bch_irq(this, bch_irq);
 638         if (ret)
 639                 goto exit_regs;
 640
 641         ret = acquire_dma_channels(this);
 642         if (ret)
 643                 goto exit_regs;
 644
 645         ret = gpmi_get_clks(this);
 646         if (ret)
 647                 goto exit_clock;
 648         return 0;
 649
 650 exit_clock:
 651         release_dma_channels(this);
 652 exit_regs:
 653         return ret;
 654 }
 655
 656 static void release_resources(struct gpmi_nand_data *this)
 657 {
 658         release_dma_channels(this);
 659 }
 660
 661 static int init_hardware(struct gpmi_nand_data *this)
 662 {
 663         int ret;
 664
 665         /*
 666          * This structure contains the "safe" GPMI timing that should succeed
 667          * with any NAND Flash device
 668          * (although, with less-than-optimal performance).
 669          */
 670         struct nand_timing  safe_timing = {
 671                 .data_setup_in_ns        = 80,
 672                 .data_hold_in_ns         = 60,
 673                 .address_setup_in_ns     = 25,
 674                 .gpmi_sample_delay_in_ns =  6,
 675                 .tREA_in_ns              = -1,
 676                 .tRLOH_in_ns             = -1,
 677                 .tRHOH_in_ns             = -1,
 678         };
 679
 680         /* Initialize the hardwares. */
 681         ret = gpmi_init(this);
 682         if (ret)
 683                 return ret;
 684
 685         this->timing = safe_timing;
 686         return 0;
 687 }
 688
 689 static int read_page_prepare(struct gpmi_nand_data *this,
 690                         void *destination, unsigned length,
 691                         void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
 692                         void **use_virt, dma_addr_t *use_phys)
 693 {
 694         struct device *dev = this->dev;
 695
 696         if (virt_addr_valid(destination)) {
 697                 dma_addr_t dest_phys;
 698
 699                 dest_phys = dma_map_single(dev, destination,
 700                                                 length, DMA_FROM_DEVICE);
 701                 if (dma_mapping_error(dev, dest_phys)) {
 702                         if (alt_size < length) {
 703                                 dev_err(dev, "Alternate buffer is too small\n");
 704                                 return -ENOMEM;
 705                         }
 706                         goto map_failed;
 707                 }
 708                 *use_virt = destination;
 709                 *use_phys = dest_phys;
 710                 this->direct_dma_map_ok = true;
 711                 return 0;
 712         }
 713
 714 map_failed:
 715         *use_virt = alt_virt;
 716         *use_phys = alt_phys;
 717         this->direct_dma_map_ok = false;
 718         return 0;
 719 }
 720
 721 static inline void read_page_end(struct gpmi_nand_data *this,
 722                         void *destination, unsigned length,
 723                         void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
 724                         void *used_virt, dma_addr_t used_phys)
 725 {
 726         if (this->direct_dma_map_ok)
 727                 dma_unmap_single(this->dev, used_phys, length, DMA_FROM_DEVICE);
 728 }
 729
 730 static inline void read_page_swap_end(struct gpmi_nand_data *this,
 731                         void *destination, unsigned length,
 732                         void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
 733                         void *used_virt, dma_addr_t used_phys)
 734 {
 735         if (!this->direct_dma_map_ok)
 736                 memcpy(destination, alt_virt, length);
 737 }
 738
 739 static int send_page_prepare(struct gpmi_nand_data *this,
 740                         const void *source, unsigned length,
 741                         void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
 742                         const void **use_virt, dma_addr_t *use_phys)
 743 {
 744         struct device *dev = this->dev;
 745
 746         if (virt_addr_valid(source)) {
 747                 dma_addr_t source_phys;
 748
 749                 source_phys = dma_map_single(dev, (void *)source, length,
 750                                                 DMA_TO_DEVICE);
 751                 if (dma_mapping_error(dev, source_phys)) {
 752                         if (alt_size < length) {
 753                                 dev_err(dev, "Alternate buffer is too small\n");
 754                                 return -ENOMEM;
 755                         }
 756                         goto map_failed;
 757                 }
 758                 *use_virt = source;
 759                 *use_phys = source_phys;
 760                 return 0;
 761         }
 762 map_failed:
 763         /*
 764          * Copy the content of the source buffer into the alternate
 765          * buffer and set up the return values accordingly.
 766          */
 767         memcpy(alt_virt, source, length);
 768
 769         *use_virt = alt_virt;
 770         *use_phys = alt_phys;
 771         return 0;
 772 }
 773
 774 static void send_page_end(struct gpmi_nand_data *this,
 775                         const void *source, unsigned length,
 776                         void *alt_virt, dma_addr_t alt_phys, unsigned alt_size,
 777                         const void *used_virt, dma_addr_t used_phys)
 778 {
 779         struct device *dev = this->dev;
 780         if (used_virt == source)
 781                 dma_unmap_single(dev, used_phys, length, DMA_TO_DEVICE);
 782 }
 783
 784 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)
 785 {
 786         struct device *dev = this->dev;
 787
 788         if (this->page_buffer_virt && virt_addr_valid(this->page_buffer_virt))
 789                 dma_free_coherent(dev, this->page_buffer_size,
 790                                         this->page_buffer_virt,
 791                                         this->page_buffer_phys);
 792         kfree(this->cmd_buffer);
 793         kfree(this->data_buffer_dma);
 794         kfree(this->raw_buffer);
 795
 796         this->cmd_buffer        = NULL;
 797         this->data_buffer_dma   = NULL;
 798         this->page_buffer_virt  = NULL;
 799         this->page_buffer_size  =  0;
 800 }
 801
 802 /* Allocate the DMA buffers */
 803 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
 804 {
 805         struct bch_geometry *geo = &this->bch_geometry;
 806         struct device *dev = this->dev;
 807         struct mtd_info *mtd = &this->mtd;
 808
 809         /* [1] Allocate a command buffer. PAGE_SIZE is enough. */
 810         this->cmd_buffer = kzalloc(PAGE_SIZE, GFP_DMA | GFP_KERNEL);
 811         if (this->cmd_buffer == NULL)
 812                 goto error_alloc;
 813
 814         /*
 815          * [2] Allocate a read/write data buffer.
 816          *     The gpmi_alloc_dma_buffer can be called twice.
 817          *     We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer
 818          *     is called before the nand_scan_ident; and we allocate a buffer
 819          *     of the real NAND page size when the gpmi_alloc_dma_buffer is
 820          *     called after the nand_scan_ident.
 821          */
 822         this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,
 823                                         GFP_DMA | GFP_KERNEL);
 824         if (this->data_buffer_dma == NULL)
 825                 goto error_alloc;
 826
 827         /*
 828          * [3] Allocate the page buffer.
 829          *
 830          * Both the payload buffer and the auxiliary buffer must appear on
 831          * 32-bit boundaries. We presume the size of the payload buffer is a
 832          * power of two and is much larger than four, which guarantees the
 833          * auxiliary buffer will appear on a 32-bit boundary.
 834          */
 835         this->page_buffer_size = geo->payload_size + geo->auxiliary_size;
 836         this->page_buffer_virt = dma_alloc_coherent(dev, this->page_buffer_size,
 837                                         &this->page_buffer_phys, GFP_DMA);
 838         if (!this->page_buffer_virt)
 839                 goto error_alloc;
 840
 841         this->raw_buffer = kzalloc(mtd->writesize + mtd->oobsize, GFP_KERNEL);
 842         if (!this->raw_buffer)
 843                 goto error_alloc;
 844
 845         /* Slice up the page buffer. */
 846         this->payload_virt = this->page_buffer_virt;
 847         this->payload_phys = this->page_buffer_phys;
 848         this->auxiliary_virt = this->payload_virt + geo->payload_size;
 849         this->auxiliary_phys = this->payload_phys + geo->payload_size;
 850         return 0;
 851
 852 error_alloc:
 853         gpmi_free_dma_buffer(this);
 854         return -ENOMEM;
 855 }
 856
 857 static void gpmi_cmd_ctrl(struct mtd_info *mtd, int data, unsigned int ctrl)
 858 {
 859         struct nand_chip *chip = mtd->priv;
 860         struct gpmi_nand_data *this = chip->priv;
 861         int ret;
 862
 863         /*
 864          * Every operation begins with a command byte and a series of zero or
 865          * more address bytes. These are distinguished by either the Address
 866          * Latch Enable (ALE) or Command Latch Enable (CLE) signals being
 867          * asserted. When MTD is ready to execute the command, it will deassert
 868          * both latch enables.
 869          *
 870          * Rather than run a separate DMA operation for every single byte, we
 871          * queue them up and run a single DMA operation for the entire series
 872          * of command and data bytes. NAND_CMD_NONE means the END of the queue.
 873          */
 874         if ((ctrl & (NAND_ALE | NAND_CLE))) {
 875                 if (data != NAND_CMD_NONE)
 876                         this->cmd_buffer[this->command_length++] = data;
 877                 return;
 878         }
 879
 880         if (!this->command_length)
 881                 return;
 882
 883         ret = gpmi_send_command(this);
 884         if (ret)
 885                 dev_err(this->dev, "Chip: %u, Error %d\n",
 886                         this->current_chip, ret);
 887
 888         this->command_length = 0;
 889 }
 890
 891 static int gpmi_dev_ready(struct mtd_info *mtd)
 892 {
 893         struct nand_chip *chip = mtd->priv;
 894         struct gpmi_nand_data *this = chip->priv;
 895
 896         return gpmi_is_ready(this, this->current_chip);
 897 }
 898
 899 static void gpmi_select_chip(struct mtd_info *mtd, int chipnr)
 900 {
 901         struct nand_chip *chip = mtd->priv;
 902         struct gpmi_nand_data *this = chip->priv;
 903
 904         if ((this->current_chip < 0) && (chipnr >= 0))
 905                 gpmi_begin(this);
 906         else if ((this->current_chip >= 0) && (chipnr < 0))
 907                 gpmi_end(this);
 908
 909         this->current_chip = chipnr;
 910 }
 911
 912 static void gpmi_read_buf(struct mtd_info *mtd, uint8_t *buf, int len)
 913 {
 914         struct nand_chip *chip = mtd->priv;
 915         struct gpmi_nand_data *this = chip->priv;
 916
 917         dev_dbg(this->dev, "len is %d\n", len);
 918         this->upper_buf = buf;
 919         this->upper_len = len;
 920
 921         gpmi_read_data(this);
 922 }
 923
 924 static void gpmi_write_buf(struct mtd_info *mtd, const uint8_t *buf, int len)
 925 {
 926         struct nand_chip *chip = mtd->priv;
 927         struct gpmi_nand_data *this = chip->priv;
 928
 929         dev_dbg(this->dev, "len is %d\n", len);
 930         this->upper_buf = (uint8_t *)buf;
 931         this->upper_len = len;
 932
 933         gpmi_send_data(this);
 934 }
 935
 936 static uint8_t gpmi_read_byte(struct mtd_info *mtd)
 937 {
 938         struct nand_chip *chip = mtd->priv;
 939         struct gpmi_nand_data *this = chip->priv;
 940         uint8_t *buf = this->data_buffer_dma;
 941
 942         gpmi_read_buf(mtd, buf, 1);
 943         return buf[0];
 944 }
 945
 946 /*
 947  * Handles block mark swapping.
 948  * It can be called in swapping the block mark, or swapping it back,
 949  * because the the operations are the same.
 950  */
 951 static void block_mark_swapping(struct gpmi_nand_data *this,
 952                                 void *payload, void *auxiliary)
 953 {
 954         struct bch_geometry *nfc_geo = &this->bch_geometry;
 955         unsigned char *p;
 956         unsigned char *a;
 957         unsigned int  bit;
 958         unsigned char mask;
 959         unsigned char from_data;
 960         unsigned char from_oob;
 961
 962         if (!this->swap_block_mark)
 963                 return;
 964
 965         /*
 966          * If control arrives here, we're swapping. Make some convenience
 967          * variables.
 968          */
 969         bit = nfc_geo->block_mark_bit_offset;
 970         p   = payload + nfc_geo->block_mark_byte_offset;
 971         a   = auxiliary;
 972
 973         /*
 974          * Get the byte from the data area that overlays the block mark. Since
 975          * the ECC engine applies its own view to the bits in the page, the
 976          * physical block mark won't (in general) appear on a byte boundary in
 977          * the data.
 978          */
 979         from_data = (p[0] >> bit) | (p[1] << (8 - bit));
 980
 981         /* Get the byte from the OOB. */
 982         from_oob = a[0];
 983
 984         /* Swap them. */
 985         a[0] = from_data;
 986
 987         mask = (0x1 << bit) - 1;
 988         p[0] = (p[0] & mask) | (from_oob << bit);
 989
 990         mask = ~0 << bit;
 991         p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
 992 }
 993
 994 static int gpmi_ecc_read_page(struct mtd_info *mtd, struct nand_chip *chip,
 995                                 uint8_t *buf, int oob_required, int page)
 996 {
 997         struct gpmi_nand_data *this = chip->priv;
 998         struct bch_geometry *nfc_geo = &this->bch_geometry;
 999         void          *payload_virt;
1000         dma_addr_t    payload_phys;
1001         void          *auxiliary_virt;
1002         dma_addr_t    auxiliary_phys;
1003         unsigned int  i;
1004         unsigned char *status;
1005         unsigned int  max_bitflips = 0;
1006         int           ret;
1007
1008         dev_dbg(this->dev, "page number is : %d\n", page);
1009         ret = read_page_prepare(this, buf, nfc_geo->payload_size,
1010                                         this->payload_virt, this->payload_phys,
1011                                         nfc_geo->payload_size,
1012                                         &payload_virt, &payload_phys);
1013         if (ret) {
1014                 dev_err(this->dev, "Inadequate DMA buffer\n");
1015                 ret = -ENOMEM;
1016                 return ret;
1017         }
1018         auxiliary_virt = this->auxiliary_virt;
1019         auxiliary_phys = this->auxiliary_phys;
1020
1021         /* go! */
1022         ret = gpmi_read_page(this, payload_phys, auxiliary_phys);
1023         read_page_end(this, buf, nfc_geo->payload_size,
1024                         this->payload_virt, this->payload_phys,
1025                         nfc_geo->payload_size,
1026                         payload_virt, payload_phys);
1027         if (ret) {
1028                 dev_err(this->dev, "Error in ECC-based read: %d\n", ret);
1029                 return ret;
1030         }
1031
1032         /* Loop over status bytes, accumulating ECC status. */
1033         status = auxiliary_virt + nfc_geo->auxiliary_status_offset;
1034
1035         read_page_swap_end(this, buf, nfc_geo->payload_size,
1036                            this->payload_virt, this->payload_phys,
1037                            nfc_geo->payload_size,
1038                            payload_virt, payload_phys);
1039
1040         for (i = 0; i < nfc_geo->ecc_chunk_count; i++, status++) {
1041                 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
1042                         continue;
1043
1044                 if (*status == STATUS_UNCORRECTABLE) {
1045                         int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1046                         u8 *eccbuf = this->raw_buffer;
1047                         int offset, bitoffset;
1048                         int eccbytes;
1049                         int flips;
1050
1051                         /* Read ECC bytes into our internal raw_buffer */
1052                         offset = nfc_geo->metadata_size * 8;
1053                         offset += ((8 * nfc_geo->ecc_chunk_size) + eccbits) * (i + 1);
1054                         offset -= eccbits;
1055                         bitoffset = offset % 8;
1056                         eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
1057                         offset /= 8;
1058                         eccbytes -= offset;
1059                         chip->cmdfunc(mtd, NAND_CMD_RNDOUT, offset, -1);
1060                         chip->read_buf(mtd, eccbuf, eccbytes);
1061
1062                         /*
1063                          * ECC data are not byte aligned and we may have
1064                          * in-band data in the first and last byte of
1065                          * eccbuf. Set non-eccbits to one so that
1066                          * nand_check_erased_ecc_chunk() does not count them
1067                          * as bitflips.
1068                          */
1069                         if (bitoffset)
1070                                 eccbuf[0] |= GENMASK(bitoffset - 1, 0);
1071
1072                         bitoffset = (bitoffset + eccbits) % 8;
1073                         if (bitoffset)
1074                                 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);
1075
1076                         /*
1077                          * The ECC hardware has an uncorrectable ECC status
1078                          * code in case we have bitflips in an erased page. As
1079                          * nothing was written into this subpage the ECC is
1080                          * obviously wrong and we can not trust it. We assume
1081                          * at this point that we are reading an erased page and
1082                          * try to correct the bitflips in buffer up to
1083                          * ecc_strength bitflips. If this is a page with random
1084                          * data, we exceed this number of bitflips and have a
1085                          * ECC failure. Otherwise we use the corrected buffer.
1086                          */
1087                         if (i == 0) {
1088                                 /* The first block includes metadata */
1089                                 flips = nand_check_erased_ecc_chunk(
1090                                                 buf + i * nfc_geo->ecc_chunk_size,
1091                                                 nfc_geo->ecc_chunk_size,
1092                                                 eccbuf, eccbytes,
1093                                                 auxiliary_virt,
1094                                                 nfc_geo->metadata_size,
1095                                                 nfc_geo->ecc_strength);
1096                         } else {
1097                                 flips = nand_check_erased_ecc_chunk(
1098                                                 buf + i * nfc_geo->ecc_chunk_size,
1099                                                 nfc_geo->ecc_chunk_size,
1100                                                 eccbuf, eccbytes,
1101                                                 NULL, 0,
1102                                                 nfc_geo->ecc_strength);
1103                         }
1104
1105                         if (flips > 0) {
1106                                 max_bitflips = max_t(unsigned int, max_bitflips,
1107                                                      flips);
1108                                 mtd->ecc_stats.corrected += flips;
1109                                 continue;
1110                         }
1111
1112                         mtd->ecc_stats.failed++;
1113                         continue;
1114                 }
1115
1116                 mtd->ecc_stats.corrected += *status;
1117                 max_bitflips = max_t(unsigned int, max_bitflips, *status);
1118         }
1119
1120         /* handle the block mark swapping */
1121         block_mark_swapping(this, buf, auxiliary_virt);
1122
1123         if (oob_required) {
1124                 /*
1125                  * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()
1126                  * for details about our policy for delivering the OOB.
1127                  *
1128                  * We fill the caller's buffer with set bits, and then copy the
1129                  * block mark to th caller's buffer. Note that, if block mark
1130                  * swapping was necessary, it has already been done, so we can
1131                  * rely on the first byte of the auxiliary buffer to contain
1132                  * the block mark.
1133                  */
1134                 memset(chip->oob_poi, ~0, mtd->oobsize);
1135                 chip->oob_poi[0] = ((uint8_t *) auxiliary_virt)[0];
1136         }
1137
1138         return max_bitflips;
1139 }
1140
1141 /* Fake a virtual small page for the subpage read */
1142 static int gpmi_ecc_read_subpage(struct mtd_info *mtd, struct nand_chip *chip,
1143                         uint32_t offs, uint32_t len, uint8_t *buf, int page)
1144 {
1145         struct gpmi_nand_data *this = chip->priv;
1146         void __iomem *bch_regs = this->resources.bch_regs;
1147         struct bch_geometry old_geo = this->bch_geometry;
1148         struct bch_geometry *geo = &this->bch_geometry;
1149         int size = chip->ecc.size; /* ECC chunk size */
1150         int meta, n, page_size;
1151         u32 r1_old, r2_old, r1_new, r2_new;
1152         unsigned int max_bitflips;
1153         int first, last, marker_pos;
1154         int ecc_parity_size;
1155         int col = 0;
1156         int old_swap_block_mark = this->swap_block_mark;
1157
1158         /* The size of ECC parity */
1159         ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1160
1161         /* Align it with the chunk size */
1162         first = offs / size;
1163         last = (offs + len - 1) / size;
1164
1165         if (this->swap_block_mark) {
1166                 /*
1167                  * Find the chunk which contains the Block Marker.
1168                  * If this chunk is in the range of [first, last],
1169                  * we have to read out the whole page.
1170                  * Why? since we had swapped the data at the position of Block
1171                  * Marker to the metadata which is bound with the chunk 0.
1172                  */
1173                 marker_pos = geo->block_mark_byte_offset / size;
1174                 if (last >= marker_pos && first <= marker_pos) {
1175                         dev_dbg(this->dev,
1176                                 "page:%d, first:%d, last:%d, marker at:%d\n",
1177                                 page, first, last, marker_pos);
1178                         return gpmi_ecc_read_page(mtd, chip, buf, 0, page);
1179                 }
1180         }
1181
1182         meta = geo->metadata_size;
1183         if (first) {
1184                 col = meta + (size + ecc_parity_size) * first;
1185                 chip->cmdfunc(mtd, NAND_CMD_RNDOUT, col, -1);
1186
1187                 meta = 0;
1188                 buf = buf + first * size;
1189         }
1190
1191         /* Save the old environment */
1192         r1_old = r1_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT0);
1193         r2_old = r2_new = readl(bch_regs + HW_BCH_FLASH0LAYOUT1);
1194
1195         /* change the BCH registers and bch_geometry{} */
1196         n = last - first + 1;
1197         page_size = meta + (size + ecc_parity_size) * n;
1198
1199         r1_new &= ~(BM_BCH_FLASH0LAYOUT0_NBLOCKS |
1200                         BM_BCH_FLASH0LAYOUT0_META_SIZE);
1201         r1_new |= BF_BCH_FLASH0LAYOUT0_NBLOCKS(n - 1)
1202                         | BF_BCH_FLASH0LAYOUT0_META_SIZE(meta);
1203         writel(r1_new, bch_regs + HW_BCH_FLASH0LAYOUT0);
1204
1205         r2_new &= ~BM_BCH_FLASH0LAYOUT1_PAGE_SIZE;
1206         r2_new |= BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size);
1207         writel(r2_new, bch_regs + HW_BCH_FLASH0LAYOUT1);
1208
1209         geo->ecc_chunk_count = n;
1210         geo->payload_size = n * size;
1211         geo->page_size = page_size;
1212         geo->auxiliary_status_offset = ALIGN(meta, 4);
1213
1214         dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",
1215                 page, offs, len, col, first, n, page_size);
1216
1217         /* Read the subpage now */
1218         this->swap_block_mark = false;
1219         max_bitflips = gpmi_ecc_read_page(mtd, chip, buf, 0, page);
1220
1221         /* Restore */
1222         writel(r1_old, bch_regs + HW_BCH_FLASH0LAYOUT0);
1223         writel(r2_old, bch_regs + HW_BCH_FLASH0LAYOUT1);
1224         this->bch_geometry = old_geo;
1225         this->swap_block_mark = old_swap_block_mark;
1226
1227         return max_bitflips;
1228 }
1229
1230 static int gpmi_ecc_write_page(struct mtd_info *mtd, struct nand_chip *chip,
1231                                 const uint8_t *buf, int oob_required, int page)
1232 {
1233         struct gpmi_nand_data *this = chip->priv;
1234         struct bch_geometry *nfc_geo = &this->bch_geometry;
1235         const void *payload_virt;
1236         dma_addr_t payload_phys;
1237         const void *auxiliary_virt;
1238         dma_addr_t auxiliary_phys;
1239         int        ret;
1240
1241         dev_dbg(this->dev, "ecc write page.\n");
1242         if (this->swap_block_mark) {
1243                 /*
1244                  * If control arrives here, we're doing block mark swapping.
1245                  * Since we can't modify the caller's buffers, we must copy them
1246                  * into our own.
1247                  */
1248                 memcpy(this->payload_virt, buf, mtd->writesize);
1249                 payload_virt = this->payload_virt;
1250                 payload_phys = this->payload_phys;
1251
1252                 memcpy(this->auxiliary_virt, chip->oob_poi,
1253                                 nfc_geo->auxiliary_size);
1254                 auxiliary_virt = this->auxiliary_virt;
1255                 auxiliary_phys = this->auxiliary_phys;
1256
1257                 /* Handle block mark swapping. */
1258                 block_mark_swapping(this,
1259                                 (void *)payload_virt, (void *)auxiliary_virt);
1260         } else {
1261                 /*
1262                  * If control arrives here, we're not doing block mark swapping,
1263                  * so we can to try and use the caller's buffers.
1264                  */
1265                 ret = send_page_prepare(this,
1266                                 buf, mtd->writesize,
1267                                 this->payload_virt, this->payload_phys,
1268                                 nfc_geo->payload_size,
1269                                 &payload_virt, &payload_phys);
1270                 if (ret) {
1271                         dev_err(this->dev, "Inadequate payload DMA buffer\n");
1272                         return 0;
1273                 }
1274
1275                 ret = send_page_prepare(this,
1276                                 chip->oob_poi, mtd->oobsize,
1277                                 this->auxiliary_virt, this->auxiliary_phys,
1278                                 nfc_geo->auxiliary_size,
1279                                 &auxiliary_virt, &auxiliary_phys);
1280                 if (ret) {
1281                         dev_err(this->dev, "Inadequate auxiliary DMA buffer\n");
1282                         goto exit_auxiliary;
1283                 }
1284         }
1285
1286         /* Ask the NFC. */
1287         ret = gpmi_send_page(this, payload_phys, auxiliary_phys);
1288         if (ret)
1289                 dev_err(this->dev, "Error in ECC-based write: %d\n", ret);
1290
1291         if (!this->swap_block_mark) {
1292                 send_page_end(this, chip->oob_poi, mtd->oobsize,
1293                                 this->auxiliary_virt, this->auxiliary_phys,
1294                                 nfc_geo->auxiliary_size,
1295                                 auxiliary_virt, auxiliary_phys);
1296 exit_auxiliary:
1297                 send_page_end(this, buf, mtd->writesize,
1298                                 this->payload_virt, this->payload_phys,
1299                                 nfc_geo->payload_size,
1300                                 payload_virt, payload_phys);
1301         }
1302
1303         return 0;
1304 }
1305
1306 /*
1307  * There are several places in this driver where we have to handle the OOB and
1308  * block marks. This is the function where things are the most complicated, so
1309  * this is where we try to explain it all. All the other places refer back to
1310  * here.
1311  *
1312  * These are the rules, in order of decreasing importance:
1313  *
1314  * 1) Nothing the caller does can be allowed to imperil the block mark.
1315  *
1316  * 2) In read operations, the first byte of the OOB we return must reflect the
1317  *    true state of the block mark, no matter where that block mark appears in
1318  *    the physical page.
1319  *
1320  * 3) ECC-based read operations return an OOB full of set bits (since we never
1321  *    allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads
1322  *    return).
1323  *
1324  * 4) "Raw" read operations return a direct view of the physical bytes in the
1325  *    page, using the conventional definition of which bytes are data and which
1326  *    are OOB. This gives the caller a way to see the actual, physical bytes
1327  *    in the page, without the distortions applied by our ECC engine.
1328  *
1329  *
1330  * What we do for this specific read operation depends on two questions:
1331  *
1332  * 1) Are we doing a "raw" read, or an ECC-based read?
1333  *
1334  * 2) Are we using block mark swapping or transcription?
1335  *
1336  * There are four cases, illustrated by the following Karnaugh map:
1337  *
1338  *                    |           Raw           |         ECC-based       |
1339  *       -------------+-------------------------+-------------------------+
1340  *                    | Read the conventional   |                         |
1341  *                    | OOB at the end of the   |                         |
1342  *       Swapping     | page and return it. It  |                         |
1343  *                    | contains exactly what   |                         |
1344  *                    | we want.                | Read the block mark and |
1345  *       -------------+-------------------------+ return it in a buffer   |
1346  *                    | Read the conventional   | full of set bits.       |
1347  *                    | OOB at the end of the   |                         |
1348  *                    | page and also the block |                         |
1349  *       Transcribing | mark in the metadata.   |                         |
1350  *                    | Copy the block mark     |                         |
1351  *                    | into the first byte of  |                         |
1352  *                    | the OOB.                |                         |
1353  *       -------------+-------------------------+-------------------------+
1354  *
1355  * Note that we break rule #4 in the Transcribing/Raw case because we're not
1356  * giving an accurate view of the actual, physical bytes in the page (we're
1357  * overwriting the block mark). That's OK because it's more important to follow
1358  * rule #2.
1359  *
1360  * It turns out that knowing whether we want an "ECC-based" or "raw" read is not
1361  * easy. When reading a page, for example, the NAND Flash MTD code calls our
1362  * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an
1363  * ECC-based or raw view of the page is implicit in which function it calls
1364  * (there is a similar pair of ECC-based/raw functions for writing).
1365  */
1366 static int gpmi_ecc_read_oob(struct mtd_info *mtd, struct nand_chip *chip,
1367                                 int page)
1368 {
1369         struct gpmi_nand_data *this = chip->priv;
1370
1371         dev_dbg(this->dev, "page number is %d\n", page);
1372         /* clear the OOB buffer */
1373         memset(chip->oob_poi, ~0, mtd->oobsize);
1374
1375         /* Read out the conventional OOB. */
1376         chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
1377         chip->read_buf(mtd, chip->oob_poi, mtd->oobsize);
1378
1379         /*
1380          * Now, we want to make sure the block mark is correct. In the
1381          * non-transcribing case (!GPMI_IS_MX23()), we already have it.
1382          * Otherwise, we need to explicitly read it.
1383          */
1384         if (GPMI_IS_MX23(this)) {
1385                 /* Read the block mark into the first byte of the OOB buffer. */
1386                 chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1387                 chip->oob_poi[0] = chip->read_byte(mtd);
1388         }
1389
1390         return 0;
1391 }
1392
1393 static int
1394 gpmi_ecc_write_oob(struct mtd_info *mtd, struct nand_chip *chip, int page)
1395 {
1396         struct nand_oobfree *of = mtd->ecclayout->oobfree;
1397         int status = 0;
1398
1399         /* Do we have available oob area? */
1400         if (!of->length)
1401                 return -EPERM;
1402
1403         if (!nand_is_slc(chip))
1404                 return -EPERM;
1405
1406         chip->cmdfunc(mtd, NAND_CMD_SEQIN, mtd->writesize + of->offset, page);
1407         chip->write_buf(mtd, chip->oob_poi + of->offset, of->length);
1408         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1409
1410         status = chip->waitfunc(mtd, chip);
1411         return status & NAND_STATUS_FAIL ? -EIO : 0;
1412 }
1413
1414 /*
1415  * This function reads a NAND page without involving the ECC engine (no HW
1416  * ECC correction).
1417  * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1418  * inline (interleaved with payload DATA), and do not align data chunk on
1419  * byte boundaries.
1420  * We thus need to take care moving the payload data and ECC bits stored in the
1421  * page into the provided buffers, which is why we're using gpmi_copy_bits.
1422  *
1423  * See set_geometry_by_ecc_info inline comments to have a full description
1424  * of the layout used by the GPMI controller.
1425  */
1426 static int gpmi_ecc_read_page_raw(struct mtd_info *mtd,
1427                                   struct nand_chip *chip, uint8_t *buf,
1428                                   int oob_required, int page)
1429 {
1430         struct gpmi_nand_data *this = chip->priv;
1431         struct bch_geometry *nfc_geo = &this->bch_geometry;
1432         int eccsize = nfc_geo->ecc_chunk_size;
1433         int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1434         u8 *tmp_buf = this->raw_buffer;
1435         size_t src_bit_off;
1436         size_t oob_bit_off;
1437         size_t oob_byte_off;
1438         uint8_t *oob = chip->oob_poi;
1439         int step;
1440
1441         chip->read_buf(mtd, tmp_buf,
1442                        mtd->writesize + mtd->oobsize);
1443
1444         /*
1445          * If required, swap the bad block marker and the data stored in the
1446          * metadata section, so that we don't wrongly consider a block as bad.
1447          *
1448          * See the layout description for a detailed explanation on why this
1449          * is needed.
1450          */
1451         if (this->swap_block_mark) {
1452                 u8 swap = tmp_buf[0];
1453
1454                 tmp_buf[0] = tmp_buf[mtd->writesize];
1455                 tmp_buf[mtd->writesize] = swap;
1456         }
1457
1458         /*
1459          * Copy the metadata section into the oob buffer (this section is
1460          * guaranteed to be aligned on a byte boundary).
1461          */
1462         if (oob_required)
1463                 memcpy(oob, tmp_buf, nfc_geo->metadata_size);
1464
1465         oob_bit_off = nfc_geo->metadata_size * 8;
1466         src_bit_off = oob_bit_off;
1467
1468         /* Extract interleaved payload data and ECC bits */
1469         for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1470                 if (buf)
1471                         gpmi_copy_bits(buf, step * eccsize * 8,
1472                                        tmp_buf, src_bit_off,
1473                                        eccsize * 8);
1474                 src_bit_off += eccsize * 8;
1475
1476                 /* Align last ECC block to align a byte boundary */
1477                 if (step == nfc_geo->ecc_chunk_count - 1 &&
1478                     (oob_bit_off + eccbits) % 8)
1479                         eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1480
1481                 if (oob_required)
1482                         gpmi_copy_bits(oob, oob_bit_off,
1483                                        tmp_buf, src_bit_off,
1484                                        eccbits);
1485
1486                 src_bit_off += eccbits;
1487                 oob_bit_off += eccbits;
1488         }
1489
1490         if (oob_required) {
1491                 oob_byte_off = oob_bit_off / 8;
1492
1493                 if (oob_byte_off < mtd->oobsize)
1494                         memcpy(oob + oob_byte_off,
1495                                tmp_buf + mtd->writesize + oob_byte_off,
1496                                mtd->oobsize - oob_byte_off);
1497         }
1498
1499         return 0;
1500 }
1501
1502 /*
1503  * This function writes a NAND page without involving the ECC engine (no HW
1504  * ECC generation).
1505  * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1506  * inline (interleaved with payload DATA), and do not align data chunk on
1507  * byte boundaries.
1508  * We thus need to take care moving the OOB area at the right place in the
1509  * final page, which is why we're using gpmi_copy_bits.
1510  *
1511  * See set_geometry_by_ecc_info inline comments to have a full description
1512  * of the layout used by the GPMI controller.
1513  */
1514 static int gpmi_ecc_write_page_raw(struct mtd_info *mtd,
1515                                    struct nand_chip *chip,
1516                                    const uint8_t *buf,
1517                                    int oob_required, int page)
1518 {
1519         struct gpmi_nand_data *this = chip->priv;
1520         struct bch_geometry *nfc_geo = &this->bch_geometry;
1521         int eccsize = nfc_geo->ecc_chunk_size;
1522         int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1523         u8 *tmp_buf = this->raw_buffer;
1524         uint8_t *oob = chip->oob_poi;
1525         size_t dst_bit_off;
1526         size_t oob_bit_off;
1527         size_t oob_byte_off;
1528         int step;
1529
1530         /*
1531          * Initialize all bits to 1 in case we don't have a buffer for the
1532          * payload or oob data in order to leave unspecified bits of data
1533          * to their initial state.
1534          */
1535         if (!buf || !oob_required)
1536                 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);
1537
1538         /*
1539          * First copy the metadata section (stored in oob buffer) at the
1540          * beginning of the page, as imposed by the GPMI layout.
1541          */
1542         memcpy(tmp_buf, oob, nfc_geo->metadata_size);
1543         oob_bit_off = nfc_geo->metadata_size * 8;
1544         dst_bit_off = oob_bit_off;
1545
1546         /* Interleave payload data and ECC bits */
1547         for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1548                 if (buf)
1549                         gpmi_copy_bits(tmp_buf, dst_bit_off,
1550                                        buf, step * eccsize * 8, eccsize * 8);
1551                 dst_bit_off += eccsize * 8;
1552
1553                 /* Align last ECC block to align a byte boundary */
1554                 if (step == nfc_geo->ecc_chunk_count - 1 &&
1555                     (oob_bit_off + eccbits) % 8)
1556                         eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1557
1558                 if (oob_required)
1559                         gpmi_copy_bits(tmp_buf, dst_bit_off,
1560                                        oob, oob_bit_off, eccbits);
1561
1562                 dst_bit_off += eccbits;
1563                 oob_bit_off += eccbits;
1564         }
1565
1566         oob_byte_off = oob_bit_off / 8;
1567
1568         if (oob_required && oob_byte_off < mtd->oobsize)
1569                 memcpy(tmp_buf + mtd->writesize + oob_byte_off,
1570                        oob + oob_byte_off, mtd->oobsize - oob_byte_off);
1571
1572         /*
1573          * If required, swap the bad block marker and the first byte of the
1574          * metadata section, so that we don't modify the bad block marker.
1575          *
1576          * See the layout description for a detailed explanation on why this
1577          * is needed.
1578          */
1579         if (this->swap_block_mark) {
1580                 u8 swap = tmp_buf[0];
1581
1582                 tmp_buf[0] = tmp_buf[mtd->writesize];
1583                 tmp_buf[mtd->writesize] = swap;
1584         }
1585
1586         chip->write_buf(mtd, tmp_buf, mtd->writesize + mtd->oobsize);
1587
1588         return 0;
1589 }
1590
1591 static int gpmi_ecc_read_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1592                                  int page)
1593 {
1594         chip->cmdfunc(mtd, NAND_CMD_READ0, 0, page);
1595
1596         return gpmi_ecc_read_page_raw(mtd, chip, NULL, 1, page);
1597 }
1598
1599 static int gpmi_ecc_write_oob_raw(struct mtd_info *mtd, struct nand_chip *chip,
1600                                  int page)
1601 {
1602         chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0, page);
1603
1604         return gpmi_ecc_write_page_raw(mtd, chip, NULL, 1, page);
1605 }
1606
1607 static int gpmi_block_markbad(struct mtd_info *mtd, loff_t ofs)
1608 {
1609         struct nand_chip *chip = mtd->priv;
1610         struct gpmi_nand_data *this = chip->priv;
1611         int ret = 0;
1612         uint8_t *block_mark;
1613         int column, page, status, chipnr;
1614
1615         chipnr = (int)(ofs >> chip->chip_shift);
1616         chip->select_chip(mtd, chipnr);
1617
1618         column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;
1619
1620         /* Write the block mark. */
1621         block_mark = this->data_buffer_dma;
1622         block_mark[0] = 0; /* bad block marker */
1623
1624         /* Shift to get page */
1625         page = (int)(ofs >> chip->page_shift);
1626
1627         chip->cmdfunc(mtd, NAND_CMD_SEQIN, column, page);
1628         chip->write_buf(mtd, block_mark, 1);
1629         chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1630
1631         status = chip->waitfunc(mtd, chip);
1632         if (status & NAND_STATUS_FAIL)
1633                 ret = -EIO;
1634
1635         chip->select_chip(mtd, -1);
1636
1637         return ret;
1638 }
1639
1640 static int nand_boot_set_geometry(struct gpmi_nand_data *this)
1641 {
1642         struct boot_rom_geometry *geometry = &this->rom_geometry;
1643
1644         /*
1645          * Set the boot block stride size.
1646          *
1647          * In principle, we should be reading this from the OTP bits, since
1648          * that's where the ROM is going to get it. In fact, we don't have any
1649          * way to read the OTP bits, so we go with the default and hope for the
1650          * best.
1651          */
1652         geometry->stride_size_in_pages = 64;
1653
1654         /*
1655          * Set the search area stride exponent.
1656          *
1657          * In principle, we should be reading this from the OTP bits, since
1658          * that's where the ROM is going to get it. In fact, we don't have any
1659          * way to read the OTP bits, so we go with the default and hope for the
1660          * best.
1661          */
1662         geometry->search_area_stride_exponent = 2;
1663         return 0;
1664 }
1665
1666 static const char  *fingerprint = "STMP";
1667 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
1668 {
1669         struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1670         struct device *dev = this->dev;
1671         struct mtd_info *mtd = &this->mtd;
1672         struct nand_chip *chip = &this->nand;
1673         unsigned int search_area_size_in_strides;
1674         unsigned int stride;
1675         unsigned int page;
1676         uint8_t *buffer = chip->buffers->databuf;
1677         int saved_chip_number;
1678         int found_an_ncb_fingerprint = false;
1679
1680         /* Compute the number of strides in a search area. */
1681         search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1682
1683         saved_chip_number = this->current_chip;
1684         chip->select_chip(mtd, 0);
1685
1686         /*
1687          * Loop through the first search area, looking for the NCB fingerprint.
1688          */
1689         dev_dbg(dev, "Scanning for an NCB fingerprint...\n");
1690
1691         for (stride = 0; stride < search_area_size_in_strides; stride++) {
1692                 /* Compute the page addresses. */
1693                 page = stride * rom_geo->stride_size_in_pages;
1694
1695                 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);
1696
1697                 /*
1698                  * Read the NCB fingerprint. The fingerprint is four bytes long
1699                  * and starts in the 12th byte of the page.
1700                  */
1701                 chip->cmdfunc(mtd, NAND_CMD_READ0, 12, page);
1702                 chip->read_buf(mtd, buffer, strlen(fingerprint));
1703
1704                 /* Look for the fingerprint. */
1705                 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {
1706                         found_an_ncb_fingerprint = true;
1707                         break;
1708                 }
1709
1710         }
1711
1712         chip->select_chip(mtd, saved_chip_number);
1713
1714         if (found_an_ncb_fingerprint)
1715                 dev_dbg(dev, "\tFound a fingerprint\n");
1716         else
1717                 dev_dbg(dev, "\tNo fingerprint found\n");
1718         return found_an_ncb_fingerprint;
1719 }
1720
1721 /* Writes a transcription stamp. */
1722 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
1723 {
1724         struct device *dev = this->dev;
1725         struct boot_rom_geometry *rom_geo = &this->rom_geometry;
1726         struct mtd_info *mtd = &this->mtd;
1727         struct nand_chip *chip = &this->nand;
1728         unsigned int block_size_in_pages;
1729         unsigned int search_area_size_in_strides;
1730         unsigned int search_area_size_in_pages;
1731         unsigned int search_area_size_in_blocks;
1732         unsigned int block;
1733         unsigned int stride;
1734         unsigned int page;
1735         uint8_t      *buffer = chip->buffers->databuf;
1736         int saved_chip_number;
1737         int status;
1738
1739         /* Compute the search area geometry. */
1740         block_size_in_pages = mtd->erasesize / mtd->writesize;
1741         search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
1742         search_area_size_in_pages = search_area_size_in_strides *
1743                                         rom_geo->stride_size_in_pages;
1744         search_area_size_in_blocks =
1745                   (search_area_size_in_pages + (block_size_in_pages - 1)) /
1746                                     block_size_in_pages;
1747
1748         dev_dbg(dev, "Search Area Geometry :\n");
1749         dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);
1750         dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);
1751         dev_dbg(dev, "\tin Pages  : %u\n", search_area_size_in_pages);
1752
1753         /* Select chip 0. */
1754         saved_chip_number = this->current_chip;
1755         chip->select_chip(mtd, 0);
1756
1757         /* Loop over blocks in the first search area, erasing them. */
1758         dev_dbg(dev, "Erasing the search area...\n");
1759
1760         for (block = 0; block < search_area_size_in_blocks; block++) {
1761                 /* Compute the page address. */
1762                 page = block * block_size_in_pages;
1763
1764                 /* Erase this block. */
1765                 dev_dbg(dev, "\tErasing block 0x%x\n", block);
1766                 chip->cmdfunc(mtd, NAND_CMD_ERASE1, -1, page);
1767                 chip->cmdfunc(mtd, NAND_CMD_ERASE2, -1, -1);
1768
1769                 /* Wait for the erase to finish. */
1770                 status = chip->waitfunc(mtd, chip);
1771                 if (status & NAND_STATUS_FAIL)
1772                         dev_err(dev, "[%s] Erase failed.\n", __func__);
1773         }
1774
1775         /* Write the NCB fingerprint into the page buffer. */
1776         memset(buffer, ~0, mtd->writesize);
1777         memcpy(buffer + 12, fingerprint, strlen(fingerprint));
1778
1779         /* Loop through the first search area, writing NCB fingerprints. */
1780         dev_dbg(dev, "Writing NCB fingerprints...\n");
1781         for (stride = 0; stride < search_area_size_in_strides; stride++) {
1782                 /* Compute the page addresses. */
1783                 page = stride * rom_geo->stride_size_in_pages;
1784
1785                 /* Write the first page of the current stride. */
1786                 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
1787                 chip->cmdfunc(mtd, NAND_CMD_SEQIN, 0x00, page);
1788                 chip->ecc.write_page_raw(mtd, chip, buffer, 0, page);
1789                 chip->cmdfunc(mtd, NAND_CMD_PAGEPROG, -1, -1);
1790
1791                 /* Wait for the write to finish. */
1792                 status = chip->waitfunc(mtd, chip);
1793                 if (status & NAND_STATUS_FAIL)
1794                         dev_err(dev, "[%s] Write failed.\n", __func__);
1795         }
1796
1797         /* Deselect chip 0. */
1798         chip->select_chip(mtd, saved_chip_number);
1799         return 0;
1800 }
1801
1802 static int mx23_boot_init(struct gpmi_nand_data  *this)
1803 {
1804         struct device *dev = this->dev;
1805         struct nand_chip *chip = &this->nand;
1806         struct mtd_info *mtd = &this->mtd;
1807         unsigned int block_count;
1808         unsigned int block;
1809         int     chipnr;
1810         int     page;
1811         loff_t  byte;
1812         uint8_t block_mark;
1813         int     ret = 0;
1814
1815         /*
1816          * If control arrives here, we can't use block mark swapping, which
1817          * means we're forced to use transcription. First, scan for the
1818          * transcription stamp. If we find it, then we don't have to do
1819          * anything -- the block marks are already transcribed.
1820          */
1821         if (mx23_check_transcription_stamp(this))
1822                 return 0;
1823
1824         /*
1825          * If control arrives here, we couldn't find a transcription stamp, so
1826          * so we presume the block marks are in the conventional location.
1827          */
1828         dev_dbg(dev, "Transcribing bad block marks...\n");
1829
1830         /* Compute the number of blocks in the entire medium. */
1831         block_count = chip->chipsize >> chip->phys_erase_shift;
1832
1833         /*
1834          * Loop over all the blocks in the medium, transcribing block marks as
1835          * we go.
1836          */
1837         for (block = 0; block < block_count; block++) {
1838                 /*
1839                  * Compute the chip, page and byte addresses for this block's
1840                  * conventional mark.
1841                  */
1842                 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);
1843                 page = block << (chip->phys_erase_shift - chip->page_shift);
1844                 byte = block <<  chip->phys_erase_shift;
1845
1846                 /* Send the command to read the conventional block mark. */
1847                 chip->select_chip(mtd, chipnr);
1848                 chip->cmdfunc(mtd, NAND_CMD_READ0, mtd->writesize, page);
1849                 block_mark = chip->read_byte(mtd);
1850                 chip->select_chip(mtd, -1);
1851
1852                 /*
1853                  * Check if the block is marked bad. If so, we need to mark it
1854                  * again, but this time the result will be a mark in the
1855                  * location where we transcribe block marks.
1856                  */
1857                 if (block_mark != 0xff) {
1858                         dev_dbg(dev, "Transcribing mark in block %u\n", block);
1859                         ret = chip->block_markbad(mtd, byte);
1860                         if (ret)
1861                                 dev_err(dev,
1862                                         "Failed to mark block bad with ret %d\n",
1863                                         ret);
1864                 }
1865         }
1866
1867         /* Write the stamp that indicates we've transcribed the block marks. */
1868         mx23_write_transcription_stamp(this);
1869         return 0;
1870 }
1871
1872 static int nand_boot_init(struct gpmi_nand_data  *this)
1873 {
1874         nand_boot_set_geometry(this);
1875
1876         /* This is ROM arch-specific initilization before the BBT scanning. */
1877         if (GPMI_IS_MX23(this))
1878                 return mx23_boot_init(this);
1879         return 0;
1880 }
1881
1882 static int gpmi_set_geometry(struct gpmi_nand_data *this)
1883 {
1884         int ret;
1885
1886         /* Free the temporary DMA memory for reading ID. */
1887         gpmi_free_dma_buffer(this);
1888
1889         /* Set up the NFC geometry which is used by BCH. */
1890         ret = bch_set_geometry(this);
1891         if (ret) {
1892                 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);
1893                 return ret;
1894         }
1895
1896         /* Alloc the new DMA buffers according to the pagesize and oobsize */
1897         return gpmi_alloc_dma_buffer(this);
1898 }
1899
1900 static void gpmi_nand_exit(struct gpmi_nand_data *this)
1901 {
1902         nand_release(&this->mtd);
1903         gpmi_free_dma_buffer(this);
1904 }
1905
1906 static int gpmi_init_last(struct gpmi_nand_data *this)
1907 {
1908         struct mtd_info *mtd = &this->mtd;
1909         struct nand_chip *chip = mtd->priv;
1910         struct nand_ecc_ctrl *ecc = &chip->ecc;
1911         struct bch_geometry *bch_geo = &this->bch_geometry;
1912         int ret;
1913
1914         /* Set up the medium geometry */
1915         ret = gpmi_set_geometry(this);
1916         if (ret)
1917                 return ret;
1918
1919         /* Init the nand_ecc_ctrl{} */
1920         ecc->read_page  = gpmi_ecc_read_page;
1921         ecc->write_page = gpmi_ecc_write_page;
1922         ecc->read_oob   = gpmi_ecc_read_oob;
1923         ecc->write_oob  = gpmi_ecc_write_oob;
1924         ecc->read_page_raw = gpmi_ecc_read_page_raw;
1925         ecc->write_page_raw = gpmi_ecc_write_page_raw;
1926         ecc->read_oob_raw = gpmi_ecc_read_oob_raw;
1927         ecc->write_oob_raw = gpmi_ecc_write_oob_raw;
1928         ecc->mode       = NAND_ECC_HW;
1929         ecc->size       = bch_geo->ecc_chunk_size;
1930         ecc->strength   = bch_geo->ecc_strength;
1931         ecc->layout     = &gpmi_hw_ecclayout;
1932
1933         /*
1934          * We only enable the subpage read when:
1935          *  (1) the chip is imx6, and
1936          *  (2) the size of the ECC parity is byte aligned.
1937          */
1938         if (GPMI_IS_MX6(this) &&
1939                 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {
1940                 ecc->read_subpage = gpmi_ecc_read_subpage;
1941                 chip->options |= NAND_SUBPAGE_READ;
1942         }
1943
1944         /*
1945          * Can we enable the extra features? such as EDO or Sync mode.
1946          *
1947          * We do not check the return value now. That's means if we fail in
1948          * enable the extra features, we still can run in the normal way.
1949          */
1950         gpmi_extra_init(this);
1951
1952         return 0;
1953 }
1954
1955 static int gpmi_nand_init(struct gpmi_nand_data *this)
1956 {
1957         struct mtd_info  *mtd = &this->mtd;
1958         struct nand_chip *chip = &this->nand;
1959         struct mtd_part_parser_data ppdata = {};
1960         int ret;
1961
1962         /* init current chip */
1963         this->current_chip      = -1;
1964
1965         /* init the MTD data structures */
1966         mtd->priv               = chip;
1967         mtd->name               = "gpmi-nand";
1968         mtd->dev.parent         = this->dev;
1969
1970         /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
1971         chip->priv              = this;
1972         chip->select_chip       = gpmi_select_chip;
1973         chip->cmd_ctrl          = gpmi_cmd_ctrl;
1974         chip->dev_ready         = gpmi_dev_ready;
1975         chip->read_byte         = gpmi_read_byte;
1976         chip->read_buf          = gpmi_read_buf;
1977         chip->write_buf         = gpmi_write_buf;
1978         chip->badblock_pattern  = &gpmi_bbt_descr;
1979         chip->block_markbad     = gpmi_block_markbad;
1980         chip->options           |= NAND_NO_SUBPAGE_WRITE;
1981
1982         /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */
1983         this->swap_block_mark = !GPMI_IS_MX23(this);
1984
1985         if (of_get_nand_on_flash_bbt(this->dev->of_node)) {
1986                 chip->bbt_options |= NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1987
1988                 if (of_property_read_bool(this->dev->of_node,
1989                                                 "fsl,no-blockmark-swap"))
1990                         this->swap_block_mark = false;
1991         }
1992         dev_dbg(this->dev, "Blockmark swapping %sabled\n",
1993                 this->swap_block_mark ? "en" : "dis");
1994
1995         /*
1996          * Allocate a temporary DMA buffer for reading ID in the
1997          * nand_scan_ident().
1998          */
1999         this->bch_geometry.payload_size = 1024;
2000         this->bch_geometry.auxiliary_size = 128;
2001         ret = gpmi_alloc_dma_buffer(this);
2002         if (ret)
2003                 goto err_out;
2004
2005         ret = nand_scan_ident(mtd, GPMI_IS_MX6(this) ? 2 : 1, NULL);
2006         if (ret)
2007                 goto err_out;
2008
2009         ret = gpmi_init_last(this);
2010         if (ret)
2011                 goto err_out;
2012
2013         chip->options |= NAND_SKIP_BBTSCAN;
2014         ret = nand_scan_tail(mtd);
2015         if (ret)
2016                 goto err_out;
2017
2018         ret = nand_boot_init(this);
2019         if (ret)
2020                 goto err_out;
2021         ret = chip->scan_bbt(mtd);
2022         if (ret)
2023                 goto err_out;
2024
2025         ppdata.of_node = this->pdev->dev.of_node;
2026         ret = mtd_device_parse_register(mtd, NULL, &ppdata, NULL, 0);
2027         if (ret)
2028                 goto err_out;
2029         return 0;
2030
2031 err_out:
2032         gpmi_nand_exit(this);
2033         return ret;
2034 }
2035
2036 static const struct of_device_id gpmi_nand_id_table[] = {
2037         {
2038                 .compatible = "fsl,imx23-gpmi-nand",
2039                 .data = &gpmi_devdata_imx23,
2040         }, {
2041                 .compatible = "fsl,imx28-gpmi-nand",
2042                 .data = &gpmi_devdata_imx28,
2043         }, {
2044                 .compatible = "fsl,imx6q-gpmi-nand",
2045                 .data = &gpmi_devdata_imx6q,
2046         }, {
2047                 .compatible = "fsl,imx6sx-gpmi-nand",
2048                 .data = &gpmi_devdata_imx6sx,
2049         }, {}
2050 };
2051 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);
2052
2053 static int gpmi_nand_probe(struct platform_device *pdev)
2054 {
2055         struct gpmi_nand_data *this;
2056         const struct of_device_id *of_id;
2057         int ret;
2058
2059         this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);
2060         if (!this)
2061                 return -ENOMEM;
2062
2063         of_id = of_match_device(gpmi_nand_id_table, &pdev->dev);
2064         if (of_id) {
2065                 this->devdata = of_id->data;
2066         } else {
2067                 dev_err(&pdev->dev, "Failed to find the right device id.\n");
2068                 return -ENODEV;
2069         }
2070
2071         platform_set_drvdata(pdev, this);
2072         this->pdev  = pdev;
2073         this->dev   = &pdev->dev;
2074
2075         ret = acquire_resources(this);
2076         if (ret)
2077                 goto exit_acquire_resources;
2078
2079         ret = init_hardware(this);
2080         if (ret)
2081                 goto exit_nfc_init;
2082
2083         ret = gpmi_nand_init(this);
2084         if (ret)
2085                 goto exit_nfc_init;
2086
2087         dev_info(this->dev, "driver registered.\n");
2088
2089         return 0;
2090
2091 exit_nfc_init:
2092         release_resources(this);
2093 exit_acquire_resources:
2094
2095         return ret;
2096 }
2097
2098 static int gpmi_nand_remove(struct platform_device *pdev)
2099 {
2100         struct gpmi_nand_data *this = platform_get_drvdata(pdev);
2101
2102         gpmi_nand_exit(this);
2103         release_resources(this);
2104         return 0;
2105 }
2106
2107 static struct platform_driver gpmi_nand_driver = {
2108         .driver = {
2109                 .name = "gpmi-nand",
2110                 .of_match_table = gpmi_nand_id_table,
2111         },
2112         .probe   = gpmi_nand_probe,
2113         .remove  = gpmi_nand_remove,
2114 };
2115 module_platform_driver(gpmi_nand_driver);
2116
2117 MODULE_AUTHOR("Freescale Semiconductor, Inc.");
2118 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");
2119 MODULE_LICENSE("GPL");