arch/powerpc/platforms/pseries/iommu.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
   4  *
   5  * Rewrite, cleanup:
   6  *
   7  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
   8  * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
   9  *
  10  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
  11  */
  12
  13 #include <linux/init.h>
  14 #include <linux/types.h>
  15 #include <linux/slab.h>
  16 #include <linux/mm.h>
  17 #include <linux/memblock.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/string.h>
  20 #include <linux/pci.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/crash_dump.h>
  23 #include <linux/memory.h>
  24 #include <linux/of.h>
  25 #include <linux/iommu.h>
  26 #include <linux/rculist.h>
  27 #include <asm/io.h>
  28 #include <asm/prom.h>
  29 #include <asm/rtas.h>
  30 #include <asm/iommu.h>
  31 #include <asm/pci-bridge.h>
  32 #include <asm/machdep.h>
  33 #include <asm/firmware.h>
  34 #include <asm/tce.h>
  35 #include <asm/ppc-pci.h>
  36 #include <asm/udbg.h>
  37 #include <asm/mmzone.h>
  38 #include <asm/plpar_wrappers.h>
  39
  40 #include "pseries.h"
  41
  42 enum {
  43         DDW_QUERY_PE_DMA_WIN  = 0,
  44         DDW_CREATE_PE_DMA_WIN = 1,
  45         DDW_REMOVE_PE_DMA_WIN = 2,
  46
  47         DDW_APPLICABLE_SIZE
  48 };
  49
  50 enum {
  51         DDW_EXT_SIZE = 0,
  52         DDW_EXT_RESET_DMA_WIN = 1,
  53         DDW_EXT_QUERY_OUT_SIZE = 2
  54 };
  55
  56 static struct iommu_table *iommu_pseries_alloc_table(int node)
  57 {
  58         struct iommu_table *tbl;
  59
  60         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);
  61         if (!tbl)
  62                 return NULL;
  63
  64         INIT_LIST_HEAD_RCU(&tbl->it_group_list);
  65         kref_init(&tbl->it_kref);
  66         return tbl;
  67 }
  68
  69 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
  70 {
  71         struct iommu_table_group *table_group;
  72
  73         table_group = kzalloc_node(sizeof(*table_group), GFP_KERNEL, node);
  74         if (!table_group)
  75                 return NULL;
  76
  77         table_group->tables[0] = iommu_pseries_alloc_table(node);
  78         if (table_group->tables[0])
  79                 return table_group;
  80
  81         kfree(table_group);
  82         return NULL;
  83 }
  84
  85 static void iommu_pseries_free_group(struct iommu_table_group *table_group,
  86                 const char *node_name)
  87 {
  88         struct iommu_table *tbl;
  89
  90         if (!table_group)
  91                 return;
  92
  93         tbl = table_group->tables[0];
  94 #ifdef CONFIG_IOMMU_API
  95         if (table_group->group) {
  96                 iommu_group_put(table_group->group);
  97                 BUG_ON(table_group->group);
  98         }
  99 #endif
 100         iommu_tce_table_put(tbl);
 101
 102         kfree(table_group);
 103 }
 104
 105 static int tce_build_pSeries(struct iommu_table *tbl, long index,
 106                               long npages, unsigned long uaddr,
 107                               enum dma_data_direction direction,
 108                               unsigned long attrs)
 109 {
 110         u64 proto_tce;
 111         __be64 *tcep;
 112         u64 rpn;
 113         const unsigned long tceshift = tbl->it_page_shift;
 114         const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
 115
 116         proto_tce = TCE_PCI_READ; // Read allowed
 117
 118         if (direction != DMA_TO_DEVICE)
 119                 proto_tce |= TCE_PCI_WRITE;
 120
 121         tcep = ((__be64 *)tbl->it_base) + index;
 122
 123         while (npages--) {
 124                 /* can't move this out since we might cross MEMBLOCK boundary */
 125                 rpn = __pa(uaddr) >> tceshift;
 126                 *tcep = cpu_to_be64(proto_tce | rpn << tceshift);
 127
 128                 uaddr += pagesize;
 129                 tcep++;
 130         }
 131         return 0;
 132 }
 133
 134
 135 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
 136 {
 137         __be64 *tcep;
 138
 139         tcep = ((__be64 *)tbl->it_base) + index;
 140
 141         while (npages--)
 142                 *(tcep++) = 0;
 143 }
 144
 145 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
 146 {
 147         __be64 *tcep;
 148
 149         tcep = ((__be64 *)tbl->it_base) + index;
 150
 151         return be64_to_cpu(*tcep);
 152 }
 153
 154 static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
 155 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
 156
 157 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 158                                 long npages, unsigned long uaddr,
 159                                 enum dma_data_direction direction,
 160                                 unsigned long attrs)
 161 {
 162         u64 rc = 0;
 163         u64 proto_tce, tce;
 164         u64 rpn;
 165         int ret = 0;
 166         long tcenum_start = tcenum, npages_start = npages;
 167
 168         rpn = __pa(uaddr) >> tceshift;
 169         proto_tce = TCE_PCI_READ;
 170         if (direction != DMA_TO_DEVICE)
 171                 proto_tce |= TCE_PCI_WRITE;
 172
 173         while (npages--) {
 174                 tce = proto_tce | rpn << tceshift;
 175                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
 176
 177                 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 178                         ret = (int)rc;
 179                         tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
 180                                            (npages_start - (npages + 1)));
 181                         break;
 182                 }
 183
 184                 if (rc && printk_ratelimit()) {
 185                         printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 186                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 187                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 188                         printk("\ttce val = 0x%llx\n", tce );
 189                         dump_stack();
 190                 }
 191
 192                 tcenum++;
 193                 rpn++;
 194         }
 195         return ret;
 196 }
 197
 198 static DEFINE_PER_CPU(__be64 *, tce_page);
 199
 200 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
 201                                      long npages, unsigned long uaddr,
 202                                      enum dma_data_direction direction,
 203                                      unsigned long attrs)
 204 {
 205         u64 rc = 0;
 206         u64 proto_tce;
 207         __be64 *tcep;
 208         u64 rpn;
 209         long l, limit;
 210         long tcenum_start = tcenum, npages_start = npages;
 211         int ret = 0;
 212         unsigned long flags;
 213         const unsigned long tceshift = tbl->it_page_shift;
 214
 215         if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 216                 return tce_build_pSeriesLP(tbl->it_index, tcenum,
 217                                            tceshift, npages, uaddr,
 218                                            direction, attrs);
 219         }
 220
 221         local_irq_save(flags);  /* to protect tcep and the page behind it */
 222
 223         tcep = __this_cpu_read(tce_page);
 224
 225         /* This is safe to do since interrupts are off when we're called
 226          * from iommu_alloc{,_sg}()
 227          */
 228         if (!tcep) {
 229                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 230                 /* If allocation fails, fall back to the loop implementation */
 231                 if (!tcep) {
 232                         local_irq_restore(flags);
 233                         return tce_build_pSeriesLP(tbl->it_index, tcenum,
 234                                         tceshift,
 235                                         npages, uaddr, direction, attrs);
 236                 }
 237                 __this_cpu_write(tce_page, tcep);
 238         }
 239
 240         rpn = __pa(uaddr) >> tceshift;
 241         proto_tce = TCE_PCI_READ;
 242         if (direction != DMA_TO_DEVICE)
 243                 proto_tce |= TCE_PCI_WRITE;
 244
 245         /* We can map max one pageful of TCEs at a time */
 246         do {
 247                 /*
 248                  * Set up the page with TCE data, looping through and setting
 249                  * the values.
 250                  */
 251                 limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
 252
 253                 for (l = 0; l < limit; l++) {
 254                         tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
 255                         rpn++;
 256                 }
 257
 258                 rc = plpar_tce_put_indirect((u64)tbl->it_index,
 259                                             (u64)tcenum << tceshift,
 260                                             (u64)__pa(tcep),
 261                                             limit);
 262
 263                 npages -= limit;
 264                 tcenum += limit;
 265         } while (npages > 0 && !rc);
 266
 267         local_irq_restore(flags);
 268
 269         if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 270                 ret = (int)rc;
 271                 tce_freemulti_pSeriesLP(tbl, tcenum_start,
 272                                         (npages_start - (npages + limit)));
 273                 return ret;
 274         }
 275
 276         if (rc && printk_ratelimit()) {
 277                 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 278                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 279                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 280                 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
 281                 dump_stack();
 282         }
 283         return ret;
 284 }
 285
 286 static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 287                                long npages)
 288 {
 289         u64 rc;
 290
 291         while (npages--) {
 292                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
 293
 294                 if (rc && printk_ratelimit()) {
 295                         printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 296                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 297                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 298                         dump_stack();
 299                 }
 300
 301                 tcenum++;
 302         }
 303 }
 304
 305
 306 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
 307 {
 308         u64 rc;
 309
 310         if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
 311                 return tce_free_pSeriesLP(tbl->it_index, tcenum,
 312                                           tbl->it_page_shift, npages);
 313
 314         rc = plpar_tce_stuff((u64)tbl->it_index,
 315                              (u64)tcenum << tbl->it_page_shift, 0, npages);
 316
 317         if (rc && printk_ratelimit()) {
 318                 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
 319                 printk("\trc      = %lld\n", rc);
 320                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 321                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 322                 dump_stack();
 323         }
 324 }
 325
 326 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
 327 {
 328         u64 rc;
 329         unsigned long tce_ret;
 330
 331         rc = plpar_tce_get((u64)tbl->it_index,
 332                            (u64)tcenum << tbl->it_page_shift, &tce_ret);
 333
 334         if (rc && printk_ratelimit()) {
 335                 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
 336                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 337                 printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 338                 dump_stack();
 339         }
 340
 341         return tce_ret;
 342 }
 343
 344 /* this is compatible with cells for the device tree property */
 345 struct dynamic_dma_window_prop {
 346         __be32  liobn;          /* tce table number */
 347         __be64  dma_base;       /* address hi,lo */
 348         __be32  tce_shift;      /* ilog2(tce_page_size) */
 349         __be32  window_shift;   /* ilog2(tce_window_size) */
 350 };
 351
 352 struct dma_win {
 353         struct device_node *device;
 354         const struct dynamic_dma_window_prop *prop;
 355         struct list_head list;
 356 };
 357
 358 /* Dynamic DMA Window support */
 359 struct ddw_query_response {
 360         u32 windows_available;
 361         u64 largest_available_block;
 362         u32 page_size;
 363         u32 migration_capable;
 364 };
 365
 366 struct ddw_create_response {
 367         u32 liobn;
 368         u32 addr_hi;
 369         u32 addr_lo;
 370 };
 371
 372 static LIST_HEAD(dma_win_list);
 373 /* prevents races between memory on/offline and window creation */
 374 static DEFINE_SPINLOCK(dma_win_list_lock);
 375 /* protects initializing window twice for same device */
 376 static DEFINE_MUTEX(dma_win_init_mutex);
 377 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
 378 #define DMA64_PROPNAME "linux,dma64-ddr-window-info"
 379
 380 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
 381                                         unsigned long num_pfn, const void *arg)
 382 {
 383         const struct dynamic_dma_window_prop *maprange = arg;
 384         int rc;
 385         u64 tce_size, num_tce, dma_offset, next;
 386         u32 tce_shift;
 387         long limit;
 388
 389         tce_shift = be32_to_cpu(maprange->tce_shift);
 390         tce_size = 1ULL << tce_shift;
 391         next = start_pfn << PAGE_SHIFT;
 392         num_tce = num_pfn << PAGE_SHIFT;
 393
 394         /* round back to the beginning of the tce page size */
 395         num_tce += next & (tce_size - 1);
 396         next &= ~(tce_size - 1);
 397
 398         /* covert to number of tces */
 399         num_tce |= tce_size - 1;
 400         num_tce >>= tce_shift;
 401
 402         do {
 403                 /*
 404                  * Set up the page with TCE data, looping through and setting
 405                  * the values.
 406                  */
 407                 limit = min_t(long, num_tce, 512);
 408                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 409
 410                 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
 411                                              dma_offset,
 412                                              0, limit);
 413                 next += limit * tce_size;
 414                 num_tce -= limit;
 415         } while (num_tce > 0 && !rc);
 416
 417         return rc;
 418 }
 419
 420 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
 421                                         unsigned long num_pfn, const void *arg)
 422 {
 423         const struct dynamic_dma_window_prop *maprange = arg;
 424         u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn;
 425         __be64 *tcep;
 426         u32 tce_shift;
 427         u64 rc = 0;
 428         long l, limit;
 429
 430         if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 431                 unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
 432                 unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
 433                                 be64_to_cpu(maprange->dma_base);
 434                 unsigned long tcenum = dmastart >> tceshift;
 435                 unsigned long npages = num_pfn << PAGE_SHIFT >> tceshift;
 436                 void *uaddr = __va(start_pfn << PAGE_SHIFT);
 437
 438                 return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn),
 439                                 tcenum, tceshift, npages, (unsigned long) uaddr,
 440                                 DMA_BIDIRECTIONAL, 0);
 441         }
 442
 443         local_irq_disable();    /* to protect tcep and the page behind it */
 444         tcep = __this_cpu_read(tce_page);
 445
 446         if (!tcep) {
 447                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 448                 if (!tcep) {
 449                         local_irq_enable();
 450                         return -ENOMEM;
 451                 }
 452                 __this_cpu_write(tce_page, tcep);
 453         }
 454
 455         proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
 456
 457         liobn = (u64)be32_to_cpu(maprange->liobn);
 458         tce_shift = be32_to_cpu(maprange->tce_shift);
 459         tce_size = 1ULL << tce_shift;
 460         next = start_pfn << PAGE_SHIFT;
 461         num_tce = num_pfn << PAGE_SHIFT;
 462
 463         /* round back to the beginning of the tce page size */
 464         num_tce += next & (tce_size - 1);
 465         next &= ~(tce_size - 1);
 466
 467         /* covert to number of tces */
 468         num_tce |= tce_size - 1;
 469         num_tce >>= tce_shift;
 470
 471         /* We can map max one pageful of TCEs at a time */
 472         do {
 473                 /*
 474                  * Set up the page with TCE data, looping through and setting
 475                  * the values.
 476                  */
 477                 limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
 478                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 479
 480                 for (l = 0; l < limit; l++) {
 481                         tcep[l] = cpu_to_be64(proto_tce | next);
 482                         next += tce_size;
 483                 }
 484
 485                 rc = plpar_tce_put_indirect(liobn,
 486                                             dma_offset,
 487                                             (u64)__pa(tcep),
 488                                             limit);
 489
 490                 num_tce -= limit;
 491         } while (num_tce > 0 && !rc);
 492
 493         /* error cleanup: caller will clear whole range */
 494
 495         local_irq_enable();
 496         return rc;
 497 }
 498
 499 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
 500                 unsigned long num_pfn, void *arg)
 501 {
 502         return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
 503 }
 504
 505 static void iommu_table_setparms_common(struct iommu_table *tbl, unsigned long busno,
 506                                         unsigned long liobn, unsigned long win_addr,
 507                                         unsigned long window_size, unsigned long page_shift,
 508                                         void *base, struct iommu_table_ops *table_ops)
 509 {
 510         tbl->it_busno = busno;
 511         tbl->it_index = liobn;
 512         tbl->it_offset = win_addr >> page_shift;
 513         tbl->it_size = window_size >> page_shift;
 514         tbl->it_page_shift = page_shift;
 515         tbl->it_base = (unsigned long)base;
 516         tbl->it_blocksize = 16;
 517         tbl->it_type = TCE_PCI;
 518         tbl->it_ops = table_ops;
 519 }
 520
 521 struct iommu_table_ops iommu_table_pseries_ops;
 522
 523 static void iommu_table_setparms(struct pci_controller *phb,
 524                                  struct device_node *dn,
 525                                  struct iommu_table *tbl)
 526 {
 527         struct device_node *node;
 528         const unsigned long *basep;
 529         const u32 *sizep;
 530
 531         /* Test if we are going over 2GB of DMA space */
 532         if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) {
 533                 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 534                 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 535         }
 536
 537         node = phb->dn;
 538         basep = of_get_property(node, "linux,tce-base", NULL);
 539         sizep = of_get_property(node, "linux,tce-size", NULL);
 540         if (basep == NULL || sizep == NULL) {
 541                 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has "
 542                                 "missing tce entries !\n", dn);
 543                 return;
 544         }
 545
 546         iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur,
 547                                     phb->dma_window_size, IOMMU_PAGE_SHIFT_4K,
 548                                     __va(*basep), &iommu_table_pseries_ops);
 549
 550         if (!is_kdump_kernel())
 551                 memset((void *)tbl->it_base, 0, *sizep);
 552
 553         phb->dma_window_base_cur += phb->dma_window_size;
 554 }
 555
 556 struct iommu_table_ops iommu_table_lpar_multi_ops;
 557
 558 /*
 559  * iommu_table_setparms_lpar
 560  *
 561  * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
 562  */
 563 static void iommu_table_setparms_lpar(struct pci_controller *phb,
 564                                       struct device_node *dn,
 565                                       struct iommu_table *tbl,
 566                                       struct iommu_table_group *table_group,
 567                                       const __be32 *dma_window)
 568 {
 569         unsigned long offset, size, liobn;
 570
 571         of_parse_dma_window(dn, dma_window, &liobn, &offset, &size);
 572
 573         iommu_table_setparms_common(tbl, phb->bus->number, liobn, offset, size, IOMMU_PAGE_SHIFT_4K, NULL,
 574                                     &iommu_table_lpar_multi_ops);
 575
 576
 577         table_group->tce32_start = offset;
 578         table_group->tce32_size = size;
 579 }
 580
 581 struct iommu_table_ops iommu_table_pseries_ops = {
 582         .set = tce_build_pSeries,
 583         .clear = tce_free_pSeries,
 584         .get = tce_get_pseries
 585 };
 586
 587 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
 588 {
 589         struct device_node *dn;
 590         struct iommu_table *tbl;
 591         struct device_node *isa_dn, *isa_dn_orig;
 592         struct device_node *tmp;
 593         struct pci_dn *pci;
 594         int children;
 595
 596         dn = pci_bus_to_OF_node(bus);
 597
 598         pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn);
 599
 600         if (bus->self) {
 601                 /* This is not a root bus, any setup will be done for the
 602                  * device-side of the bridge in iommu_dev_setup_pSeries().
 603                  */
 604                 return;
 605         }
 606         pci = PCI_DN(dn);
 607
 608         /* Check if the ISA bus on the system is under
 609          * this PHB.
 610          */
 611         isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
 612
 613         while (isa_dn && isa_dn != dn)
 614                 isa_dn = isa_dn->parent;
 615
 616         of_node_put(isa_dn_orig);
 617
 618         /* Count number of direct PCI children of the PHB. */
 619         for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
 620                 children++;
 621
 622         pr_debug("Children: %d\n", children);
 623
 624         /* Calculate amount of DMA window per slot. Each window must be
 625          * a power of two (due to pci_alloc_consistent requirements).
 626          *
 627          * Keep 256MB aside for PHBs with ISA.
 628          */
 629
 630         if (!isa_dn) {
 631                 /* No ISA/IDE - just set window size and return */
 632                 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
 633
 634                 while (pci->phb->dma_window_size * children > 0x80000000ul)
 635                         pci->phb->dma_window_size >>= 1;
 636                 pr_debug("No ISA/IDE, window size is 0x%llx\n",
 637                          pci->phb->dma_window_size);
 638                 pci->phb->dma_window_base_cur = 0;
 639
 640                 return;
 641         }
 642
 643         /* If we have ISA, then we probably have an IDE
 644          * controller too. Allocate a 128MB table but
 645          * skip the first 128MB to avoid stepping on ISA
 646          * space.
 647          */
 648         pci->phb->dma_window_size = 0x8000000ul;
 649         pci->phb->dma_window_base_cur = 0x8000000ul;
 650
 651         pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
 652         tbl = pci->table_group->tables[0];
 653
 654         iommu_table_setparms(pci->phb, dn, tbl);
 655
 656         if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
 657                 panic("Failed to initialize iommu table");
 658
 659         /* Divide the rest (1.75GB) among the children */
 660         pci->phb->dma_window_size = 0x80000000ul;
 661         while (pci->phb->dma_window_size * children > 0x70000000ul)
 662                 pci->phb->dma_window_size >>= 1;
 663
 664         pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
 665 }
 666
 667 #ifdef CONFIG_IOMMU_API
 668 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
 669                                 long *tce, enum dma_data_direction *direction,
 670                                 bool realmode)
 671 {
 672         long rc;
 673         unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
 674         unsigned long flags, oldtce = 0;
 675         u64 proto_tce = iommu_direction_to_tce_perm(*direction);
 676         unsigned long newtce = *tce | proto_tce;
 677
 678         spin_lock_irqsave(&tbl->large_pool.lock, flags);
 679
 680         rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce);
 681         if (!rc)
 682                 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce);
 683
 684         if (!rc) {
 685                 *direction = iommu_tce_direction(oldtce);
 686                 *tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
 687         }
 688
 689         spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
 690
 691         return rc;
 692 }
 693 #endif
 694
 695 struct iommu_table_ops iommu_table_lpar_multi_ops = {
 696         .set = tce_buildmulti_pSeriesLP,
 697 #ifdef CONFIG_IOMMU_API
 698         .xchg_no_kill = tce_exchange_pseries,
 699 #endif
 700         .clear = tce_freemulti_pSeriesLP,
 701         .get = tce_get_pSeriesLP
 702 };
 703
 704 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
 705 {
 706         struct iommu_table *tbl;
 707         struct device_node *dn, *pdn;
 708         struct pci_dn *ppci;
 709         const __be32 *dma_window = NULL;
 710
 711         dn = pci_bus_to_OF_node(bus);
 712
 713         pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n",
 714                  dn);
 715
 716         /*
 717          * Find nearest ibm,dma-window (default DMA window), walking up the
 718          * device tree
 719          */
 720         for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
 721                 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
 722                 if (dma_window != NULL)
 723                         break;
 724         }
 725
 726         if (dma_window == NULL) {
 727                 pr_debug("  no ibm,dma-window property !\n");
 728                 return;
 729         }
 730
 731         ppci = PCI_DN(pdn);
 732
 733         pr_debug("  parent is %pOF, iommu_table: 0x%p\n",
 734                  pdn, ppci->table_group);
 735
 736         if (!ppci->table_group) {
 737                 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node);
 738                 tbl = ppci->table_group->tables[0];
 739                 iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
 740                                 ppci->table_group, dma_window);
 741
 742                 if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
 743                         panic("Failed to initialize iommu table");
 744                 iommu_register_group(ppci->table_group,
 745                                 pci_domain_nr(bus), 0);
 746                 pr_debug("  created table: %p\n", ppci->table_group);
 747         }
 748 }
 749
 750
 751 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
 752 {
 753         struct device_node *dn;
 754         struct iommu_table *tbl;
 755
 756         pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
 757
 758         dn = dev->dev.of_node;
 759
 760         /* If we're the direct child of a root bus, then we need to allocate
 761          * an iommu table ourselves. The bus setup code should have setup
 762          * the window sizes already.
 763          */
 764         if (!dev->bus->self) {
 765                 struct pci_controller *phb = PCI_DN(dn)->phb;
 766
 767                 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
 768                 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node);
 769                 tbl = PCI_DN(dn)->table_group->tables[0];
 770                 iommu_table_setparms(phb, dn, tbl);
 771
 772                 if (!iommu_init_table(tbl, phb->node, 0, 0))
 773                         panic("Failed to initialize iommu table");
 774
 775                 set_iommu_table_base(&dev->dev, tbl);
 776                 return;
 777         }
 778
 779         /* If this device is further down the bus tree, search upwards until
 780          * an already allocated iommu table is found and use that.
 781          */
 782
 783         while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
 784                 dn = dn->parent;
 785
 786         if (dn && PCI_DN(dn))
 787                 set_iommu_table_base(&dev->dev,
 788                                 PCI_DN(dn)->table_group->tables[0]);
 789         else
 790                 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
 791                        pci_name(dev));
 792 }
 793
 794 static int __read_mostly disable_ddw;
 795
 796 static int __init disable_ddw_setup(char *str)
 797 {
 798         disable_ddw = 1;
 799         printk(KERN_INFO "ppc iommu: disabling ddw.\n");
 800
 801         return 0;
 802 }
 803
 804 early_param("disable_ddw", disable_ddw_setup);
 805
 806 static void clean_dma_window(struct device_node *np, struct dynamic_dma_window_prop *dwp)
 807 {
 808         int ret;
 809
 810         ret = tce_clearrange_multi_pSeriesLP(0,
 811                 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
 812         if (ret)
 813                 pr_warn("%pOF failed to clear tces in window.\n",
 814                         np);
 815         else
 816                 pr_debug("%pOF successfully cleared tces in window.\n",
 817                          np);
 818 }
 819
 820 /*
 821  * Call only if DMA window is clean.
 822  */
 823 static void __remove_dma_window(struct device_node *np, u32 *ddw_avail, u64 liobn)
 824 {
 825         int ret;
 826
 827         ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
 828         if (ret)
 829                 pr_warn("%pOF: failed to remove DMA window: rtas returned "
 830                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 831                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 832         else
 833                 pr_debug("%pOF: successfully removed DMA window: rtas returned "
 834                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 835                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 836 }
 837
 838 static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
 839                               struct property *win)
 840 {
 841         struct dynamic_dma_window_prop *dwp;
 842         u64 liobn;
 843
 844         dwp = win->value;
 845         liobn = (u64)be32_to_cpu(dwp->liobn);
 846
 847         clean_dma_window(np, dwp);
 848         __remove_dma_window(np, ddw_avail, liobn);
 849 }
 850
 851 static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_name)
 852 {
 853         struct property *win;
 854         u32 ddw_avail[DDW_APPLICABLE_SIZE];
 855         int ret = 0;
 856
 857         win = of_find_property(np, win_name, NULL);
 858         if (!win)
 859                 return -EINVAL;
 860
 861         ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
 862                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
 863         if (ret)
 864                 return 0;
 865
 866
 867         if (win->length >= sizeof(struct dynamic_dma_window_prop))
 868                 remove_dma_window(np, ddw_avail, win);
 869
 870         if (!remove_prop)
 871                 return 0;
 872
 873         ret = of_remove_property(np, win);
 874         if (ret)
 875                 pr_warn("%pOF: failed to remove DMA window property: %d\n",
 876                         np, ret);
 877         return 0;
 878 }
 879
 880 static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift)
 881 {
 882         struct dma_win *window;
 883         const struct dynamic_dma_window_prop *dma64;
 884         bool found = false;
 885
 886         spin_lock(&dma_win_list_lock);
 887         /* check if we already created a window and dupe that config if so */
 888         list_for_each_entry(window, &dma_win_list, list) {
 889                 if (window->device == pdn) {
 890                         dma64 = window->prop;
 891                         *dma_addr = be64_to_cpu(dma64->dma_base);
 892                         *window_shift = be32_to_cpu(dma64->window_shift);
 893                         found = true;
 894                         break;
 895                 }
 896         }
 897         spin_unlock(&dma_win_list_lock);
 898
 899         return found;
 900 }
 901
 902 static struct dma_win *ddw_list_new_entry(struct device_node *pdn,
 903                                           const struct dynamic_dma_window_prop *dma64)
 904 {
 905         struct dma_win *window;
 906
 907         window = kzalloc(sizeof(*window), GFP_KERNEL);
 908         if (!window)
 909                 return NULL;
 910
 911         window->device = pdn;
 912         window->prop = dma64;
 913
 914         return window;
 915 }
 916
 917 static void find_existing_ddw_windows_named(const char *name)
 918 {
 919         int len;
 920         struct device_node *pdn;
 921         struct dma_win *window;
 922         const struct dynamic_dma_window_prop *dma64;
 923
 924         for_each_node_with_property(pdn, name) {
 925                 dma64 = of_get_property(pdn, name, &len);
 926                 if (!dma64 || len < sizeof(*dma64)) {
 927                         remove_ddw(pdn, true, name);
 928                         continue;
 929                 }
 930
 931                 window = ddw_list_new_entry(pdn, dma64);
 932                 if (!window)
 933                         break;
 934
 935                 spin_lock(&dma_win_list_lock);
 936                 list_add(&window->list, &dma_win_list);
 937                 spin_unlock(&dma_win_list_lock);
 938         }
 939 }
 940
 941 static int find_existing_ddw_windows(void)
 942 {
 943         if (!firmware_has_feature(FW_FEATURE_LPAR))
 944                 return 0;
 945
 946         find_existing_ddw_windows_named(DIRECT64_PROPNAME);
 947         find_existing_ddw_windows_named(DMA64_PROPNAME);
 948
 949         return 0;
 950 }
 951 machine_arch_initcall(pseries, find_existing_ddw_windows);
 952
 953 /**
 954  * ddw_read_ext - Get the value of an DDW extension
 955  * @np:         device node from which the extension value is to be read.
 956  * @extnum:     index number of the extension.
 957  * @value:      pointer to return value, modified when extension is available.
 958  *
 959  * Checks if "ibm,ddw-extensions" exists for this node, and get the value
 960  * on index 'extnum'.
 961  * It can be used only to check if a property exists, passing value == NULL.
 962  *
 963  * Returns:
 964  *      0 if extension successfully read
 965  *      -EINVAL if the "ibm,ddw-extensions" does not exist,
 966  *      -ENODATA if "ibm,ddw-extensions" does not have a value, and
 967  *      -EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
 968  */
 969 static inline int ddw_read_ext(const struct device_node *np, int extnum,
 970                                u32 *value)
 971 {
 972         static const char propname[] = "ibm,ddw-extensions";
 973         u32 count;
 974         int ret;
 975
 976         ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
 977         if (ret)
 978                 return ret;
 979
 980         if (count < extnum)
 981                 return -EOVERFLOW;
 982
 983         if (!value)
 984                 value = &count;
 985
 986         return of_property_read_u32_index(np, propname, extnum, value);
 987 }
 988
 989 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
 990                      struct ddw_query_response *query,
 991                      struct device_node *parent)
 992 {
 993         struct device_node *dn;
 994         struct pci_dn *pdn;
 995         u32 cfg_addr, ext_query, query_out[5];
 996         u64 buid;
 997         int ret, out_sz;
 998
 999         /*
1000          * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
1001          * output parameters ibm,query-pe-dma-windows will have, ranging from
1002          * 5 to 6.
1003          */
1004         ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
1005         if (!ret && ext_query == 1)
1006                 out_sz = 6;
1007         else
1008                 out_sz = 5;
1009
1010         /*
1011          * Get the config address and phb buid of the PE window.
1012          * Rely on eeh to retrieve this for us.
1013          * Retrieve them from the pci device, not the node with the
1014          * dma-window property
1015          */
1016         dn = pci_device_to_OF_node(dev);
1017         pdn = PCI_DN(dn);
1018         buid = pdn->phb->buid;
1019         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1020
1021         ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
1022                         cfg_addr, BUID_HI(buid), BUID_LO(buid));
1023         dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d\n",
1024                  ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1025                  BUID_LO(buid), ret);
1026
1027         switch (out_sz) {
1028         case 5:
1029                 query->windows_available = query_out[0];
1030                 query->largest_available_block = query_out[1];
1031                 query->page_size = query_out[2];
1032                 query->migration_capable = query_out[3];
1033                 break;
1034         case 6:
1035                 query->windows_available = query_out[0];
1036                 query->largest_available_block = ((u64)query_out[1] << 32) |
1037                                                  query_out[2];
1038                 query->page_size = query_out[3];
1039                 query->migration_capable = query_out[4];
1040                 break;
1041         }
1042
1043         return ret;
1044 }
1045
1046 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1047                         struct ddw_create_response *create, int page_shift,
1048                         int window_shift)
1049 {
1050         struct device_node *dn;
1051         struct pci_dn *pdn;
1052         u32 cfg_addr;
1053         u64 buid;
1054         int ret;
1055
1056         /*
1057          * Get the config address and phb buid of the PE window.
1058          * Rely on eeh to retrieve this for us.
1059          * Retrieve them from the pci device, not the node with the
1060          * dma-window property
1061          */
1062         dn = pci_device_to_OF_node(dev);
1063         pdn = PCI_DN(dn);
1064         buid = pdn->phb->buid;
1065         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1066
1067         do {
1068                 /* extra outputs are LIOBN and dma-addr (hi, lo) */
1069                 ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1070                                 (u32 *)create, cfg_addr, BUID_HI(buid),
1071                                 BUID_LO(buid), page_shift, window_shift);
1072         } while (rtas_busy_delay(ret));
1073         dev_info(&dev->dev,
1074                 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
1075                 "(liobn = 0x%x starting addr = %x %x)\n",
1076                  ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1077                  BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1078                  create->addr_hi, create->addr_lo);
1079
1080         return ret;
1081 }
1082
1083 struct failed_ddw_pdn {
1084         struct device_node *pdn;
1085         struct list_head list;
1086 };
1087
1088 static LIST_HEAD(failed_ddw_pdn_list);
1089
1090 static phys_addr_t ddw_memory_hotplug_max(void)
1091 {
1092         phys_addr_t max_addr = memory_hotplug_max();
1093         struct device_node *memory;
1094
1095         for_each_node_by_type(memory, "memory") {
1096                 unsigned long start, size;
1097                 int n_mem_addr_cells, n_mem_size_cells, len;
1098                 const __be32 *memcell_buf;
1099
1100                 memcell_buf = of_get_property(memory, "reg", &len);
1101                 if (!memcell_buf || len <= 0)
1102                         continue;
1103
1104                 n_mem_addr_cells = of_n_addr_cells(memory);
1105                 n_mem_size_cells = of_n_size_cells(memory);
1106
1107                 start = of_read_number(memcell_buf, n_mem_addr_cells);
1108                 memcell_buf += n_mem_addr_cells;
1109                 size = of_read_number(memcell_buf, n_mem_size_cells);
1110                 memcell_buf += n_mem_size_cells;
1111
1112                 max_addr = max_t(phys_addr_t, max_addr, start + size);
1113         }
1114
1115         return max_addr;
1116 }
1117
1118 /*
1119  * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1120  * ibm,ddw-extensions, which carries the rtas token for
1121  * ibm,reset-pe-dma-windows.
1122  * That rtas-call can be used to restore the default DMA window for the device.
1123  */
1124 static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1125 {
1126         int ret;
1127         u32 cfg_addr, reset_dma_win;
1128         u64 buid;
1129         struct device_node *dn;
1130         struct pci_dn *pdn;
1131
1132         ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1133         if (ret)
1134                 return;
1135
1136         dn = pci_device_to_OF_node(dev);
1137         pdn = PCI_DN(dn);
1138         buid = pdn->phb->buid;
1139         cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1140
1141         ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1142                         BUID_LO(buid));
1143         if (ret)
1144                 dev_info(&dev->dev,
1145                          "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1146                          reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1147                          ret);
1148 }
1149
1150 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1151 static int iommu_get_page_shift(u32 query_page_size)
1152 {
1153         /* Supported IO page-sizes according to LoPAR */
1154         const int shift[] = {
1155                 __builtin_ctzll(SZ_4K),   __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
1156                 __builtin_ctzll(SZ_32M),  __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
1157                 __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G)
1158         };
1159
1160         int i = ARRAY_SIZE(shift) - 1;
1161
1162         /*
1163          * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
1164          * - bit 31 means 4k pages are supported,
1165          * - bit 30 means 64k pages are supported, and so on.
1166          * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
1167          */
1168         for (; i >= 0 ; i--) {
1169                 if (query_page_size & (1 << i))
1170                         return shift[i];
1171         }
1172
1173         /* No valid page size found. */
1174         return 0;
1175 }
1176
1177 static struct property *ddw_property_create(const char *propname, u32 liobn, u64 dma_addr,
1178                                             u32 page_shift, u32 window_shift)
1179 {
1180         struct dynamic_dma_window_prop *ddwprop;
1181         struct property *win64;
1182
1183         win64 = kzalloc(sizeof(*win64), GFP_KERNEL);
1184         if (!win64)
1185                 return NULL;
1186
1187         win64->name = kstrdup(propname, GFP_KERNEL);
1188         ddwprop = kzalloc(sizeof(*ddwprop), GFP_KERNEL);
1189         win64->value = ddwprop;
1190         win64->length = sizeof(*ddwprop);
1191         if (!win64->name || !win64->value) {
1192                 kfree(win64->name);
1193                 kfree(win64->value);
1194                 kfree(win64);
1195                 return NULL;
1196         }
1197
1198         ddwprop->liobn = cpu_to_be32(liobn);
1199         ddwprop->dma_base = cpu_to_be64(dma_addr);
1200         ddwprop->tce_shift = cpu_to_be32(page_shift);
1201         ddwprop->window_shift = cpu_to_be32(window_shift);
1202
1203         return win64;
1204 }
1205
1206 /*
1207  * If the PE supports dynamic dma windows, and there is space for a table
1208  * that can map all pages in a linear offset, then setup such a table,
1209  * and record the dma-offset in the struct device.
1210  *
1211  * dev: the pci device we are checking
1212  * pdn: the parent pe node with the ibm,dma_window property
1213  * Future: also check if we can remap the base window for our base page size
1214  *
1215  * returns true if can map all pages (direct mapping), false otherwise..
1216  */
1217 static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
1218 {
1219         int len = 0, ret;
1220         int max_ram_len = order_base_2(ddw_memory_hotplug_max());
1221         struct ddw_query_response query;
1222         struct ddw_create_response create;
1223         int page_shift;
1224         u64 win_addr;
1225         const char *win_name;
1226         struct device_node *dn;
1227         u32 ddw_avail[DDW_APPLICABLE_SIZE];
1228         struct dma_win *window;
1229         struct property *win64;
1230         bool ddw_enabled = false;
1231         struct failed_ddw_pdn *fpdn;
1232         bool default_win_removed = false, direct_mapping = false;
1233         bool pmem_present;
1234         struct pci_dn *pci = PCI_DN(pdn);
1235         struct iommu_table *tbl = pci->table_group->tables[0];
1236
1237         dn = of_find_node_by_type(NULL, "ibm,pmemory");
1238         pmem_present = dn != NULL;
1239         of_node_put(dn);
1240
1241         mutex_lock(&dma_win_init_mutex);
1242
1243         if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) {
1244                 direct_mapping = (len >= max_ram_len);
1245                 ddw_enabled = true;
1246                 goto out_unlock;
1247         }
1248
1249         /*
1250          * If we already went through this for a previous function of
1251          * the same device and failed, we don't want to muck with the
1252          * DMA window again, as it will race with in-flight operations
1253          * and can lead to EEHs. The above mutex protects access to the
1254          * list.
1255          */
1256         list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
1257                 if (fpdn->pdn == pdn)
1258                         goto out_unlock;
1259         }
1260
1261         /*
1262          * the ibm,ddw-applicable property holds the tokens for:
1263          * ibm,query-pe-dma-window
1264          * ibm,create-pe-dma-window
1265          * ibm,remove-pe-dma-window
1266          * for the given node in that order.
1267          * the property is actually in the parent, not the PE
1268          */
1269         ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1270                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
1271         if (ret)
1272                 goto out_failed;
1273
1274        /*
1275          * Query if there is a second window of size to map the
1276          * whole partition.  Query returns number of windows, largest
1277          * block assigned to PE (partition endpoint), and two bitmasks
1278          * of page sizes: supported and supported for migrate-dma.
1279          */
1280         dn = pci_device_to_OF_node(dev);
1281         ret = query_ddw(dev, ddw_avail, &query, pdn);
1282         if (ret != 0)
1283                 goto out_failed;
1284
1285         /*
1286          * If there is no window available, remove the default DMA window,
1287          * if it's present. This will make all the resources available to the
1288          * new DDW window.
1289          * If anything fails after this, we need to restore it, so also check
1290          * for extensions presence.
1291          */
1292         if (query.windows_available == 0) {
1293                 struct property *default_win;
1294                 int reset_win_ext;
1295
1296                 /* DDW + IOMMU on single window may fail if there is any allocation */
1297                 if (iommu_table_in_use(tbl)) {
1298                         dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n");
1299                         goto out_failed;
1300                 }
1301
1302                 default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1303                 if (!default_win)
1304                         goto out_failed;
1305
1306                 reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1307                 if (reset_win_ext)
1308                         goto out_failed;
1309
1310                 remove_dma_window(pdn, ddw_avail, default_win);
1311                 default_win_removed = true;
1312
1313                 /* Query again, to check if the window is available */
1314                 ret = query_ddw(dev, ddw_avail, &query, pdn);
1315                 if (ret != 0)
1316                         goto out_failed;
1317
1318                 if (query.windows_available == 0) {
1319                         /* no windows are available for this device. */
1320                         dev_dbg(&dev->dev, "no free dynamic windows");
1321                         goto out_failed;
1322                 }
1323         }
1324
1325         page_shift = iommu_get_page_shift(query.page_size);
1326         if (!page_shift) {
1327                 dev_dbg(&dev->dev, "no supported page size in mask %x",
1328                         query.page_size);
1329                 goto out_failed;
1330         }
1331
1332
1333         /*
1334          * The "ibm,pmemory" can appear anywhere in the address space.
1335          * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
1336          * for the upper limit and fallback to max RAM otherwise but this
1337          * disables device::dma_ops_bypass.
1338          */
1339         len = max_ram_len;
1340         if (pmem_present) {
1341                 if (query.largest_available_block >=
1342                     (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1343                         len = MAX_PHYSMEM_BITS;
1344                 else
1345                         dev_info(&dev->dev, "Skipping ibm,pmemory");
1346         }
1347
1348         /* check if the available block * number of ptes will map everything */
1349         if (query.largest_available_block < (1ULL << (len - page_shift))) {
1350                 dev_dbg(&dev->dev,
1351                         "can't map partition max 0x%llx with %llu %llu-sized pages\n",
1352                         1ULL << len,
1353                         query.largest_available_block,
1354                         1ULL << page_shift);
1355
1356                 len = order_base_2(query.largest_available_block << page_shift);
1357                 win_name = DMA64_PROPNAME;
1358         } else {
1359                 direct_mapping = !default_win_removed ||
1360                         (len == MAX_PHYSMEM_BITS) ||
1361                         (!pmem_present && (len == max_ram_len));
1362                 win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
1363         }
1364
1365         ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1366         if (ret != 0)
1367                 goto out_failed;
1368
1369         dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n",
1370                   create.liobn, dn);
1371
1372         win_addr = ((u64)create.addr_hi << 32) | create.addr_lo;
1373         win64 = ddw_property_create(win_name, create.liobn, win_addr, page_shift, len);
1374
1375         if (!win64) {
1376                 dev_info(&dev->dev,
1377                          "couldn't allocate property, property name, or value\n");
1378                 goto out_remove_win;
1379         }
1380
1381         ret = of_add_property(pdn, win64);
1382         if (ret) {
1383                 dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d",
1384                         pdn, ret);
1385                 goto out_free_prop;
1386         }
1387
1388         window = ddw_list_new_entry(pdn, win64->value);
1389         if (!window)
1390                 goto out_del_prop;
1391
1392         if (direct_mapping) {
1393                 /* DDW maps the whole partition, so enable direct DMA mapping */
1394                 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1395                                             win64->value, tce_setrange_multi_pSeriesLP_walk);
1396                 if (ret) {
1397                         dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n",
1398                                  dn, ret);
1399
1400                 /* Make sure to clean DDW if any TCE was set*/
1401                 clean_dma_window(pdn, win64->value);
1402                         goto out_del_list;
1403                 }
1404         } else {
1405                 struct iommu_table *newtbl;
1406                 int i;
1407                 unsigned long start = 0, end = 0;
1408
1409                 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) {
1410                         const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM;
1411
1412                         /* Look for MMIO32 */
1413                         if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) {
1414                                 start = pci->phb->mem_resources[i].start;
1415                                 end = pci->phb->mem_resources[i].end;
1416                                 break;
1417                         }
1418                 }
1419
1420                 /* New table for using DDW instead of the default DMA window */
1421                 newtbl = iommu_pseries_alloc_table(pci->phb->node);
1422                 if (!newtbl) {
1423                         dev_dbg(&dev->dev, "couldn't create new IOMMU table\n");
1424                         goto out_del_list;
1425                 }
1426
1427                 iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, win_addr,
1428                                             1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops);
1429                 iommu_init_table(newtbl, pci->phb->node, start, end);
1430
1431                 pci->table_group->tables[1] = newtbl;
1432
1433                 /* Keep default DMA window stuct if removed */
1434                 if (default_win_removed) {
1435                         tbl->it_size = 0;
1436                         vfree(tbl->it_map);
1437                         tbl->it_map = NULL;
1438                 }
1439
1440                 set_iommu_table_base(&dev->dev, newtbl);
1441         }
1442
1443         spin_lock(&dma_win_list_lock);
1444         list_add(&window->list, &dma_win_list);
1445         spin_unlock(&dma_win_list_lock);
1446
1447         dev->dev.archdata.dma_offset = win_addr;
1448         ddw_enabled = true;
1449         goto out_unlock;
1450
1451 out_del_list:
1452         kfree(window);
1453
1454 out_del_prop:
1455         of_remove_property(pdn, win64);
1456
1457 out_free_prop:
1458         kfree(win64->name);
1459         kfree(win64->value);
1460         kfree(win64);
1461
1462 out_remove_win:
1463         /* DDW is clean, so it's ok to call this directly. */
1464         __remove_dma_window(pdn, ddw_avail, create.liobn);
1465
1466 out_failed:
1467         if (default_win_removed)
1468                 reset_dma_window(dev, pdn);
1469
1470         fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1471         if (!fpdn)
1472                 goto out_unlock;
1473         fpdn->pdn = pdn;
1474         list_add(&fpdn->list, &failed_ddw_pdn_list);
1475
1476 out_unlock:
1477         mutex_unlock(&dma_win_init_mutex);
1478
1479         /*
1480          * If we have persistent memory and the window size is only as big
1481          * as RAM, then we failed to create a window to cover persistent
1482          * memory and need to set the DMA limit.
1483          */
1484         if (pmem_present && ddw_enabled && direct_mapping && len == max_ram_len)
1485                 dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << len);
1486
1487     return ddw_enabled && direct_mapping;
1488 }
1489
1490 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1491 {
1492         struct device_node *pdn, *dn;
1493         struct iommu_table *tbl;
1494         const __be32 *dma_window = NULL;
1495         struct pci_dn *pci;
1496
1497         pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1498
1499         /* dev setup for LPAR is a little tricky, since the device tree might
1500          * contain the dma-window properties per-device and not necessarily
1501          * for the bus. So we need to search upwards in the tree until we
1502          * either hit a dma-window property, OR find a parent with a table
1503          * already allocated.
1504          */
1505         dn = pci_device_to_OF_node(dev);
1506         pr_debug("  node is %pOF\n", dn);
1507
1508         for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group;
1509              pdn = pdn->parent) {
1510                 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1511                 if (dma_window)
1512                         break;
1513         }
1514
1515         if (!pdn || !PCI_DN(pdn)) {
1516                 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1517                        "no DMA window found for pci dev=%s dn=%pOF\n",
1518                                  pci_name(dev), dn);
1519                 return;
1520         }
1521         pr_debug("  parent is %pOF\n", pdn);
1522
1523         pci = PCI_DN(pdn);
1524         if (!pci->table_group) {
1525                 pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
1526                 tbl = pci->table_group->tables[0];
1527                 iommu_table_setparms_lpar(pci->phb, pdn, tbl,
1528                                 pci->table_group, dma_window);
1529
1530                 iommu_init_table(tbl, pci->phb->node, 0, 0);
1531                 iommu_register_group(pci->table_group,
1532                                 pci_domain_nr(pci->phb->bus), 0);
1533                 pr_debug("  created table: %p\n", pci->table_group);
1534         } else {
1535                 pr_debug("  found DMA window, table: %p\n", pci->table_group);
1536         }
1537
1538         set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1539         iommu_add_device(pci->table_group, &dev->dev);
1540 }
1541
1542 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
1543 {
1544         struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
1545         const __be32 *dma_window = NULL;
1546
1547         /* only attempt to use a new window if 64-bit DMA is requested */
1548         if (dma_mask < DMA_BIT_MASK(64))
1549                 return false;
1550
1551         dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1552
1553         /*
1554          * the device tree might contain the dma-window properties
1555          * per-device and not necessarily for the bus. So we need to
1556          * search upwards in the tree until we either hit a dma-window
1557          * property, OR find a parent with a table already allocated.
1558          */
1559         for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->table_group;
1560                         pdn = pdn->parent) {
1561                 dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
1562                 if (dma_window)
1563                         break;
1564         }
1565
1566         if (pdn && PCI_DN(pdn))
1567                 return enable_ddw(pdev, pdn);
1568
1569         return false;
1570 }
1571
1572 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1573                 void *data)
1574 {
1575         struct dma_win *window;
1576         struct memory_notify *arg = data;
1577         int ret = 0;
1578
1579         switch (action) {
1580         case MEM_GOING_ONLINE:
1581                 spin_lock(&dma_win_list_lock);
1582                 list_for_each_entry(window, &dma_win_list, list) {
1583                         ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1584                                         arg->nr_pages, window->prop);
1585                         /* XXX log error */
1586                 }
1587                 spin_unlock(&dma_win_list_lock);
1588                 break;
1589         case MEM_CANCEL_ONLINE:
1590         case MEM_OFFLINE:
1591                 spin_lock(&dma_win_list_lock);
1592                 list_for_each_entry(window, &dma_win_list, list) {
1593                         ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1594                                         arg->nr_pages, window->prop);
1595                         /* XXX log error */
1596                 }
1597                 spin_unlock(&dma_win_list_lock);
1598                 break;
1599         default:
1600                 break;
1601         }
1602         if (ret && action != MEM_CANCEL_ONLINE)
1603                 return NOTIFY_BAD;
1604
1605         return NOTIFY_OK;
1606 }
1607
1608 static struct notifier_block iommu_mem_nb = {
1609         .notifier_call = iommu_mem_notifier,
1610 };
1611
1612 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data)
1613 {
1614         int err = NOTIFY_OK;
1615         struct of_reconfig_data *rd = data;
1616         struct device_node *np = rd->dn;
1617         struct pci_dn *pci = PCI_DN(np);
1618         struct dma_win *window;
1619
1620         switch (action) {
1621         case OF_RECONFIG_DETACH_NODE:
1622                 /*
1623                  * Removing the property will invoke the reconfig
1624                  * notifier again, which causes dead-lock on the
1625                  * read-write semaphore of the notifier chain. So
1626                  * we have to remove the property when releasing
1627                  * the device node.
1628                  */
1629                 if (remove_ddw(np, false, DIRECT64_PROPNAME))
1630                         remove_ddw(np, false, DMA64_PROPNAME);
1631
1632                 if (pci && pci->table_group)
1633                         iommu_pseries_free_group(pci->table_group,
1634                                         np->full_name);
1635
1636                 spin_lock(&dma_win_list_lock);
1637                 list_for_each_entry(window, &dma_win_list, list) {
1638                         if (window->device == np) {
1639                                 list_del(&window->list);
1640                                 kfree(window);
1641                                 break;
1642                         }
1643                 }
1644                 spin_unlock(&dma_win_list_lock);
1645                 break;
1646         default:
1647                 err = NOTIFY_DONE;
1648                 break;
1649         }
1650         return err;
1651 }
1652
1653 static struct notifier_block iommu_reconfig_nb = {
1654         .notifier_call = iommu_reconfig_notifier,
1655 };
1656
1657 /* These are called very early. */
1658 void iommu_init_early_pSeries(void)
1659 {
1660         if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1661                 return;
1662
1663         if (firmware_has_feature(FW_FEATURE_LPAR)) {
1664                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1665                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1666                 if (!disable_ddw)
1667                         pseries_pci_controller_ops.iommu_bypass_supported =
1668                                 iommu_bypass_supported_pSeriesLP;
1669         } else {
1670                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
1671                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
1672         }
1673
1674
1675         of_reconfig_notifier_register(&iommu_reconfig_nb);
1676         register_memory_notifier(&iommu_mem_nb);
1677
1678         set_pci_dma_ops(&dma_iommu_ops);
1679 }
1680
1681 static int __init disable_multitce(char *str)
1682 {
1683         if (strcmp(str, "off") == 0 &&
1684             firmware_has_feature(FW_FEATURE_LPAR) &&
1685             (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1686              firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
1687                 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1688                 powerpc_firmware_features &=
1689                         ~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
1690         }
1691         return 1;
1692 }
1693
1694 __setup("multitce=", disable_multitce);
1695
1696 static int tce_iommu_bus_notifier(struct notifier_block *nb,
1697                 unsigned long action, void *data)
1698 {
1699         struct device *dev = data;
1700
1701         switch (action) {
1702         case BUS_NOTIFY_DEL_DEVICE:
1703                 iommu_del_device(dev);
1704                 return 0;
1705         default:
1706                 return 0;
1707         }
1708 }
1709
1710 static struct notifier_block tce_iommu_bus_nb = {
1711         .notifier_call = tce_iommu_bus_notifier,
1712 };
1713
1714 static int __init tce_iommu_bus_notifier_init(void)
1715 {
1716         bus_register_notifier(&pci_bus_type, &tce_iommu_bus_nb);
1717         return 0;
1718 }
1719 machine_subsys_initcall_sync(pseries, tce_iommu_bus_notifier_init);