platforms/pseries/iommu.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
   4  *
   5  * Rewrite, cleanup:
   6  *
   7  * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
   8  * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
   9  *
  10  * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
  11  */
  12
  13 #include <linux/init.h>
  14 #include <linux/types.h>
  15 #include <linux/slab.h>
  16 #include <linux/mm.h>
  17 #include <linux/memblock.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/string.h>
  20 #include <linux/pci.h>
  21 #include <linux/dma-mapping.h>
  22 #include <linux/crash_dump.h>
  23 #include <linux/memory.h>
  24 #include <linux/of.h>
  25 #include <linux/of_address.h>
  26 #include <linux/iommu.h>
  27 #include <linux/rculist.h>
  28 #include <asm/io.h>
  29 #include <asm/prom.h>
  30 #include <asm/rtas.h>
  31 #include <asm/iommu.h>
  32 #include <asm/pci-bridge.h>
  33 #include <asm/machdep.h>
  34 #include <asm/firmware.h>
  35 #include <asm/tce.h>
  36 #include <asm/ppc-pci.h>
  37 #include <asm/udbg.h>
  38 #include <asm/mmzone.h>
  39 #include <asm/plpar_wrappers.h>
  40
  41 #include "pseries.h"
  42
  43 enum {
  44         DDW_QUERY_PE_DMA_WIN  = 0,
  45         DDW_CREATE_PE_DMA_WIN = 1,
  46         DDW_REMOVE_PE_DMA_WIN = 2,
  47
  48         DDW_APPLICABLE_SIZE
  49 };
  50
  51 enum {
  52         DDW_EXT_SIZE = 0,
  53         DDW_EXT_RESET_DMA_WIN = 1,
  54         DDW_EXT_QUERY_OUT_SIZE = 2
  55 };
  56
  57 static struct iommu_table *iommu_pseries_alloc_table(int node)
  58 {
  59         struct iommu_table *tbl;
  60
  61         tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);
  62         if (!tbl)
  63                 return NULL;
  64
  65         INIT_LIST_HEAD_RCU(&tbl->it_group_list);
  66         kref_init(&tbl->it_kref);
  67         return tbl;
  68 }
  69
  70 static struct iommu_table_group *iommu_pseries_alloc_group(int node)
  71 {
  72         struct iommu_table_group *table_group;
  73
  74         table_group = kzalloc_node(sizeof(*table_group), GFP_KERNEL, node);
  75         if (!table_group)
  76                 return NULL;
  77
  78 #ifdef CONFIG_IOMMU_API
  79         table_group->ops = &spapr_tce_table_group_ops;
  80         table_group->pgsizes = SZ_4K;
  81 #endif
  82
  83         table_group->tables[0] = iommu_pseries_alloc_table(node);
  84         if (table_group->tables[0])
  85                 return table_group;
  86
  87         kfree(table_group);
  88         return NULL;
  89 }
  90
  91 static void iommu_pseries_free_group(struct iommu_table_group *table_group,
  92                 const char *node_name)
  93 {
  94         if (!table_group)
  95                 return;
  96
  97 #ifdef CONFIG_IOMMU_API
  98         if (table_group->group) {
  99                 iommu_group_put(table_group->group);
 100                 BUG_ON(table_group->group);
 101         }
 102 #endif
 103
 104         /* Default DMA window table is at index 0, while DDW at 1. SR-IOV
 105          * adapters only have table on index 1.
 106          */
 107         if (table_group->tables[0])
 108                 iommu_tce_table_put(table_group->tables[0]);
 109
 110         if (table_group->tables[1])
 111                 iommu_tce_table_put(table_group->tables[1]);
 112
 113         kfree(table_group);
 114 }
 115
 116 static int tce_build_pSeries(struct iommu_table *tbl, long index,
 117                               long npages, unsigned long uaddr,
 118                               enum dma_data_direction direction,
 119                               unsigned long attrs)
 120 {
 121         u64 proto_tce;
 122         __be64 *tcep;
 123         u64 rpn;
 124         const unsigned long tceshift = tbl->it_page_shift;
 125         const unsigned long pagesize = IOMMU_PAGE_SIZE(tbl);
 126
 127         proto_tce = TCE_PCI_READ; // Read allowed
 128
 129         if (direction != DMA_TO_DEVICE)
 130                 proto_tce |= TCE_PCI_WRITE;
 131
 132         tcep = ((__be64 *)tbl->it_base) + index;
 133
 134         while (npages--) {
 135                 /* can't move this out since we might cross MEMBLOCK boundary */
 136                 rpn = __pa(uaddr) >> tceshift;
 137                 *tcep = cpu_to_be64(proto_tce | rpn << tceshift);
 138
 139                 uaddr += pagesize;
 140                 tcep++;
 141         }
 142         return 0;
 143 }
 144
 145
 146 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
 147 {
 148         __be64 *tcep;
 149
 150         tcep = ((__be64 *)tbl->it_base) + index;
 151
 152         while (npages--)
 153                 *(tcep++) = 0;
 154 }
 155
 156 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
 157 {
 158         __be64 *tcep;
 159
 160         tcep = ((__be64 *)tbl->it_base) + index;
 161
 162         return be64_to_cpu(*tcep);
 163 }
 164
 165 static void tce_free_pSeriesLP(unsigned long liobn, long, long, long);
 166 static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);
 167
 168 static int tce_build_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 169                                 long npages, unsigned long uaddr,
 170                                 enum dma_data_direction direction,
 171                                 unsigned long attrs)
 172 {
 173         u64 rc = 0;
 174         u64 proto_tce, tce;
 175         u64 rpn;
 176         int ret = 0;
 177         long tcenum_start = tcenum, npages_start = npages;
 178
 179         rpn = __pa(uaddr) >> tceshift;
 180         proto_tce = TCE_PCI_READ;
 181         if (direction != DMA_TO_DEVICE)
 182                 proto_tce |= TCE_PCI_WRITE;
 183
 184         while (npages--) {
 185                 tce = proto_tce | rpn << tceshift;
 186                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, tce);
 187
 188                 if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 189                         ret = (int)rc;
 190                         tce_free_pSeriesLP(liobn, tcenum_start, tceshift,
 191                                            (npages_start - (npages + 1)));
 192                         break;
 193                 }
 194
 195                 if (rc && printk_ratelimit()) {
 196                         printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 197                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 198                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 199                         printk("\ttce val = 0x%llx\n", tce );
 200                         dump_stack();
 201                 }
 202
 203                 tcenum++;
 204                 rpn++;
 205         }
 206         return ret;
 207 }
 208
 209 static DEFINE_PER_CPU(__be64 *, tce_page);
 210
 211 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
 212                                      long npages, unsigned long uaddr,
 213                                      enum dma_data_direction direction,
 214                                      unsigned long attrs)
 215 {
 216         u64 rc = 0;
 217         u64 proto_tce;
 218         __be64 *tcep;
 219         u64 rpn;
 220         long l, limit;
 221         long tcenum_start = tcenum, npages_start = npages;
 222         int ret = 0;
 223         unsigned long flags;
 224         const unsigned long tceshift = tbl->it_page_shift;
 225
 226         if ((npages == 1) || !firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 227                 return tce_build_pSeriesLP(tbl->it_index, tcenum,
 228                                            tceshift, npages, uaddr,
 229                                            direction, attrs);
 230         }
 231
 232         local_irq_save(flags);  /* to protect tcep and the page behind it */
 233
 234         tcep = __this_cpu_read(tce_page);
 235
 236         /* This is safe to do since interrupts are off when we're called
 237          * from iommu_alloc{,_sg}()
 238          */
 239         if (!tcep) {
 240                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 241                 /* If allocation fails, fall back to the loop implementation */
 242                 if (!tcep) {
 243                         local_irq_restore(flags);
 244                         return tce_build_pSeriesLP(tbl->it_index, tcenum,
 245                                         tceshift,
 246                                         npages, uaddr, direction, attrs);
 247                 }
 248                 __this_cpu_write(tce_page, tcep);
 249         }
 250
 251         rpn = __pa(uaddr) >> tceshift;
 252         proto_tce = TCE_PCI_READ;
 253         if (direction != DMA_TO_DEVICE)
 254                 proto_tce |= TCE_PCI_WRITE;
 255
 256         /* We can map max one pageful of TCEs at a time */
 257         do {
 258                 /*
 259                  * Set up the page with TCE data, looping through and setting
 260                  * the values.
 261                  */
 262                 limit = min_t(long, npages, 4096 / TCE_ENTRY_SIZE);
 263
 264                 for (l = 0; l < limit; l++) {
 265                         tcep[l] = cpu_to_be64(proto_tce | rpn << tceshift);
 266                         rpn++;
 267                 }
 268
 269                 rc = plpar_tce_put_indirect((u64)tbl->it_index,
 270                                             (u64)tcenum << tceshift,
 271                                             (u64)__pa(tcep),
 272                                             limit);
 273
 274                 npages -= limit;
 275                 tcenum += limit;
 276         } while (npages > 0 && !rc);
 277
 278         local_irq_restore(flags);
 279
 280         if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
 281                 ret = (int)rc;
 282                 tce_freemulti_pSeriesLP(tbl, tcenum_start,
 283                                         (npages_start - (npages + limit)));
 284                 return ret;
 285         }
 286
 287         if (rc && printk_ratelimit()) {
 288                 printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 289                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 290                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 291                 printk("\ttce[0] val = 0x%llx\n", tcep[0]);
 292                 dump_stack();
 293         }
 294         return ret;
 295 }
 296
 297 static void tce_free_pSeriesLP(unsigned long liobn, long tcenum, long tceshift,
 298                                long npages)
 299 {
 300         u64 rc;
 301
 302         while (npages--) {
 303                 rc = plpar_tce_put((u64)liobn, (u64)tcenum << tceshift, 0);
 304
 305                 if (rc && printk_ratelimit()) {
 306                         printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
 307                         printk("\tindex   = 0x%llx\n", (u64)liobn);
 308                         printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 309                         dump_stack();
 310                 }
 311
 312                 tcenum++;
 313         }
 314 }
 315
 316
 317 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
 318 {
 319         u64 rc;
 320         long rpages = npages;
 321         unsigned long limit;
 322
 323         if (!firmware_has_feature(FW_FEATURE_STUFF_TCE))
 324                 return tce_free_pSeriesLP(tbl->it_index, tcenum,
 325                                           tbl->it_page_shift, npages);
 326
 327         do {
 328                 limit = min_t(unsigned long, rpages, 512);
 329
 330                 rc = plpar_tce_stuff((u64)tbl->it_index,
 331                                      (u64)tcenum << tbl->it_page_shift, 0, limit);
 332
 333                 rpages -= limit;
 334                 tcenum += limit;
 335         } while (rpages > 0 && !rc);
 336
 337         if (rc && printk_ratelimit()) {
 338                 printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
 339                 printk("\trc      = %lld\n", rc);
 340                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 341                 printk("\tnpages  = 0x%llx\n", (u64)npages);
 342                 dump_stack();
 343         }
 344 }
 345
 346 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
 347 {
 348         u64 rc;
 349         unsigned long tce_ret;
 350
 351         rc = plpar_tce_get((u64)tbl->it_index,
 352                            (u64)tcenum << tbl->it_page_shift, &tce_ret);
 353
 354         if (rc && printk_ratelimit()) {
 355                 printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
 356                 printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
 357                 printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
 358                 dump_stack();
 359         }
 360
 361         return tce_ret;
 362 }
 363
 364 /* this is compatible with cells for the device tree property */
 365 struct dynamic_dma_window_prop {
 366         __be32  liobn;          /* tce table number */
 367         __be64  dma_base;       /* address hi,lo */
 368         __be32  tce_shift;      /* ilog2(tce_page_size) */
 369         __be32  window_shift;   /* ilog2(tce_window_size) */
 370 };
 371
 372 struct dma_win {
 373         struct device_node *device;
 374         const struct dynamic_dma_window_prop *prop;
 375         bool    direct;
 376         struct list_head list;
 377 };
 378
 379 /* Dynamic DMA Window support */
 380 struct ddw_query_response {
 381         u32 windows_available;
 382         u64 largest_available_block;
 383         u32 page_size;
 384         u32 migration_capable;
 385 };
 386
 387 struct ddw_create_response {
 388         u32 liobn;
 389         u32 addr_hi;
 390         u32 addr_lo;
 391 };
 392
 393 static LIST_HEAD(dma_win_list);
 394 /* prevents races between memory on/offline and window creation */
 395 static DEFINE_SPINLOCK(dma_win_list_lock);
 396 /* protects initializing window twice for same device */
 397 static DEFINE_MUTEX(dma_win_init_mutex);
 398 #define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"
 399 #define DMA64_PROPNAME "linux,dma64-ddr-window-info"
 400
 401 static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
 402                                         unsigned long num_pfn, const void *arg)
 403 {
 404         const struct dynamic_dma_window_prop *maprange = arg;
 405         int rc;
 406         u64 tce_size, num_tce, dma_offset, next;
 407         u32 tce_shift;
 408         long limit;
 409
 410         tce_shift = be32_to_cpu(maprange->tce_shift);
 411         tce_size = 1ULL << tce_shift;
 412         next = start_pfn << PAGE_SHIFT;
 413         num_tce = num_pfn << PAGE_SHIFT;
 414
 415         /* round back to the beginning of the tce page size */
 416         num_tce += next & (tce_size - 1);
 417         next &= ~(tce_size - 1);
 418
 419         /* covert to number of tces */
 420         num_tce |= tce_size - 1;
 421         num_tce >>= tce_shift;
 422
 423         do {
 424                 /*
 425                  * Set up the page with TCE data, looping through and setting
 426                  * the values.
 427                  */
 428                 limit = min_t(long, num_tce, 512);
 429                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 430
 431                 rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
 432                                              dma_offset,
 433                                              0, limit);
 434                 next += limit * tce_size;
 435                 num_tce -= limit;
 436         } while (num_tce > 0 && !rc);
 437
 438         return rc;
 439 }
 440
 441 static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
 442                                         unsigned long num_pfn, const void *arg)
 443 {
 444         const struct dynamic_dma_window_prop *maprange = arg;
 445         u64 tce_size, num_tce, dma_offset, next, proto_tce, liobn;
 446         __be64 *tcep;
 447         u32 tce_shift;
 448         u64 rc = 0;
 449         long l, limit;
 450
 451         if (!firmware_has_feature(FW_FEATURE_PUT_TCE_IND)) {
 452                 unsigned long tceshift = be32_to_cpu(maprange->tce_shift);
 453                 unsigned long dmastart = (start_pfn << PAGE_SHIFT) +
 454                                 be64_to_cpu(maprange->dma_base);
 455                 unsigned long tcenum = dmastart >> tceshift;
 456                 unsigned long npages = num_pfn << PAGE_SHIFT >> tceshift;
 457                 void *uaddr = __va(start_pfn << PAGE_SHIFT);
 458
 459                 return tce_build_pSeriesLP(be32_to_cpu(maprange->liobn),
 460                                 tcenum, tceshift, npages, (unsigned long) uaddr,
 461                                 DMA_BIDIRECTIONAL, 0);
 462         }
 463
 464         local_irq_disable();    /* to protect tcep and the page behind it */
 465         tcep = __this_cpu_read(tce_page);
 466
 467         if (!tcep) {
 468                 tcep = (__be64 *)__get_free_page(GFP_ATOMIC);
 469                 if (!tcep) {
 470                         local_irq_enable();
 471                         return -ENOMEM;
 472                 }
 473                 __this_cpu_write(tce_page, tcep);
 474         }
 475
 476         proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;
 477
 478         liobn = (u64)be32_to_cpu(maprange->liobn);
 479         tce_shift = be32_to_cpu(maprange->tce_shift);
 480         tce_size = 1ULL << tce_shift;
 481         next = start_pfn << PAGE_SHIFT;
 482         num_tce = num_pfn << PAGE_SHIFT;
 483
 484         /* round back to the beginning of the tce page size */
 485         num_tce += next & (tce_size - 1);
 486         next &= ~(tce_size - 1);
 487
 488         /* covert to number of tces */
 489         num_tce |= tce_size - 1;
 490         num_tce >>= tce_shift;
 491
 492         /* We can map max one pageful of TCEs at a time */
 493         do {
 494                 /*
 495                  * Set up the page with TCE data, looping through and setting
 496                  * the values.
 497                  */
 498                 limit = min_t(long, num_tce, 4096 / TCE_ENTRY_SIZE);
 499                 dma_offset = next + be64_to_cpu(maprange->dma_base);
 500
 501                 for (l = 0; l < limit; l++) {
 502                         tcep[l] = cpu_to_be64(proto_tce | next);
 503                         next += tce_size;
 504                 }
 505
 506                 rc = plpar_tce_put_indirect(liobn,
 507                                             dma_offset,
 508                                             (u64)__pa(tcep),
 509                                             limit);
 510
 511                 num_tce -= limit;
 512         } while (num_tce > 0 && !rc);
 513
 514         /* error cleanup: caller will clear whole range */
 515
 516         local_irq_enable();
 517         return rc;
 518 }
 519
 520 static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
 521                 unsigned long num_pfn, void *arg)
 522 {
 523         return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
 524 }
 525
 526 static void iommu_table_setparms_common(struct iommu_table *tbl, unsigned long busno,
 527                                         unsigned long liobn, unsigned long win_addr,
 528                                         unsigned long window_size, unsigned long page_shift,
 529                                         void *base, struct iommu_table_ops *table_ops)
 530 {
 531         tbl->it_busno = busno;
 532         tbl->it_index = liobn;
 533         tbl->it_offset = win_addr >> page_shift;
 534         tbl->it_size = window_size >> page_shift;
 535         tbl->it_page_shift = page_shift;
 536         tbl->it_base = (unsigned long)base;
 537         tbl->it_blocksize = 16;
 538         tbl->it_type = TCE_PCI;
 539         tbl->it_ops = table_ops;
 540 }
 541
 542 struct iommu_table_ops iommu_table_pseries_ops;
 543
 544 static void iommu_table_setparms(struct pci_controller *phb,
 545                                  struct device_node *dn,
 546                                  struct iommu_table *tbl)
 547 {
 548         struct device_node *node;
 549         const unsigned long *basep;
 550         const u32 *sizep;
 551
 552         /* Test if we are going over 2GB of DMA space */
 553         if (phb->dma_window_base_cur + phb->dma_window_size > SZ_2G) {
 554                 udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 555                 panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
 556         }
 557
 558         node = phb->dn;
 559         basep = of_get_property(node, "linux,tce-base", NULL);
 560         sizep = of_get_property(node, "linux,tce-size", NULL);
 561         if (basep == NULL || sizep == NULL) {
 562                 printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %pOF has "
 563                                 "missing tce entries !\n", dn);
 564                 return;
 565         }
 566
 567         iommu_table_setparms_common(tbl, phb->bus->number, 0, phb->dma_window_base_cur,
 568                                     phb->dma_window_size, IOMMU_PAGE_SHIFT_4K,
 569                                     __va(*basep), &iommu_table_pseries_ops);
 570
 571         if (!is_kdump_kernel())
 572                 memset((void *)tbl->it_base, 0, *sizep);
 573
 574         phb->dma_window_base_cur += phb->dma_window_size;
 575 }
 576
 577 struct iommu_table_ops iommu_table_lpar_multi_ops;
 578
 579 /*
 580  * iommu_table_setparms_lpar
 581  *
 582  * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
 583  */
 584 static void iommu_table_setparms_lpar(struct pci_controller *phb,
 585                                       struct device_node *dn,
 586                                       struct iommu_table *tbl,
 587                                       struct iommu_table_group *table_group,
 588                                       const __be32 *dma_window)
 589 {
 590         unsigned long offset, size, liobn;
 591
 592         of_parse_dma_window(dn, dma_window, &liobn, &offset, &size);
 593
 594         iommu_table_setparms_common(tbl, phb->bus->number, liobn, offset, size, IOMMU_PAGE_SHIFT_4K, NULL,
 595                                     &iommu_table_lpar_multi_ops);
 596
 597
 598         table_group->tce32_start = offset;
 599         table_group->tce32_size = size;
 600 }
 601
 602 struct iommu_table_ops iommu_table_pseries_ops = {
 603         .set = tce_build_pSeries,
 604         .clear = tce_free_pSeries,
 605         .get = tce_get_pseries
 606 };
 607
 608 static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
 609 {
 610         struct device_node *dn;
 611         struct iommu_table *tbl;
 612         struct device_node *isa_dn, *isa_dn_orig;
 613         struct device_node *tmp;
 614         struct pci_dn *pci;
 615         int children;
 616
 617         dn = pci_bus_to_OF_node(bus);
 618
 619         pr_debug("pci_dma_bus_setup_pSeries: setting up bus %pOF\n", dn);
 620
 621         if (bus->self) {
 622                 /* This is not a root bus, any setup will be done for the
 623                  * device-side of the bridge in iommu_dev_setup_pSeries().
 624                  */
 625                 return;
 626         }
 627         pci = PCI_DN(dn);
 628
 629         /* Check if the ISA bus on the system is under
 630          * this PHB.
 631          */
 632         isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
 633
 634         while (isa_dn && isa_dn != dn)
 635                 isa_dn = isa_dn->parent;
 636
 637         of_node_put(isa_dn_orig);
 638
 639         /* Count number of direct PCI children of the PHB. */
 640         for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
 641                 children++;
 642
 643         pr_debug("Children: %d\n", children);
 644
 645         /* Calculate amount of DMA window per slot. Each window must be
 646          * a power of two (due to pci_alloc_consistent requirements).
 647          *
 648          * Keep 256MB aside for PHBs with ISA.
 649          */
 650
 651         if (!isa_dn) {
 652                 /* No ISA/IDE - just set window size and return */
 653                 pci->phb->dma_window_size = 0x80000000ul; /* To be divided */
 654
 655                 while (pci->phb->dma_window_size * children > 0x80000000ul)
 656                         pci->phb->dma_window_size >>= 1;
 657                 pr_debug("No ISA/IDE, window size is 0x%llx\n",
 658                          pci->phb->dma_window_size);
 659                 pci->phb->dma_window_base_cur = 0;
 660
 661                 return;
 662         }
 663
 664         /* If we have ISA, then we probably have an IDE
 665          * controller too. Allocate a 128MB table but
 666          * skip the first 128MB to avoid stepping on ISA
 667          * space.
 668          */
 669         pci->phb->dma_window_size = 0x8000000ul;
 670         pci->phb->dma_window_base_cur = 0x8000000ul;
 671
 672         pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
 673         tbl = pci->table_group->tables[0];
 674
 675         iommu_table_setparms(pci->phb, dn, tbl);
 676
 677         if (!iommu_init_table(tbl, pci->phb->node, 0, 0))
 678                 panic("Failed to initialize iommu table");
 679
 680         /* Divide the rest (1.75GB) among the children */
 681         pci->phb->dma_window_size = 0x80000000ul;
 682         while (pci->phb->dma_window_size * children > 0x70000000ul)
 683                 pci->phb->dma_window_size >>= 1;
 684
 685         pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
 686 }
 687
 688 #ifdef CONFIG_IOMMU_API
 689 static int tce_exchange_pseries(struct iommu_table *tbl, long index, unsigned
 690                                 long *tce, enum dma_data_direction *direction)
 691 {
 692         long rc;
 693         unsigned long ioba = (unsigned long) index << tbl->it_page_shift;
 694         unsigned long flags, oldtce = 0;
 695         u64 proto_tce = iommu_direction_to_tce_perm(*direction);
 696         unsigned long newtce = *tce | proto_tce;
 697
 698         spin_lock_irqsave(&tbl->large_pool.lock, flags);
 699
 700         rc = plpar_tce_get((u64)tbl->it_index, ioba, &oldtce);
 701         if (!rc)
 702                 rc = plpar_tce_put((u64)tbl->it_index, ioba, newtce);
 703
 704         if (!rc) {
 705                 *direction = iommu_tce_direction(oldtce);
 706                 *tce = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE);
 707         }
 708
 709         spin_unlock_irqrestore(&tbl->large_pool.lock, flags);
 710
 711         return rc;
 712 }
 713 #endif
 714
 715 struct iommu_table_ops iommu_table_lpar_multi_ops = {
 716         .set = tce_buildmulti_pSeriesLP,
 717 #ifdef CONFIG_IOMMU_API
 718         .xchg_no_kill = tce_exchange_pseries,
 719 #endif
 720         .clear = tce_freemulti_pSeriesLP,
 721         .get = tce_get_pSeriesLP
 722 };
 723
 724 /*
 725  * Find nearest ibm,dma-window (default DMA window) or direct DMA window or
 726  * dynamic 64bit DMA window, walking up the device tree.
 727  */
 728 static struct device_node *pci_dma_find(struct device_node *dn,
 729                                         const __be32 **dma_window)
 730 {
 731         const __be32 *dw = NULL;
 732
 733         for ( ; dn && PCI_DN(dn); dn = dn->parent) {
 734                 dw = of_get_property(dn, "ibm,dma-window", NULL);
 735                 if (dw) {
 736                         if (dma_window)
 737                                 *dma_window = dw;
 738                         return dn;
 739                 }
 740                 dw = of_get_property(dn, DIRECT64_PROPNAME, NULL);
 741                 if (dw)
 742                         return dn;
 743                 dw = of_get_property(dn, DMA64_PROPNAME, NULL);
 744                 if (dw)
 745                         return dn;
 746         }
 747
 748         return NULL;
 749 }
 750
 751 static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
 752 {
 753         struct iommu_table *tbl;
 754         struct device_node *dn, *pdn;
 755         struct pci_dn *ppci;
 756         const __be32 *dma_window = NULL;
 757
 758         dn = pci_bus_to_OF_node(bus);
 759
 760         pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %pOF\n",
 761                  dn);
 762
 763         pdn = pci_dma_find(dn, &dma_window);
 764
 765         if (dma_window == NULL)
 766                 pr_debug("  no ibm,dma-window property !\n");
 767
 768         ppci = PCI_DN(pdn);
 769
 770         pr_debug("  parent is %pOF, iommu_table: 0x%p\n",
 771                  pdn, ppci->table_group);
 772
 773         if (!ppci->table_group) {
 774                 ppci->table_group = iommu_pseries_alloc_group(ppci->phb->node);
 775                 tbl = ppci->table_group->tables[0];
 776                 if (dma_window) {
 777                         iommu_table_setparms_lpar(ppci->phb, pdn, tbl,
 778                                                   ppci->table_group, dma_window);
 779
 780                         if (!iommu_init_table(tbl, ppci->phb->node, 0, 0))
 781                                 panic("Failed to initialize iommu table");
 782                 }
 783                 iommu_register_group(ppci->table_group,
 784                                 pci_domain_nr(bus), 0);
 785                 pr_debug("  created table: %p\n", ppci->table_group);
 786         }
 787 }
 788
 789
 790 static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
 791 {
 792         struct device_node *dn;
 793         struct iommu_table *tbl;
 794
 795         pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
 796
 797         dn = dev->dev.of_node;
 798
 799         /* If we're the direct child of a root bus, then we need to allocate
 800          * an iommu table ourselves. The bus setup code should have setup
 801          * the window sizes already.
 802          */
 803         if (!dev->bus->self) {
 804                 struct pci_controller *phb = PCI_DN(dn)->phb;
 805
 806                 pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
 807                 PCI_DN(dn)->table_group = iommu_pseries_alloc_group(phb->node);
 808                 tbl = PCI_DN(dn)->table_group->tables[0];
 809                 iommu_table_setparms(phb, dn, tbl);
 810
 811                 if (!iommu_init_table(tbl, phb->node, 0, 0))
 812                         panic("Failed to initialize iommu table");
 813
 814                 set_iommu_table_base(&dev->dev, tbl);
 815                 return;
 816         }
 817
 818         /* If this device is further down the bus tree, search upwards until
 819          * an already allocated iommu table is found and use that.
 820          */
 821
 822         while (dn && PCI_DN(dn) && PCI_DN(dn)->table_group == NULL)
 823                 dn = dn->parent;
 824
 825         if (dn && PCI_DN(dn))
 826                 set_iommu_table_base(&dev->dev,
 827                                 PCI_DN(dn)->table_group->tables[0]);
 828         else
 829                 printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
 830                        pci_name(dev));
 831 }
 832
 833 static int __read_mostly disable_ddw;
 834
 835 static int __init disable_ddw_setup(char *str)
 836 {
 837         disable_ddw = 1;
 838         printk(KERN_INFO "ppc iommu: disabling ddw.\n");
 839
 840         return 0;
 841 }
 842
 843 early_param("disable_ddw", disable_ddw_setup);
 844
 845 static void clean_dma_window(struct device_node *np, struct dynamic_dma_window_prop *dwp)
 846 {
 847         int ret;
 848
 849         ret = tce_clearrange_multi_pSeriesLP(0,
 850                 1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
 851         if (ret)
 852                 pr_warn("%pOF failed to clear tces in window.\n",
 853                         np);
 854         else
 855                 pr_debug("%pOF successfully cleared tces in window.\n",
 856                          np);
 857 }
 858
 859 /*
 860  * Call only if DMA window is clean.
 861  */
 862 static void __remove_dma_window(struct device_node *np, u32 *ddw_avail, u64 liobn)
 863 {
 864         int ret;
 865
 866         ret = rtas_call(ddw_avail[DDW_REMOVE_PE_DMA_WIN], 1, 1, NULL, liobn);
 867         if (ret)
 868                 pr_warn("%pOF: failed to remove DMA window: rtas returned "
 869                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 870                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 871         else
 872                 pr_debug("%pOF: successfully removed DMA window: rtas returned "
 873                         "%d to ibm,remove-pe-dma-window(%x) %llx\n",
 874                         np, ret, ddw_avail[DDW_REMOVE_PE_DMA_WIN], liobn);
 875 }
 876
 877 static void remove_dma_window(struct device_node *np, u32 *ddw_avail,
 878                               struct property *win)
 879 {
 880         struct dynamic_dma_window_prop *dwp;
 881         u64 liobn;
 882
 883         dwp = win->value;
 884         liobn = (u64)be32_to_cpu(dwp->liobn);
 885
 886         clean_dma_window(np, dwp);
 887         __remove_dma_window(np, ddw_avail, liobn);
 888 }
 889
 890 static int remove_ddw(struct device_node *np, bool remove_prop, const char *win_name)
 891 {
 892         struct property *win;
 893         u32 ddw_avail[DDW_APPLICABLE_SIZE];
 894         int ret = 0;
 895
 896         win = of_find_property(np, win_name, NULL);
 897         if (!win)
 898                 return -EINVAL;
 899
 900         ret = of_property_read_u32_array(np, "ibm,ddw-applicable",
 901                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
 902         if (ret)
 903                 return 0;
 904
 905
 906         if (win->length >= sizeof(struct dynamic_dma_window_prop))
 907                 remove_dma_window(np, ddw_avail, win);
 908
 909         if (!remove_prop)
 910                 return 0;
 911
 912         ret = of_remove_property(np, win);
 913         if (ret)
 914                 pr_warn("%pOF: failed to remove DMA window property: %d\n",
 915                         np, ret);
 916         return 0;
 917 }
 918
 919 static bool find_existing_ddw(struct device_node *pdn, u64 *dma_addr, int *window_shift)
 920 {
 921         struct dma_win *window;
 922         const struct dynamic_dma_window_prop *dma64;
 923         bool found = false;
 924
 925         spin_lock(&dma_win_list_lock);
 926         /* check if we already created a window and dupe that config if so */
 927         list_for_each_entry(window, &dma_win_list, list) {
 928                 if (window->device == pdn) {
 929                         dma64 = window->prop;
 930                         *dma_addr = be64_to_cpu(dma64->dma_base);
 931                         *window_shift = be32_to_cpu(dma64->window_shift);
 932                         found = true;
 933                         break;
 934                 }
 935         }
 936         spin_unlock(&dma_win_list_lock);
 937
 938         return found;
 939 }
 940
 941 static struct dma_win *ddw_list_new_entry(struct device_node *pdn,
 942                                           const struct dynamic_dma_window_prop *dma64)
 943 {
 944         struct dma_win *window;
 945
 946         window = kzalloc(sizeof(*window), GFP_KERNEL);
 947         if (!window)
 948                 return NULL;
 949
 950         window->device = pdn;
 951         window->prop = dma64;
 952         window->direct = false;
 953
 954         return window;
 955 }
 956
 957 static void find_existing_ddw_windows_named(const char *name)
 958 {
 959         int len;
 960         struct device_node *pdn;
 961         struct dma_win *window;
 962         const struct dynamic_dma_window_prop *dma64;
 963
 964         for_each_node_with_property(pdn, name) {
 965                 dma64 = of_get_property(pdn, name, &len);
 966                 if (!dma64 || len < sizeof(*dma64)) {
 967                         remove_ddw(pdn, true, name);
 968                         continue;
 969                 }
 970
 971                 window = ddw_list_new_entry(pdn, dma64);
 972                 if (!window) {
 973                         of_node_put(pdn);
 974                         break;
 975                 }
 976
 977                 spin_lock(&dma_win_list_lock);
 978                 list_add(&window->list, &dma_win_list);
 979                 spin_unlock(&dma_win_list_lock);
 980         }
 981 }
 982
 983 static int find_existing_ddw_windows(void)
 984 {
 985         if (!firmware_has_feature(FW_FEATURE_LPAR))
 986                 return 0;
 987
 988         find_existing_ddw_windows_named(DIRECT64_PROPNAME);
 989         find_existing_ddw_windows_named(DMA64_PROPNAME);
 990
 991         return 0;
 992 }
 993 machine_arch_initcall(pseries, find_existing_ddw_windows);
 994
 995 /**
 996  * ddw_read_ext - Get the value of an DDW extension
 997  * @np:         device node from which the extension value is to be read.
 998  * @extnum:     index number of the extension.
 999  * @value:      pointer to return value, modified when extension is available.
1000  *
1001  * Checks if "ibm,ddw-extensions" exists for this node, and get the value
1002  * on index 'extnum'.
1003  * It can be used only to check if a property exists, passing value == NULL.
1004  *
1005  * Returns:
1006  *      0 if extension successfully read
1007  *      -EINVAL if the "ibm,ddw-extensions" does not exist,
1008  *      -ENODATA if "ibm,ddw-extensions" does not have a value, and
1009  *      -EOVERFLOW if "ibm,ddw-extensions" does not contain this extension.
1010  */
1011 static inline int ddw_read_ext(const struct device_node *np, int extnum,
1012                                u32 *value)
1013 {
1014         static const char propname[] = "ibm,ddw-extensions";
1015         u32 count;
1016         int ret;
1017
1018         ret = of_property_read_u32_index(np, propname, DDW_EXT_SIZE, &count);
1019         if (ret)
1020                 return ret;
1021
1022         if (count < extnum)
1023                 return -EOVERFLOW;
1024
1025         if (!value)
1026                 value = &count;
1027
1028         return of_property_read_u32_index(np, propname, extnum, value);
1029 }
1030
1031 static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1032                      struct ddw_query_response *query,
1033                      struct device_node *parent)
1034 {
1035         struct device_node *dn;
1036         struct pci_dn *pdn;
1037         u32 cfg_addr, ext_query, query_out[5];
1038         u64 buid;
1039         int ret, out_sz;
1040
1041         /*
1042          * From LoPAR level 2.8, "ibm,ddw-extensions" index 3 can rule how many
1043          * output parameters ibm,query-pe-dma-windows will have, ranging from
1044          * 5 to 6.
1045          */
1046         ret = ddw_read_ext(parent, DDW_EXT_QUERY_OUT_SIZE, &ext_query);
1047         if (!ret && ext_query == 1)
1048                 out_sz = 6;
1049         else
1050                 out_sz = 5;
1051
1052         /*
1053          * Get the config address and phb buid of the PE window.
1054          * Rely on eeh to retrieve this for us.
1055          * Retrieve them from the pci device, not the node with the
1056          * dma-window property
1057          */
1058         dn = pci_device_to_OF_node(dev);
1059         pdn = PCI_DN(dn);
1060         buid = pdn->phb->buid;
1061         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1062
1063         ret = rtas_call(ddw_avail[DDW_QUERY_PE_DMA_WIN], 3, out_sz, query_out,
1064                         cfg_addr, BUID_HI(buid), BUID_LO(buid));
1065
1066         switch (out_sz) {
1067         case 5:
1068                 query->windows_available = query_out[0];
1069                 query->largest_available_block = query_out[1];
1070                 query->page_size = query_out[2];
1071                 query->migration_capable = query_out[3];
1072                 break;
1073         case 6:
1074                 query->windows_available = query_out[0];
1075                 query->largest_available_block = ((u64)query_out[1] << 32) |
1076                                                  query_out[2];
1077                 query->page_size = query_out[3];
1078                 query->migration_capable = query_out[4];
1079                 break;
1080         }
1081
1082         dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x returned %d, lb=%llx ps=%x wn=%d\n",
1083                  ddw_avail[DDW_QUERY_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1084                  BUID_LO(buid), ret, query->largest_available_block,
1085                  query->page_size, query->windows_available);
1086
1087         return ret;
1088 }
1089
1090 static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
1091                         struct ddw_create_response *create, int page_shift,
1092                         int window_shift)
1093 {
1094         struct device_node *dn;
1095         struct pci_dn *pdn;
1096         u32 cfg_addr;
1097         u64 buid;
1098         int ret;
1099
1100         /*
1101          * Get the config address and phb buid of the PE window.
1102          * Rely on eeh to retrieve this for us.
1103          * Retrieve them from the pci device, not the node with the
1104          * dma-window property
1105          */
1106         dn = pci_device_to_OF_node(dev);
1107         pdn = PCI_DN(dn);
1108         buid = pdn->phb->buid;
1109         cfg_addr = ((pdn->busno << 16) | (pdn->devfn << 8));
1110
1111         do {
1112                 /* extra outputs are LIOBN and dma-addr (hi, lo) */
1113                 ret = rtas_call(ddw_avail[DDW_CREATE_PE_DMA_WIN], 5, 4,
1114                                 (u32 *)create, cfg_addr, BUID_HI(buid),
1115                                 BUID_LO(buid), page_shift, window_shift);
1116         } while (rtas_busy_delay(ret));
1117         dev_info(&dev->dev,
1118                 "ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
1119                 "(liobn = 0x%x starting addr = %x %x)\n",
1120                  ddw_avail[DDW_CREATE_PE_DMA_WIN], cfg_addr, BUID_HI(buid),
1121                  BUID_LO(buid), page_shift, window_shift, ret, create->liobn,
1122                  create->addr_hi, create->addr_lo);
1123
1124         return ret;
1125 }
1126
1127 struct failed_ddw_pdn {
1128         struct device_node *pdn;
1129         struct list_head list;
1130 };
1131
1132 static LIST_HEAD(failed_ddw_pdn_list);
1133
1134 static phys_addr_t ddw_memory_hotplug_max(void)
1135 {
1136         resource_size_t max_addr = memory_hotplug_max();
1137         struct device_node *memory;
1138
1139         for_each_node_by_type(memory, "memory") {
1140                 struct resource res;
1141
1142                 if (of_address_to_resource(memory, 0, &res))
1143                         continue;
1144
1145                 max_addr = max_t(resource_size_t, max_addr, res.end + 1);
1146         }
1147
1148         return max_addr;
1149 }
1150
1151 /*
1152  * Platforms supporting the DDW option starting with LoPAR level 2.7 implement
1153  * ibm,ddw-extensions, which carries the rtas token for
1154  * ibm,reset-pe-dma-windows.
1155  * That rtas-call can be used to restore the default DMA window for the device.
1156  */
1157 static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn)
1158 {
1159         int ret;
1160         u32 cfg_addr, reset_dma_win;
1161         u64 buid;
1162         struct device_node *dn;
1163         struct pci_dn *pdn;
1164
1165         ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win);
1166         if (ret)
1167                 return;
1168
1169         dn = pci_device_to_OF_node(dev);
1170         pdn = PCI_DN(dn);
1171         buid = pdn->phb->buid;
1172         cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8);
1173
1174         ret = rtas_call(reset_dma_win, 3, 1, NULL, cfg_addr, BUID_HI(buid),
1175                         BUID_LO(buid));
1176         if (ret)
1177                 dev_info(&dev->dev,
1178                          "ibm,reset-pe-dma-windows(%x) %x %x %x returned %d ",
1179                          reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid),
1180                          ret);
1181 }
1182
1183 /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */
1184 static int iommu_get_page_shift(u32 query_page_size)
1185 {
1186         /* Supported IO page-sizes according to LoPAR, note that 2M is out of order */
1187         const int shift[] = {
1188                 __builtin_ctzll(SZ_4K),   __builtin_ctzll(SZ_64K), __builtin_ctzll(SZ_16M),
1189                 __builtin_ctzll(SZ_32M),  __builtin_ctzll(SZ_64M), __builtin_ctzll(SZ_128M),
1190                 __builtin_ctzll(SZ_256M), __builtin_ctzll(SZ_16G), __builtin_ctzll(SZ_2M)
1191         };
1192
1193         int i = ARRAY_SIZE(shift) - 1;
1194         int ret = 0;
1195
1196         /*
1197          * On LoPAR, ibm,query-pe-dma-window outputs "IO Page Sizes" using a bit field:
1198          * - bit 31 means 4k pages are supported,
1199          * - bit 30 means 64k pages are supported, and so on.
1200          * Larger pagesizes map more memory with the same amount of TCEs, so start probing them.
1201          */
1202         for (; i >= 0 ; i--) {
1203                 if (query_page_size & (1 << i))
1204                         ret = max(ret, shift[i]);
1205         }
1206
1207         return ret;
1208 }
1209
1210 static struct property *ddw_property_create(const char *propname, u32 liobn, u64 dma_addr,
1211                                             u32 page_shift, u32 window_shift)
1212 {
1213         struct dynamic_dma_window_prop *ddwprop;
1214         struct property *win64;
1215
1216         win64 = kzalloc(sizeof(*win64), GFP_KERNEL);
1217         if (!win64)
1218                 return NULL;
1219
1220         win64->name = kstrdup(propname, GFP_KERNEL);
1221         ddwprop = kzalloc(sizeof(*ddwprop), GFP_KERNEL);
1222         win64->value = ddwprop;
1223         win64->length = sizeof(*ddwprop);
1224         if (!win64->name || !win64->value) {
1225                 kfree(win64->name);
1226                 kfree(win64->value);
1227                 kfree(win64);
1228                 return NULL;
1229         }
1230
1231         ddwprop->liobn = cpu_to_be32(liobn);
1232         ddwprop->dma_base = cpu_to_be64(dma_addr);
1233         ddwprop->tce_shift = cpu_to_be32(page_shift);
1234         ddwprop->window_shift = cpu_to_be32(window_shift);
1235
1236         return win64;
1237 }
1238
1239 /*
1240  * If the PE supports dynamic dma windows, and there is space for a table
1241  * that can map all pages in a linear offset, then setup such a table,
1242  * and record the dma-offset in the struct device.
1243  *
1244  * dev: the pci device we are checking
1245  * pdn: the parent pe node with the ibm,dma_window property
1246  * Future: also check if we can remap the base window for our base page size
1247  *
1248  * returns true if can map all pages (direct mapping), false otherwise..
1249  */
1250 static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn)
1251 {
1252         int len = 0, ret;
1253         int max_ram_len = order_base_2(ddw_memory_hotplug_max());
1254         struct ddw_query_response query;
1255         struct ddw_create_response create;
1256         int page_shift;
1257         u64 win_addr;
1258         const char *win_name;
1259         struct device_node *dn;
1260         u32 ddw_avail[DDW_APPLICABLE_SIZE];
1261         struct dma_win *window;
1262         struct property *win64;
1263         struct failed_ddw_pdn *fpdn;
1264         bool default_win_removed = false, direct_mapping = false;
1265         bool pmem_present;
1266         struct pci_dn *pci = PCI_DN(pdn);
1267         struct property *default_win = NULL;
1268
1269         dn = of_find_node_by_type(NULL, "ibm,pmemory");
1270         pmem_present = dn != NULL;
1271         of_node_put(dn);
1272
1273         mutex_lock(&dma_win_init_mutex);
1274
1275         if (find_existing_ddw(pdn, &dev->dev.archdata.dma_offset, &len)) {
1276                 direct_mapping = (len >= max_ram_len);
1277                 goto out_unlock;
1278         }
1279
1280         /*
1281          * If we already went through this for a previous function of
1282          * the same device and failed, we don't want to muck with the
1283          * DMA window again, as it will race with in-flight operations
1284          * and can lead to EEHs. The above mutex protects access to the
1285          * list.
1286          */
1287         list_for_each_entry(fpdn, &failed_ddw_pdn_list, list) {
1288                 if (fpdn->pdn == pdn)
1289                         goto out_unlock;
1290         }
1291
1292         /*
1293          * the ibm,ddw-applicable property holds the tokens for:
1294          * ibm,query-pe-dma-window
1295          * ibm,create-pe-dma-window
1296          * ibm,remove-pe-dma-window
1297          * for the given node in that order.
1298          * the property is actually in the parent, not the PE
1299          */
1300         ret = of_property_read_u32_array(pdn, "ibm,ddw-applicable",
1301                                          &ddw_avail[0], DDW_APPLICABLE_SIZE);
1302         if (ret)
1303                 goto out_failed;
1304
1305        /*
1306          * Query if there is a second window of size to map the
1307          * whole partition.  Query returns number of windows, largest
1308          * block assigned to PE (partition endpoint), and two bitmasks
1309          * of page sizes: supported and supported for migrate-dma.
1310          */
1311         dn = pci_device_to_OF_node(dev);
1312         ret = query_ddw(dev, ddw_avail, &query, pdn);
1313         if (ret != 0)
1314                 goto out_failed;
1315
1316         /*
1317          * If there is no window available, remove the default DMA window,
1318          * if it's present. This will make all the resources available to the
1319          * new DDW window.
1320          * If anything fails after this, we need to restore it, so also check
1321          * for extensions presence.
1322          */
1323         if (query.windows_available == 0) {
1324                 int reset_win_ext;
1325
1326                 /* DDW + IOMMU on single window may fail if there is any allocation */
1327                 if (iommu_table_in_use(pci->table_group->tables[0])) {
1328                         dev_warn(&dev->dev, "current IOMMU table in use, can't be replaced.\n");
1329                         goto out_failed;
1330                 }
1331
1332                 default_win = of_find_property(pdn, "ibm,dma-window", NULL);
1333                 if (!default_win)
1334                         goto out_failed;
1335
1336                 reset_win_ext = ddw_read_ext(pdn, DDW_EXT_RESET_DMA_WIN, NULL);
1337                 if (reset_win_ext)
1338                         goto out_failed;
1339
1340                 remove_dma_window(pdn, ddw_avail, default_win);
1341                 default_win_removed = true;
1342
1343                 /* Query again, to check if the window is available */
1344                 ret = query_ddw(dev, ddw_avail, &query, pdn);
1345                 if (ret != 0)
1346                         goto out_failed;
1347
1348                 if (query.windows_available == 0) {
1349                         /* no windows are available for this device. */
1350                         dev_dbg(&dev->dev, "no free dynamic windows");
1351                         goto out_failed;
1352                 }
1353         }
1354
1355         page_shift = iommu_get_page_shift(query.page_size);
1356         if (!page_shift) {
1357                 dev_dbg(&dev->dev, "no supported page size in mask %x",
1358                         query.page_size);
1359                 goto out_failed;
1360         }
1361
1362
1363         /*
1364          * The "ibm,pmemory" can appear anywhere in the address space.
1365          * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS
1366          * for the upper limit and fallback to max RAM otherwise but this
1367          * disables device::dma_ops_bypass.
1368          */
1369         len = max_ram_len;
1370         if (pmem_present) {
1371                 if (query.largest_available_block >=
1372                     (1ULL << (MAX_PHYSMEM_BITS - page_shift)))
1373                         len = MAX_PHYSMEM_BITS;
1374                 else
1375                         dev_info(&dev->dev, "Skipping ibm,pmemory");
1376         }
1377
1378         /* check if the available block * number of ptes will map everything */
1379         if (query.largest_available_block < (1ULL << (len - page_shift))) {
1380                 dev_dbg(&dev->dev,
1381                         "can't map partition max 0x%llx with %llu %llu-sized pages\n",
1382                         1ULL << len,
1383                         query.largest_available_block,
1384                         1ULL << page_shift);
1385
1386                 len = order_base_2(query.largest_available_block << page_shift);
1387                 win_name = DMA64_PROPNAME;
1388         } else {
1389                 direct_mapping = !default_win_removed ||
1390                         (len == MAX_PHYSMEM_BITS) ||
1391                         (!pmem_present && (len == max_ram_len));
1392                 win_name = direct_mapping ? DIRECT64_PROPNAME : DMA64_PROPNAME;
1393         }
1394
1395         ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
1396         if (ret != 0)
1397                 goto out_failed;
1398
1399         dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %pOF\n",
1400                   create.liobn, dn);
1401
1402         win_addr = ((u64)create.addr_hi << 32) | create.addr_lo;
1403         win64 = ddw_property_create(win_name, create.liobn, win_addr, page_shift, len);
1404
1405         if (!win64) {
1406                 dev_info(&dev->dev,
1407                          "couldn't allocate property, property name, or value\n");
1408                 goto out_remove_win;
1409         }
1410
1411         ret = of_add_property(pdn, win64);
1412         if (ret) {
1413                 dev_err(&dev->dev, "unable to add DMA window property for %pOF: %d",
1414                         pdn, ret);
1415                 goto out_free_prop;
1416         }
1417
1418         window = ddw_list_new_entry(pdn, win64->value);
1419         if (!window)
1420                 goto out_del_prop;
1421
1422         if (direct_mapping) {
1423                 window->direct = true;
1424
1425                 /* DDW maps the whole partition, so enable direct DMA mapping */
1426                 ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
1427                                             win64->value, tce_setrange_multi_pSeriesLP_walk);
1428                 if (ret) {
1429                         dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n",
1430                                  dn, ret);
1431
1432                         /* Make sure to clean DDW if any TCE was set*/
1433                         clean_dma_window(pdn, win64->value);
1434                         goto out_del_list;
1435                 }
1436         } else {
1437                 struct iommu_table *newtbl;
1438                 int i;
1439                 unsigned long start = 0, end = 0;
1440
1441                 window->direct = false;
1442
1443                 for (i = 0; i < ARRAY_SIZE(pci->phb->mem_resources); i++) {
1444                         const unsigned long mask = IORESOURCE_MEM_64 | IORESOURCE_MEM;
1445
1446                         /* Look for MMIO32 */
1447                         if ((pci->phb->mem_resources[i].flags & mask) == IORESOURCE_MEM) {
1448                                 start = pci->phb->mem_resources[i].start;
1449                                 end = pci->phb->mem_resources[i].end;
1450                                 break;
1451                         }
1452                 }
1453
1454                 /* New table for using DDW instead of the default DMA window */
1455                 newtbl = iommu_pseries_alloc_table(pci->phb->node);
1456                 if (!newtbl) {
1457                         dev_dbg(&dev->dev, "couldn't create new IOMMU table\n");
1458                         goto out_del_list;
1459                 }
1460
1461                 iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, win_addr,
1462                                             1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops);
1463                 iommu_init_table(newtbl, pci->phb->node, start, end);
1464
1465                 pci->table_group->tables[1] = newtbl;
1466
1467                 set_iommu_table_base(&dev->dev, newtbl);
1468         }
1469
1470         if (default_win_removed) {
1471                 iommu_tce_table_put(pci->table_group->tables[0]);
1472                 pci->table_group->tables[0] = NULL;
1473
1474                 /* default_win is valid here because default_win_removed == true */
1475                 of_remove_property(pdn, default_win);
1476                 dev_info(&dev->dev, "Removed default DMA window for %pOF\n", pdn);
1477         }
1478
1479         spin_lock(&dma_win_list_lock);
1480         list_add(&window->list, &dma_win_list);
1481         spin_unlock(&dma_win_list_lock);
1482
1483         dev->dev.archdata.dma_offset = win_addr;
1484         goto out_unlock;
1485
1486 out_del_list:
1487         kfree(window);
1488
1489 out_del_prop:
1490         of_remove_property(pdn, win64);
1491
1492 out_free_prop:
1493         kfree(win64->name);
1494         kfree(win64->value);
1495         kfree(win64);
1496
1497 out_remove_win:
1498         /* DDW is clean, so it's ok to call this directly. */
1499         __remove_dma_window(pdn, ddw_avail, create.liobn);
1500
1501 out_failed:
1502         if (default_win_removed)
1503                 reset_dma_window(dev, pdn);
1504
1505         fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL);
1506         if (!fpdn)
1507                 goto out_unlock;
1508         fpdn->pdn = pdn;
1509         list_add(&fpdn->list, &failed_ddw_pdn_list);
1510
1511 out_unlock:
1512         mutex_unlock(&dma_win_init_mutex);
1513
1514         /*
1515          * If we have persistent memory and the window size is only as big
1516          * as RAM, then we failed to create a window to cover persistent
1517          * memory and need to set the DMA limit.
1518          */
1519         if (pmem_present && direct_mapping && len == max_ram_len)
1520                 dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << len);
1521
1522         return direct_mapping;
1523 }
1524
1525 static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
1526 {
1527         struct device_node *pdn, *dn;
1528         struct iommu_table *tbl;
1529         const __be32 *dma_window = NULL;
1530         struct pci_dn *pci;
1531
1532         pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1533
1534         /* dev setup for LPAR is a little tricky, since the device tree might
1535          * contain the dma-window properties per-device and not necessarily
1536          * for the bus. So we need to search upwards in the tree until we
1537          * either hit a dma-window property, OR find a parent with a table
1538          * already allocated.
1539          */
1540         dn = pci_device_to_OF_node(dev);
1541         pr_debug("  node is %pOF\n", dn);
1542
1543         pdn = pci_dma_find(dn, &dma_window);
1544         if (!pdn || !PCI_DN(pdn)) {
1545                 printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
1546                        "no DMA window found for pci dev=%s dn=%pOF\n",
1547                                  pci_name(dev), dn);
1548                 return;
1549         }
1550         pr_debug("  parent is %pOF\n", pdn);
1551
1552         pci = PCI_DN(pdn);
1553         if (!pci->table_group) {
1554                 pci->table_group = iommu_pseries_alloc_group(pci->phb->node);
1555                 tbl = pci->table_group->tables[0];
1556                 iommu_table_setparms_lpar(pci->phb, pdn, tbl,
1557                                 pci->table_group, dma_window);
1558
1559                 iommu_init_table(tbl, pci->phb->node, 0, 0);
1560                 iommu_register_group(pci->table_group,
1561                                 pci_domain_nr(pci->phb->bus), 0);
1562                 pr_debug("  created table: %p\n", pci->table_group);
1563         } else {
1564                 pr_debug("  found DMA window, table: %p\n", pci->table_group);
1565         }
1566
1567         set_iommu_table_base(&dev->dev, pci->table_group->tables[0]);
1568         iommu_add_device(pci->table_group, &dev->dev);
1569 }
1570
1571 static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask)
1572 {
1573         struct device_node *dn = pci_device_to_OF_node(pdev), *pdn;
1574
1575         /* only attempt to use a new window if 64-bit DMA is requested */
1576         if (dma_mask < DMA_BIT_MASK(64))
1577                 return false;
1578
1579         dev_dbg(&pdev->dev, "node is %pOF\n", dn);
1580
1581         /*
1582          * the device tree might contain the dma-window properties
1583          * per-device and not necessarily for the bus. So we need to
1584          * search upwards in the tree until we either hit a dma-window
1585          * property, OR find a parent with a table already allocated.
1586          */
1587         pdn = pci_dma_find(dn, NULL);
1588         if (pdn && PCI_DN(pdn))
1589                 return enable_ddw(pdev, pdn);
1590
1591         return false;
1592 }
1593
1594 static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
1595                 void *data)
1596 {
1597         struct dma_win *window;
1598         struct memory_notify *arg = data;
1599         int ret = 0;
1600
1601         switch (action) {
1602         case MEM_GOING_ONLINE:
1603                 spin_lock(&dma_win_list_lock);
1604                 list_for_each_entry(window, &dma_win_list, list) {
1605                         if (window->direct) {
1606                                 ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
1607                                                 arg->nr_pages, window->prop);
1608                         }
1609                         /* XXX log error */
1610                 }
1611                 spin_unlock(&dma_win_list_lock);
1612                 break;
1613         case MEM_CANCEL_ONLINE:
1614         case MEM_OFFLINE:
1615                 spin_lock(&dma_win_list_lock);
1616                 list_for_each_entry(window, &dma_win_list, list) {
1617                         if (window->direct) {
1618                                 ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
1619                                                 arg->nr_pages, window->prop);
1620                         }
1621                         /* XXX log error */
1622                 }
1623                 spin_unlock(&dma_win_list_lock);
1624                 break;
1625         default:
1626                 break;
1627         }
1628         if (ret && action != MEM_CANCEL_ONLINE)
1629                 return NOTIFY_BAD;
1630
1631         return NOTIFY_OK;
1632 }
1633
1634 static struct notifier_block iommu_mem_nb = {
1635         .notifier_call = iommu_mem_notifier,
1636 };
1637
1638 static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *data)
1639 {
1640         int err = NOTIFY_OK;
1641         struct of_reconfig_data *rd = data;
1642         struct device_node *np = rd->dn;
1643         struct pci_dn *pci = PCI_DN(np);
1644         struct dma_win *window;
1645
1646         switch (action) {
1647         case OF_RECONFIG_DETACH_NODE:
1648                 /*
1649                  * Removing the property will invoke the reconfig
1650                  * notifier again, which causes dead-lock on the
1651                  * read-write semaphore of the notifier chain. So
1652                  * we have to remove the property when releasing
1653                  * the device node.
1654                  */
1655                 if (remove_ddw(np, false, DIRECT64_PROPNAME))
1656                         remove_ddw(np, false, DMA64_PROPNAME);
1657
1658                 if (pci && pci->table_group)
1659                         iommu_pseries_free_group(pci->table_group,
1660                                         np->full_name);
1661
1662                 spin_lock(&dma_win_list_lock);
1663                 list_for_each_entry(window, &dma_win_list, list) {
1664                         if (window->device == np) {
1665                                 list_del(&window->list);
1666                                 kfree(window);
1667                                 break;
1668                         }
1669                 }
1670                 spin_unlock(&dma_win_list_lock);
1671                 break;
1672         default:
1673                 err = NOTIFY_DONE;
1674                 break;
1675         }
1676         return err;
1677 }
1678
1679 static struct notifier_block iommu_reconfig_nb = {
1680         .notifier_call = iommu_reconfig_notifier,
1681 };
1682
1683 /* These are called very early. */
1684 void __init iommu_init_early_pSeries(void)
1685 {
1686         if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
1687                 return;
1688
1689         if (firmware_has_feature(FW_FEATURE_LPAR)) {
1690                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
1691                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1692                 if (!disable_ddw)
1693                         pseries_pci_controller_ops.iommu_bypass_supported =
1694                                 iommu_bypass_supported_pSeriesLP;
1695         } else {
1696                 pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries;
1697                 pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries;
1698         }
1699
1700
1701         of_reconfig_notifier_register(&iommu_reconfig_nb);
1702         register_memory_notifier(&iommu_mem_nb);
1703
1704         set_pci_dma_ops(&dma_iommu_ops);
1705 }
1706
1707 static int __init disable_multitce(char *str)
1708 {
1709         if (strcmp(str, "off") == 0 &&
1710             firmware_has_feature(FW_FEATURE_LPAR) &&
1711             (firmware_has_feature(FW_FEATURE_PUT_TCE_IND) ||
1712              firmware_has_feature(FW_FEATURE_STUFF_TCE))) {
1713                 printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
1714                 powerpc_firmware_features &=
1715                         ~(FW_FEATURE_PUT_TCE_IND | FW_FEATURE_STUFF_TCE);
1716         }
1717         return 1;
1718 }
1719
1720 __setup("multitce=", disable_multitce);
1721
1722 #ifdef CONFIG_SPAPR_TCE_IOMMU
1723 struct iommu_group *pSeries_pci_device_group(struct pci_controller *hose,
1724                                              struct pci_dev *pdev)
1725 {
1726         struct device_node *pdn, *dn = pdev->dev.of_node;
1727         struct iommu_group *grp;
1728         struct pci_dn *pci;
1729
1730         pdn = pci_dma_find(dn, NULL);
1731         if (!pdn || !PCI_DN(pdn))
1732                 return ERR_PTR(-ENODEV);
1733
1734         pci = PCI_DN(pdn);
1735         if (!pci->table_group)
1736                 return ERR_PTR(-ENODEV);
1737
1738         grp = pci->table_group->group;
1739         if (!grp)
1740                 return ERR_PTR(-ENODEV);
1741
1742         return iommu_group_ref_get(grp);
1743 }
1744 #endif