drivers/acpi/numa/hmat.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019, Intel Corporation.
   4  *
   5  * Heterogeneous Memory Attributes Table (HMAT) representation
   6  *
   7  * This program parses and reports the platform's HMAT tables, and registers
   8  * the applicable attributes with the node's interfaces.
   9  */
  10
  11 #define pr_fmt(fmt) "acpi/hmat: " fmt
  12
  13 #include <linux/acpi.h>
  14 #include <linux/bitops.h>
  15 #include <linux/device.h>
  16 #include <linux/init.h>
  17 #include <linux/list.h>
  18 #include <linux/mm.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/list_sort.h>
  21 #include <linux/memregion.h>
  22 #include <linux/memory.h>
  23 #include <linux/mutex.h>
  24 #include <linux/node.h>
  25 #include <linux/sysfs.h>
  26 #include <linux/dax.h>
  27 #include <linux/memory-tiers.h>
  28
  29 static u8 hmat_revision;
  30 static int hmat_disable __initdata;
  31
  32 void __init disable_hmat(void)
  33 {
  34         hmat_disable = 1;
  35 }
  36
  37 static LIST_HEAD(targets);
  38 static LIST_HEAD(initiators);
  39 static LIST_HEAD(localities);
  40
  41 static DEFINE_MUTEX(target_lock);
  42
  43 /*
  44  * The defined enum order is used to prioritize attributes to break ties when
  45  * selecting the best performing node.
  46  */
  47 enum locality_types {
  48         WRITE_LATENCY,
  49         READ_LATENCY,
  50         WRITE_BANDWIDTH,
  51         READ_BANDWIDTH,
  52 };
  53
  54 static struct memory_locality *localities_types[4];
  55
  56 struct target_cache {
  57         struct list_head node;
  58         struct node_cache_attrs cache_attrs;
  59 };
  60
  61 struct memory_target {
  62         struct list_head node;
  63         unsigned int memory_pxm;
  64         unsigned int processor_pxm;
  65         struct resource memregions;
  66         struct node_hmem_attrs hmem_attrs[2];
  67         struct list_head caches;
  68         struct node_cache_attrs cache_attrs;
  69         bool registered;
  70 };
  71
  72 struct memory_initiator {
  73         struct list_head node;
  74         unsigned int processor_pxm;
  75         bool has_cpu;
  76 };
  77
  78 struct memory_locality {
  79         struct list_head node;
  80         struct acpi_hmat_locality *hmat_loc;
  81 };
  82
  83 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
  84 {
  85         struct memory_initiator *initiator;
  86
  87         list_for_each_entry(initiator, &initiators, node)
  88                 if (initiator->processor_pxm == cpu_pxm)
  89                         return initiator;
  90         return NULL;
  91 }
  92
  93 static struct memory_target *find_mem_target(unsigned int mem_pxm)
  94 {
  95         struct memory_target *target;
  96
  97         list_for_each_entry(target, &targets, node)
  98                 if (target->memory_pxm == mem_pxm)
  99                         return target;
 100         return NULL;
 101 }
 102
 103 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
 104 {
 105         struct memory_initiator *initiator;
 106
 107         if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
 108                 return;
 109
 110         initiator = find_mem_initiator(cpu_pxm);
 111         if (initiator)
 112                 return;
 113
 114         initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
 115         if (!initiator)
 116                 return;
 117
 118         initiator->processor_pxm = cpu_pxm;
 119         initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
 120         list_add_tail(&initiator->node, &initiators);
 121 }
 122
 123 static __init void alloc_memory_target(unsigned int mem_pxm,
 124                 resource_size_t start, resource_size_t len)
 125 {
 126         struct memory_target *target;
 127
 128         target = find_mem_target(mem_pxm);
 129         if (!target) {
 130                 target = kzalloc(sizeof(*target), GFP_KERNEL);
 131                 if (!target)
 132                         return;
 133                 target->memory_pxm = mem_pxm;
 134                 target->processor_pxm = PXM_INVAL;
 135                 target->memregions = (struct resource) {
 136                         .name   = "ACPI mem",
 137                         .start  = 0,
 138                         .end    = -1,
 139                         .flags  = IORESOURCE_MEM,
 140                 };
 141                 list_add_tail(&target->node, &targets);
 142                 INIT_LIST_HEAD(&target->caches);
 143         }
 144
 145         /*
 146          * There are potentially multiple ranges per PXM, so record each
 147          * in the per-target memregions resource tree.
 148          */
 149         if (!__request_region(&target->memregions, start, len, "memory target",
 150                                 IORESOURCE_MEM))
 151                 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
 152                                 start, start + len, mem_pxm);
 153 }
 154
 155 static __init const char *hmat_data_type(u8 type)
 156 {
 157         switch (type) {
 158         case ACPI_HMAT_ACCESS_LATENCY:
 159                 return "Access Latency";
 160         case ACPI_HMAT_READ_LATENCY:
 161                 return "Read Latency";
 162         case ACPI_HMAT_WRITE_LATENCY:
 163                 return "Write Latency";
 164         case ACPI_HMAT_ACCESS_BANDWIDTH:
 165                 return "Access Bandwidth";
 166         case ACPI_HMAT_READ_BANDWIDTH:
 167                 return "Read Bandwidth";
 168         case ACPI_HMAT_WRITE_BANDWIDTH:
 169                 return "Write Bandwidth";
 170         default:
 171                 return "Reserved";
 172         }
 173 }
 174
 175 static __init const char *hmat_data_type_suffix(u8 type)
 176 {
 177         switch (type) {
 178         case ACPI_HMAT_ACCESS_LATENCY:
 179         case ACPI_HMAT_READ_LATENCY:
 180         case ACPI_HMAT_WRITE_LATENCY:
 181                 return " nsec";
 182         case ACPI_HMAT_ACCESS_BANDWIDTH:
 183         case ACPI_HMAT_READ_BANDWIDTH:
 184         case ACPI_HMAT_WRITE_BANDWIDTH:
 185                 return " MB/s";
 186         default:
 187                 return "";
 188         }
 189 }
 190
 191 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
 192 {
 193         u32 value;
 194
 195         /*
 196          * Check for invalid and overflow values
 197          */
 198         if (entry == 0xffff || !entry)
 199                 return 0;
 200         else if (base > (UINT_MAX / (entry)))
 201                 return 0;
 202
 203         /*
 204          * Divide by the base unit for version 1, convert latency from
 205          * picosenonds to nanoseconds if revision 2.
 206          */
 207         value = entry * base;
 208         if (hmat_revision == 1) {
 209                 if (value < 10)
 210                         return 0;
 211                 value = DIV_ROUND_UP(value, 10);
 212         } else if (hmat_revision == 2) {
 213                 switch (type) {
 214                 case ACPI_HMAT_ACCESS_LATENCY:
 215                 case ACPI_HMAT_READ_LATENCY:
 216                 case ACPI_HMAT_WRITE_LATENCY:
 217                         value = DIV_ROUND_UP(value, 1000);
 218                         break;
 219                 default:
 220                         break;
 221                 }
 222         }
 223         return value;
 224 }
 225
 226 static void hmat_update_target_access(struct memory_target *target,
 227                                       u8 type, u32 value, int access)
 228 {
 229         switch (type) {
 230         case ACPI_HMAT_ACCESS_LATENCY:
 231                 target->hmem_attrs[access].read_latency = value;
 232                 target->hmem_attrs[access].write_latency = value;
 233                 break;
 234         case ACPI_HMAT_READ_LATENCY:
 235                 target->hmem_attrs[access].read_latency = value;
 236                 break;
 237         case ACPI_HMAT_WRITE_LATENCY:
 238                 target->hmem_attrs[access].write_latency = value;
 239                 break;
 240         case ACPI_HMAT_ACCESS_BANDWIDTH:
 241                 target->hmem_attrs[access].read_bandwidth = value;
 242                 target->hmem_attrs[access].write_bandwidth = value;
 243                 break;
 244         case ACPI_HMAT_READ_BANDWIDTH:
 245                 target->hmem_attrs[access].read_bandwidth = value;
 246                 break;
 247         case ACPI_HMAT_WRITE_BANDWIDTH:
 248                 target->hmem_attrs[access].write_bandwidth = value;
 249                 break;
 250         default:
 251                 break;
 252         }
 253 }
 254
 255 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
 256 {
 257         struct memory_locality *loc;
 258
 259         loc = kzalloc(sizeof(*loc), GFP_KERNEL);
 260         if (!loc) {
 261                 pr_notice_once("Failed to allocate HMAT locality\n");
 262                 return;
 263         }
 264
 265         loc->hmat_loc = hmat_loc;
 266         list_add_tail(&loc->node, &localities);
 267
 268         switch (hmat_loc->data_type) {
 269         case ACPI_HMAT_ACCESS_LATENCY:
 270                 localities_types[READ_LATENCY] = loc;
 271                 localities_types[WRITE_LATENCY] = loc;
 272                 break;
 273         case ACPI_HMAT_READ_LATENCY:
 274                 localities_types[READ_LATENCY] = loc;
 275                 break;
 276         case ACPI_HMAT_WRITE_LATENCY:
 277                 localities_types[WRITE_LATENCY] = loc;
 278                 break;
 279         case ACPI_HMAT_ACCESS_BANDWIDTH:
 280                 localities_types[READ_BANDWIDTH] = loc;
 281                 localities_types[WRITE_BANDWIDTH] = loc;
 282                 break;
 283         case ACPI_HMAT_READ_BANDWIDTH:
 284                 localities_types[READ_BANDWIDTH] = loc;
 285                 break;
 286         case ACPI_HMAT_WRITE_BANDWIDTH:
 287                 localities_types[WRITE_BANDWIDTH] = loc;
 288                 break;
 289         default:
 290                 break;
 291         }
 292 }
 293
 294 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
 295                                       const unsigned long end)
 296 {
 297         struct acpi_hmat_locality *hmat_loc = (void *)header;
 298         struct memory_target *target;
 299         unsigned int init, targ, total_size, ipds, tpds;
 300         u32 *inits, *targs, value;
 301         u16 *entries;
 302         u8 type, mem_hier;
 303
 304         if (hmat_loc->header.length < sizeof(*hmat_loc)) {
 305                 pr_notice("Unexpected locality header length: %u\n",
 306                          hmat_loc->header.length);
 307                 return -EINVAL;
 308         }
 309
 310         type = hmat_loc->data_type;
 311         mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
 312         ipds = hmat_loc->number_of_initiator_Pds;
 313         tpds = hmat_loc->number_of_target_Pds;
 314         total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
 315                      sizeof(*inits) * ipds + sizeof(*targs) * tpds;
 316         if (hmat_loc->header.length < total_size) {
 317                 pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
 318                          hmat_loc->header.length, total_size);
 319                 return -EINVAL;
 320         }
 321
 322         pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
 323                 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
 324                 hmat_loc->entry_base_unit);
 325
 326         inits = (u32 *)(hmat_loc + 1);
 327         targs = inits + ipds;
 328         entries = (u16 *)(targs + tpds);
 329         for (init = 0; init < ipds; init++) {
 330                 alloc_memory_initiator(inits[init]);
 331                 for (targ = 0; targ < tpds; targ++) {
 332                         value = hmat_normalize(entries[init * tpds + targ],
 333                                                hmat_loc->entry_base_unit,
 334                                                type);
 335                         pr_info("  Initiator-Target[%u-%u]:%u%s\n",
 336                                 inits[init], targs[targ], value,
 337                                 hmat_data_type_suffix(type));
 338
 339                         if (mem_hier == ACPI_HMAT_MEMORY) {
 340                                 target = find_mem_target(targs[targ]);
 341                                 if (target && target->processor_pxm == inits[init]) {
 342                                         hmat_update_target_access(target, type, value, 0);
 343                                         /* If the node has a CPU, update access 1 */
 344                                         if (node_state(pxm_to_node(inits[init]), N_CPU))
 345                                                 hmat_update_target_access(target, type, value, 1);
 346                                 }
 347                         }
 348                 }
 349         }
 350
 351         if (mem_hier == ACPI_HMAT_MEMORY)
 352                 hmat_add_locality(hmat_loc);
 353
 354         return 0;
 355 }
 356
 357 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 358                                    const unsigned long end)
 359 {
 360         struct acpi_hmat_cache *cache = (void *)header;
 361         struct memory_target *target;
 362         struct target_cache *tcache;
 363         u32 attrs;
 364
 365         if (cache->header.length < sizeof(*cache)) {
 366                 pr_notice("Unexpected cache header length: %u\n",
 367                          cache->header.length);
 368                 return -EINVAL;
 369         }
 370
 371         attrs = cache->cache_attributes;
 372         pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
 373                 cache->memory_PD, cache->cache_size, attrs,
 374                 cache->number_of_SMBIOShandles);
 375
 376         target = find_mem_target(cache->memory_PD);
 377         if (!target)
 378                 return 0;
 379
 380         tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
 381         if (!tcache) {
 382                 pr_notice_once("Failed to allocate HMAT cache info\n");
 383                 return 0;
 384         }
 385
 386         tcache->cache_attrs.size = cache->cache_size;
 387         tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
 388         tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
 389
 390         switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
 391         case ACPI_HMAT_CA_DIRECT_MAPPED:
 392                 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
 393                 break;
 394         case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
 395                 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
 396                 break;
 397         case ACPI_HMAT_CA_NONE:
 398         default:
 399                 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
 400                 break;
 401         }
 402
 403         switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
 404         case ACPI_HMAT_CP_WB:
 405                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
 406                 break;
 407         case ACPI_HMAT_CP_WT:
 408                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
 409                 break;
 410         case ACPI_HMAT_CP_NONE:
 411         default:
 412                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
 413                 break;
 414         }
 415         list_add_tail(&tcache->node, &target->caches);
 416
 417         return 0;
 418 }
 419
 420 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
 421                                               const unsigned long end)
 422 {
 423         struct acpi_hmat_proximity_domain *p = (void *)header;
 424         struct memory_target *target = NULL;
 425
 426         if (p->header.length != sizeof(*p)) {
 427                 pr_notice("Unexpected address range header length: %u\n",
 428                          p->header.length);
 429                 return -EINVAL;
 430         }
 431
 432         if (hmat_revision == 1)
 433                 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 434                         p->reserved3, p->reserved4, p->flags, p->processor_PD,
 435                         p->memory_PD);
 436         else
 437                 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 438                         p->flags, p->processor_PD, p->memory_PD);
 439
 440         if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
 441             hmat_revision > 1) {
 442                 target = find_mem_target(p->memory_PD);
 443                 if (!target) {
 444                         pr_debug("Memory Domain missing from SRAT\n");
 445                         return -EINVAL;
 446                 }
 447         }
 448         if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
 449                 int p_node = pxm_to_node(p->processor_PD);
 450
 451                 if (p_node == NUMA_NO_NODE) {
 452                         pr_debug("Invalid Processor Domain\n");
 453                         return -EINVAL;
 454                 }
 455                 target->processor_pxm = p->processor_PD;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
 462                                       const unsigned long end)
 463 {
 464         struct acpi_hmat_structure *hdr = (void *)header;
 465
 466         if (!hdr)
 467                 return -EINVAL;
 468
 469         switch (hdr->type) {
 470         case ACPI_HMAT_TYPE_PROXIMITY:
 471                 return hmat_parse_proximity_domain(header, end);
 472         case ACPI_HMAT_TYPE_LOCALITY:
 473                 return hmat_parse_locality(header, end);
 474         case ACPI_HMAT_TYPE_CACHE:
 475                 return hmat_parse_cache(header, end);
 476         default:
 477                 return -EINVAL;
 478         }
 479 }
 480
 481 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
 482                                           const unsigned long end)
 483 {
 484         struct acpi_srat_mem_affinity *ma = (void *)header;
 485
 486         if (!ma)
 487                 return -EINVAL;
 488         if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
 489                 return 0;
 490         alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
 491         return 0;
 492 }
 493
 494 static u32 hmat_initiator_perf(struct memory_target *target,
 495                                struct memory_initiator *initiator,
 496                                struct acpi_hmat_locality *hmat_loc)
 497 {
 498         unsigned int ipds, tpds, i, idx = 0, tdx = 0;
 499         u32 *inits, *targs;
 500         u16 *entries;
 501
 502         ipds = hmat_loc->number_of_initiator_Pds;
 503         tpds = hmat_loc->number_of_target_Pds;
 504         inits = (u32 *)(hmat_loc + 1);
 505         targs = inits + ipds;
 506         entries = (u16 *)(targs + tpds);
 507
 508         for (i = 0; i < ipds; i++) {
 509                 if (inits[i] == initiator->processor_pxm) {
 510                         idx = i;
 511                         break;
 512                 }
 513         }
 514
 515         if (i == ipds)
 516                 return 0;
 517
 518         for (i = 0; i < tpds; i++) {
 519                 if (targs[i] == target->memory_pxm) {
 520                         tdx = i;
 521                         break;
 522                 }
 523         }
 524         if (i == tpds)
 525                 return 0;
 526
 527         return hmat_normalize(entries[idx * tpds + tdx],
 528                               hmat_loc->entry_base_unit,
 529                               hmat_loc->data_type);
 530 }
 531
 532 static bool hmat_update_best(u8 type, u32 value, u32 *best)
 533 {
 534         bool updated = false;
 535
 536         if (!value)
 537                 return false;
 538
 539         switch (type) {
 540         case ACPI_HMAT_ACCESS_LATENCY:
 541         case ACPI_HMAT_READ_LATENCY:
 542         case ACPI_HMAT_WRITE_LATENCY:
 543                 if (!*best || *best > value) {
 544                         *best = value;
 545                         updated = true;
 546                 }
 547                 break;
 548         case ACPI_HMAT_ACCESS_BANDWIDTH:
 549         case ACPI_HMAT_READ_BANDWIDTH:
 550         case ACPI_HMAT_WRITE_BANDWIDTH:
 551                 if (!*best || *best < value) {
 552                         *best = value;
 553                         updated = true;
 554                 }
 555                 break;
 556         }
 557
 558         return updated;
 559 }
 560
 561 static int initiator_cmp(void *priv, const struct list_head *a,
 562                          const struct list_head *b)
 563 {
 564         struct memory_initiator *ia;
 565         struct memory_initiator *ib;
 566
 567         ia = list_entry(a, struct memory_initiator, node);
 568         ib = list_entry(b, struct memory_initiator, node);
 569
 570         return ia->processor_pxm - ib->processor_pxm;
 571 }
 572
 573 static int initiators_to_nodemask(unsigned long *p_nodes)
 574 {
 575         struct memory_initiator *initiator;
 576
 577         if (list_empty(&initiators))
 578                 return -ENXIO;
 579
 580         list_for_each_entry(initiator, &initiators, node)
 581                 set_bit(initiator->processor_pxm, p_nodes);
 582
 583         return 0;
 584 }
 585
 586 static void hmat_update_target_attrs(struct memory_target *target,
 587                                      unsigned long *p_nodes, int access)
 588 {
 589         struct memory_initiator *initiator;
 590         unsigned int cpu_nid;
 591         struct memory_locality *loc = NULL;
 592         u32 best = 0;
 593         int i;
 594
 595         bitmap_zero(p_nodes, MAX_NUMNODES);
 596         /*
 597          * If the Address Range Structure provides a local processor pxm, set
 598          * only that one. Otherwise, find the best performance attributes and
 599          * collect all initiators that match.
 600          */
 601         if (target->processor_pxm != PXM_INVAL) {
 602                 cpu_nid = pxm_to_node(target->processor_pxm);
 603                 if (access == 0 || node_state(cpu_nid, N_CPU)) {
 604                         set_bit(target->processor_pxm, p_nodes);
 605                         return;
 606                 }
 607         }
 608
 609         if (list_empty(&localities))
 610                 return;
 611
 612         /*
 613          * We need the initiator list sorted so we can use bitmap_clear for
 614          * previously set initiators when we find a better memory accessor.
 615          * We'll also use the sorting to prime the candidate nodes with known
 616          * initiators.
 617          */
 618         list_sort(NULL, &initiators, initiator_cmp);
 619         if (initiators_to_nodemask(p_nodes) < 0)
 620                 return;
 621
 622         for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
 623                 loc = localities_types[i];
 624                 if (!loc)
 625                         continue;
 626
 627                 best = 0;
 628                 list_for_each_entry(initiator, &initiators, node) {
 629                         u32 value;
 630
 631                         if (access == 1 && !initiator->has_cpu) {
 632                                 clear_bit(initiator->processor_pxm, p_nodes);
 633                                 continue;
 634                         }
 635                         if (!test_bit(initiator->processor_pxm, p_nodes))
 636                                 continue;
 637
 638                         value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
 639                         if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
 640                                 bitmap_clear(p_nodes, 0, initiator->processor_pxm);
 641                         if (value != best)
 642                                 clear_bit(initiator->processor_pxm, p_nodes);
 643                 }
 644                 if (best)
 645                         hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);
 646         }
 647 }
 648
 649 static void __hmat_register_target_initiators(struct memory_target *target,
 650                                               unsigned long *p_nodes,
 651                                               int access)
 652 {
 653         unsigned int mem_nid, cpu_nid;
 654         int i;
 655
 656         mem_nid = pxm_to_node(target->memory_pxm);
 657         hmat_update_target_attrs(target, p_nodes, access);
 658         for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
 659                 cpu_nid = pxm_to_node(i);
 660                 register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
 661         }
 662 }
 663
 664 static void hmat_register_target_initiators(struct memory_target *target)
 665 {
 666         static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
 667
 668         __hmat_register_target_initiators(target, p_nodes, 0);
 669         __hmat_register_target_initiators(target, p_nodes, 1);
 670 }
 671
 672 static void hmat_register_target_cache(struct memory_target *target)
 673 {
 674         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 675         struct target_cache *tcache;
 676
 677         list_for_each_entry(tcache, &target->caches, node)
 678                 node_add_cache(mem_nid, &tcache->cache_attrs);
 679 }
 680
 681 static void hmat_register_target_perf(struct memory_target *target, int access)
 682 {
 683         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 684         node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access);
 685 }
 686
 687 static void hmat_register_target_devices(struct memory_target *target)
 688 {
 689         struct resource *res;
 690
 691         /*
 692          * Do not bother creating devices if no driver is available to
 693          * consume them.
 694          */
 695         if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
 696                 return;
 697
 698         for (res = target->memregions.child; res; res = res->sibling) {
 699                 int target_nid = pxm_to_node(target->memory_pxm);
 700
 701                 hmem_register_resource(target_nid, res);
 702         }
 703 }
 704
 705 static void hmat_register_target(struct memory_target *target)
 706 {
 707         int nid = pxm_to_node(target->memory_pxm);
 708
 709         /*
 710          * Devices may belong to either an offline or online
 711          * node, so unconditionally add them.
 712          */
 713         hmat_register_target_devices(target);
 714
 715         /*
 716          * Skip offline nodes. This can happen when memory
 717          * marked EFI_MEMORY_SP, "specific purpose", is applied
 718          * to all the memory in a proximity domain leading to
 719          * the node being marked offline / unplugged, or if
 720          * memory-only "hotplug" node is offline.
 721          */
 722         if (nid == NUMA_NO_NODE || !node_online(nid))
 723                 return;
 724
 725         mutex_lock(&target_lock);
 726         if (!target->registered) {
 727                 hmat_register_target_initiators(target);
 728                 hmat_register_target_cache(target);
 729                 hmat_register_target_perf(target, 0);
 730                 hmat_register_target_perf(target, 1);
 731                 target->registered = true;
 732         }
 733         mutex_unlock(&target_lock);
 734 }
 735
 736 static void hmat_register_targets(void)
 737 {
 738         struct memory_target *target;
 739
 740         list_for_each_entry(target, &targets, node)
 741                 hmat_register_target(target);
 742 }
 743
 744 static int hmat_callback(struct notifier_block *self,
 745                          unsigned long action, void *arg)
 746 {
 747         struct memory_target *target;
 748         struct memory_notify *mnb = arg;
 749         int pxm, nid = mnb->status_change_nid;
 750
 751         if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
 752                 return NOTIFY_OK;
 753
 754         pxm = node_to_pxm(nid);
 755         target = find_mem_target(pxm);
 756         if (!target)
 757                 return NOTIFY_OK;
 758
 759         hmat_register_target(target);
 760         return NOTIFY_OK;
 761 }
 762
 763 static int hmat_set_default_dram_perf(void)
 764 {
 765         int rc;
 766         int nid, pxm;
 767         struct memory_target *target;
 768         struct node_hmem_attrs *attrs;
 769
 770         if (!default_dram_type)
 771                 return -EIO;
 772
 773         for_each_node_mask(nid, default_dram_type->nodes) {
 774                 pxm = node_to_pxm(nid);
 775                 target = find_mem_target(pxm);
 776                 if (!target)
 777                         continue;
 778                 attrs = &target->hmem_attrs[1];
 779                 rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");
 780                 if (rc)
 781                         return rc;
 782         }
 783
 784         return 0;
 785 }
 786
 787 static int hmat_calculate_adistance(struct notifier_block *self,
 788                                     unsigned long nid, void *data)
 789 {
 790         static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
 791         struct memory_target *target;
 792         struct node_hmem_attrs *perf;
 793         int *adist = data;
 794         int pxm;
 795
 796         pxm = node_to_pxm(nid);
 797         target = find_mem_target(pxm);
 798         if (!target)
 799                 return NOTIFY_OK;
 800
 801         mutex_lock(&target_lock);
 802         hmat_update_target_attrs(target, p_nodes, 1);
 803         mutex_unlock(&target_lock);
 804
 805         perf = &target->hmem_attrs[1];
 806
 807         if (mt_perf_to_adistance(perf, adist))
 808                 return NOTIFY_OK;
 809
 810         return NOTIFY_STOP;
 811 }
 812
 813 static struct notifier_block hmat_adist_nb __meminitdata = {
 814         .notifier_call = hmat_calculate_adistance,
 815         .priority = 100,
 816 };
 817
 818 static __init void hmat_free_structures(void)
 819 {
 820         struct memory_target *target, *tnext;
 821         struct memory_locality *loc, *lnext;
 822         struct memory_initiator *initiator, *inext;
 823         struct target_cache *tcache, *cnext;
 824
 825         list_for_each_entry_safe(target, tnext, &targets, node) {
 826                 struct resource *res, *res_next;
 827
 828                 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
 829                         list_del(&tcache->node);
 830                         kfree(tcache);
 831                 }
 832
 833                 list_del(&target->node);
 834                 res = target->memregions.child;
 835                 while (res) {
 836                         res_next = res->sibling;
 837                         __release_region(&target->memregions, res->start,
 838                                         resource_size(res));
 839                         res = res_next;
 840                 }
 841                 kfree(target);
 842         }
 843
 844         list_for_each_entry_safe(initiator, inext, &initiators, node) {
 845                 list_del(&initiator->node);
 846                 kfree(initiator);
 847         }
 848
 849         list_for_each_entry_safe(loc, lnext, &localities, node) {
 850                 list_del(&loc->node);
 851                 kfree(loc);
 852         }
 853 }
 854
 855 static __init int hmat_init(void)
 856 {
 857         struct acpi_table_header *tbl;
 858         enum acpi_hmat_type i;
 859         acpi_status status;
 860
 861         if (srat_disabled() || hmat_disable)
 862                 return 0;
 863
 864         status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
 865         if (ACPI_FAILURE(status))
 866                 return 0;
 867
 868         if (acpi_table_parse_entries(ACPI_SIG_SRAT,
 869                                 sizeof(struct acpi_table_srat),
 870                                 ACPI_SRAT_TYPE_MEMORY_AFFINITY,
 871                                 srat_parse_mem_affinity, 0) < 0)
 872                 goto out_put;
 873         acpi_put_table(tbl);
 874
 875         status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
 876         if (ACPI_FAILURE(status))
 877                 goto out_put;
 878
 879         hmat_revision = tbl->revision;
 880         switch (hmat_revision) {
 881         case 1:
 882         case 2:
 883                 break;
 884         default:
 885                 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
 886                 goto out_put;
 887         }
 888
 889         for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
 890                 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
 891                                              sizeof(struct acpi_table_hmat), i,
 892                                              hmat_parse_subtable, 0) < 0) {
 893                         pr_notice("Ignoring: Invalid table");
 894                         goto out_put;
 895                 }
 896         }
 897         hmat_register_targets();
 898
 899         /* Keep the table and structures if the notifier may use them */
 900         if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
 901                 goto out_put;
 902
 903         if (!hmat_set_default_dram_perf())
 904                 register_mt_adistance_algorithm(&hmat_adist_nb);
 905
 906         return 0;
 907 out_put:
 908         hmat_free_structures();
 909         acpi_put_table(tbl);
 910         return 0;
 911 }
 912 subsys_initcall(hmat_init);