drivers/acpi/numa/hmat.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2019, Intel Corporation.
   4  *
   5  * Heterogeneous Memory Attributes Table (HMAT) representation
   6  *
   7  * This program parses and reports the platform's HMAT tables, and registers
   8  * the applicable attributes with the node's interfaces.
   9  */
  10
  11 #define pr_fmt(fmt) "acpi/hmat: " fmt
  12 #define dev_fmt(fmt) "acpi/hmat: " fmt
  13
  14 #include <linux/acpi.h>
  15 #include <linux/bitops.h>
  16 #include <linux/device.h>
  17 #include <linux/init.h>
  18 #include <linux/list.h>
  19 #include <linux/mm.h>
  20 #include <linux/platform_device.h>
  21 #include <linux/list_sort.h>
  22 #include <linux/memregion.h>
  23 #include <linux/memory.h>
  24 #include <linux/mutex.h>
  25 #include <linux/node.h>
  26 #include <linux/sysfs.h>
  27 #include <linux/dax.h>
  28
  29 static u8 hmat_revision;
  30 static int hmat_disable __initdata;
  31
  32 void __init disable_hmat(void)
  33 {
  34         hmat_disable = 1;
  35 }
  36
  37 static LIST_HEAD(targets);
  38 static LIST_HEAD(initiators);
  39 static LIST_HEAD(localities);
  40
  41 static DEFINE_MUTEX(target_lock);
  42
  43 /*
  44  * The defined enum order is used to prioritize attributes to break ties when
  45  * selecting the best performing node.
  46  */
  47 enum locality_types {
  48         WRITE_LATENCY,
  49         READ_LATENCY,
  50         WRITE_BANDWIDTH,
  51         READ_BANDWIDTH,
  52 };
  53
  54 static struct memory_locality *localities_types[4];
  55
  56 struct target_cache {
  57         struct list_head node;
  58         struct node_cache_attrs cache_attrs;
  59 };
  60
  61 struct memory_target {
  62         struct list_head node;
  63         unsigned int memory_pxm;
  64         unsigned int processor_pxm;
  65         struct resource memregions;
  66         struct node_hmem_attrs hmem_attrs[2];
  67         struct list_head caches;
  68         struct node_cache_attrs cache_attrs;
  69         bool registered;
  70 };
  71
  72 struct memory_initiator {
  73         struct list_head node;
  74         unsigned int processor_pxm;
  75         bool has_cpu;
  76 };
  77
  78 struct memory_locality {
  79         struct list_head node;
  80         struct acpi_hmat_locality *hmat_loc;
  81 };
  82
  83 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
  84 {
  85         struct memory_initiator *initiator;
  86
  87         list_for_each_entry(initiator, &initiators, node)
  88                 if (initiator->processor_pxm == cpu_pxm)
  89                         return initiator;
  90         return NULL;
  91 }
  92
  93 static struct memory_target *find_mem_target(unsigned int mem_pxm)
  94 {
  95         struct memory_target *target;
  96
  97         list_for_each_entry(target, &targets, node)
  98                 if (target->memory_pxm == mem_pxm)
  99                         return target;
 100         return NULL;
 101 }
 102
 103 static __init void alloc_memory_initiator(unsigned int cpu_pxm)
 104 {
 105         struct memory_initiator *initiator;
 106
 107         if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
 108                 return;
 109
 110         initiator = find_mem_initiator(cpu_pxm);
 111         if (initiator)
 112                 return;
 113
 114         initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
 115         if (!initiator)
 116                 return;
 117
 118         initiator->processor_pxm = cpu_pxm;
 119         initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
 120         list_add_tail(&initiator->node, &initiators);
 121 }
 122
 123 static __init void alloc_memory_target(unsigned int mem_pxm,
 124                 resource_size_t start, resource_size_t len)
 125 {
 126         struct memory_target *target;
 127
 128         target = find_mem_target(mem_pxm);
 129         if (!target) {
 130                 target = kzalloc(sizeof(*target), GFP_KERNEL);
 131                 if (!target)
 132                         return;
 133                 target->memory_pxm = mem_pxm;
 134                 target->processor_pxm = PXM_INVAL;
 135                 target->memregions = (struct resource) {
 136                         .name   = "ACPI mem",
 137                         .start  = 0,
 138                         .end    = -1,
 139                         .flags  = IORESOURCE_MEM,
 140                 };
 141                 list_add_tail(&target->node, &targets);
 142                 INIT_LIST_HEAD(&target->caches);
 143         }
 144
 145         /*
 146          * There are potentially multiple ranges per PXM, so record each
 147          * in the per-target memregions resource tree.
 148          */
 149         if (!__request_region(&target->memregions, start, len, "memory target",
 150                                 IORESOURCE_MEM))
 151                 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
 152                                 start, start + len, mem_pxm);
 153 }
 154
 155 static __init const char *hmat_data_type(u8 type)
 156 {
 157         switch (type) {
 158         case ACPI_HMAT_ACCESS_LATENCY:
 159                 return "Access Latency";
 160         case ACPI_HMAT_READ_LATENCY:
 161                 return "Read Latency";
 162         case ACPI_HMAT_WRITE_LATENCY:
 163                 return "Write Latency";
 164         case ACPI_HMAT_ACCESS_BANDWIDTH:
 165                 return "Access Bandwidth";
 166         case ACPI_HMAT_READ_BANDWIDTH:
 167                 return "Read Bandwidth";
 168         case ACPI_HMAT_WRITE_BANDWIDTH:
 169                 return "Write Bandwidth";
 170         default:
 171                 return "Reserved";
 172         }
 173 }
 174
 175 static __init const char *hmat_data_type_suffix(u8 type)
 176 {
 177         switch (type) {
 178         case ACPI_HMAT_ACCESS_LATENCY:
 179         case ACPI_HMAT_READ_LATENCY:
 180         case ACPI_HMAT_WRITE_LATENCY:
 181                 return " nsec";
 182         case ACPI_HMAT_ACCESS_BANDWIDTH:
 183         case ACPI_HMAT_READ_BANDWIDTH:
 184         case ACPI_HMAT_WRITE_BANDWIDTH:
 185                 return " MB/s";
 186         default:
 187                 return "";
 188         }
 189 }
 190
 191 static u32 hmat_normalize(u16 entry, u64 base, u8 type)
 192 {
 193         u32 value;
 194
 195         /*
 196          * Check for invalid and overflow values
 197          */
 198         if (entry == 0xffff || !entry)
 199                 return 0;
 200         else if (base > (UINT_MAX / (entry)))
 201                 return 0;
 202
 203         /*
 204          * Divide by the base unit for version 1, convert latency from
 205          * picosenonds to nanoseconds if revision 2.
 206          */
 207         value = entry * base;
 208         if (hmat_revision == 1) {
 209                 if (value < 10)
 210                         return 0;
 211                 value = DIV_ROUND_UP(value, 10);
 212         } else if (hmat_revision == 2) {
 213                 switch (type) {
 214                 case ACPI_HMAT_ACCESS_LATENCY:
 215                 case ACPI_HMAT_READ_LATENCY:
 216                 case ACPI_HMAT_WRITE_LATENCY:
 217                         value = DIV_ROUND_UP(value, 1000);
 218                         break;
 219                 default:
 220                         break;
 221                 }
 222         }
 223         return value;
 224 }
 225
 226 static void hmat_update_target_access(struct memory_target *target,
 227                                       u8 type, u32 value, int access)
 228 {
 229         switch (type) {
 230         case ACPI_HMAT_ACCESS_LATENCY:
 231                 target->hmem_attrs[access].read_latency = value;
 232                 target->hmem_attrs[access].write_latency = value;
 233                 break;
 234         case ACPI_HMAT_READ_LATENCY:
 235                 target->hmem_attrs[access].read_latency = value;
 236                 break;
 237         case ACPI_HMAT_WRITE_LATENCY:
 238                 target->hmem_attrs[access].write_latency = value;
 239                 break;
 240         case ACPI_HMAT_ACCESS_BANDWIDTH:
 241                 target->hmem_attrs[access].read_bandwidth = value;
 242                 target->hmem_attrs[access].write_bandwidth = value;
 243                 break;
 244         case ACPI_HMAT_READ_BANDWIDTH:
 245                 target->hmem_attrs[access].read_bandwidth = value;
 246                 break;
 247         case ACPI_HMAT_WRITE_BANDWIDTH:
 248                 target->hmem_attrs[access].write_bandwidth = value;
 249                 break;
 250         default:
 251                 break;
 252         }
 253 }
 254
 255 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
 256 {
 257         struct memory_locality *loc;
 258
 259         loc = kzalloc(sizeof(*loc), GFP_KERNEL);
 260         if (!loc) {
 261                 pr_notice_once("Failed to allocate HMAT locality\n");
 262                 return;
 263         }
 264
 265         loc->hmat_loc = hmat_loc;
 266         list_add_tail(&loc->node, &localities);
 267
 268         switch (hmat_loc->data_type) {
 269         case ACPI_HMAT_ACCESS_LATENCY:
 270                 localities_types[READ_LATENCY] = loc;
 271                 localities_types[WRITE_LATENCY] = loc;
 272                 break;
 273         case ACPI_HMAT_READ_LATENCY:
 274                 localities_types[READ_LATENCY] = loc;
 275                 break;
 276         case ACPI_HMAT_WRITE_LATENCY:
 277                 localities_types[WRITE_LATENCY] = loc;
 278                 break;
 279         case ACPI_HMAT_ACCESS_BANDWIDTH:
 280                 localities_types[READ_BANDWIDTH] = loc;
 281                 localities_types[WRITE_BANDWIDTH] = loc;
 282                 break;
 283         case ACPI_HMAT_READ_BANDWIDTH:
 284                 localities_types[READ_BANDWIDTH] = loc;
 285                 break;
 286         case ACPI_HMAT_WRITE_BANDWIDTH:
 287                 localities_types[WRITE_BANDWIDTH] = loc;
 288                 break;
 289         default:
 290                 break;
 291         }
 292 }
 293
 294 static __init int hmat_parse_locality(union acpi_subtable_headers *header,
 295                                       const unsigned long end)
 296 {
 297         struct acpi_hmat_locality *hmat_loc = (void *)header;
 298         struct memory_target *target;
 299         unsigned int init, targ, total_size, ipds, tpds;
 300         u32 *inits, *targs, value;
 301         u16 *entries;
 302         u8 type, mem_hier;
 303
 304         if (hmat_loc->header.length < sizeof(*hmat_loc)) {
 305                 pr_notice("HMAT: Unexpected locality header length: %u\n",
 306                          hmat_loc->header.length);
 307                 return -EINVAL;
 308         }
 309
 310         type = hmat_loc->data_type;
 311         mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
 312         ipds = hmat_loc->number_of_initiator_Pds;
 313         tpds = hmat_loc->number_of_target_Pds;
 314         total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
 315                      sizeof(*inits) * ipds + sizeof(*targs) * tpds;
 316         if (hmat_loc->header.length < total_size) {
 317                 pr_notice("HMAT: Unexpected locality header length:%u, minimum required:%u\n",
 318                          hmat_loc->header.length, total_size);
 319                 return -EINVAL;
 320         }
 321
 322         pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
 323                 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
 324                 hmat_loc->entry_base_unit);
 325
 326         inits = (u32 *)(hmat_loc + 1);
 327         targs = inits + ipds;
 328         entries = (u16 *)(targs + tpds);
 329         for (init = 0; init < ipds; init++) {
 330                 alloc_memory_initiator(inits[init]);
 331                 for (targ = 0; targ < tpds; targ++) {
 332                         value = hmat_normalize(entries[init * tpds + targ],
 333                                                hmat_loc->entry_base_unit,
 334                                                type);
 335                         pr_info("  Initiator-Target[%u-%u]:%u%s\n",
 336                                 inits[init], targs[targ], value,
 337                                 hmat_data_type_suffix(type));
 338
 339                         if (mem_hier == ACPI_HMAT_MEMORY) {
 340                                 target = find_mem_target(targs[targ]);
 341                                 if (target && target->processor_pxm == inits[init]) {
 342                                         hmat_update_target_access(target, type, value, 0);
 343                                         /* If the node has a CPU, update access 1 */
 344                                         if (node_state(pxm_to_node(inits[init]), N_CPU))
 345                                                 hmat_update_target_access(target, type, value, 1);
 346                                 }
 347                         }
 348                 }
 349         }
 350
 351         if (mem_hier == ACPI_HMAT_MEMORY)
 352                 hmat_add_locality(hmat_loc);
 353
 354         return 0;
 355 }
 356
 357 static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 358                                    const unsigned long end)
 359 {
 360         struct acpi_hmat_cache *cache = (void *)header;
 361         struct memory_target *target;
 362         struct target_cache *tcache;
 363         u32 attrs;
 364
 365         if (cache->header.length < sizeof(*cache)) {
 366                 pr_notice("HMAT: Unexpected cache header length: %u\n",
 367                          cache->header.length);
 368                 return -EINVAL;
 369         }
 370
 371         attrs = cache->cache_attributes;
 372         pr_info("HMAT: Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
 373                 cache->memory_PD, cache->cache_size, attrs,
 374                 cache->number_of_SMBIOShandles);
 375
 376         target = find_mem_target(cache->memory_PD);
 377         if (!target)
 378                 return 0;
 379
 380         tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
 381         if (!tcache) {
 382                 pr_notice_once("Failed to allocate HMAT cache info\n");
 383                 return 0;
 384         }
 385
 386         tcache->cache_attrs.size = cache->cache_size;
 387         tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
 388         tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
 389
 390         switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
 391         case ACPI_HMAT_CA_DIRECT_MAPPED:
 392                 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
 393                 break;
 394         case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
 395                 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
 396                 break;
 397         case ACPI_HMAT_CA_NONE:
 398         default:
 399                 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
 400                 break;
 401         }
 402
 403         switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
 404         case ACPI_HMAT_CP_WB:
 405                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
 406                 break;
 407         case ACPI_HMAT_CP_WT:
 408                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
 409                 break;
 410         case ACPI_HMAT_CP_NONE:
 411         default:
 412                 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
 413                 break;
 414         }
 415         list_add_tail(&tcache->node, &target->caches);
 416
 417         return 0;
 418 }
 419
 420 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
 421                                               const unsigned long end)
 422 {
 423         struct acpi_hmat_proximity_domain *p = (void *)header;
 424         struct memory_target *target = NULL;
 425
 426         if (p->header.length != sizeof(*p)) {
 427                 pr_notice("HMAT: Unexpected address range header length: %u\n",
 428                          p->header.length);
 429                 return -EINVAL;
 430         }
 431
 432         if (hmat_revision == 1)
 433                 pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 434                         p->reserved3, p->reserved4, p->flags, p->processor_PD,
 435                         p->memory_PD);
 436         else
 437                 pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 438                         p->flags, p->processor_PD, p->memory_PD);
 439
 440         if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
 441             hmat_revision > 1) {
 442                 target = find_mem_target(p->memory_PD);
 443                 if (!target) {
 444                         pr_debug("HMAT: Memory Domain missing from SRAT\n");
 445                         return -EINVAL;
 446                 }
 447         }
 448         if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
 449                 int p_node = pxm_to_node(p->processor_PD);
 450
 451                 if (p_node == NUMA_NO_NODE) {
 452                         pr_debug("HMAT: Invalid Processor Domain\n");
 453                         return -EINVAL;
 454                 }
 455                 target->processor_pxm = p->processor_PD;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
 462                                       const unsigned long end)
 463 {
 464         struct acpi_hmat_structure *hdr = (void *)header;
 465
 466         if (!hdr)
 467                 return -EINVAL;
 468
 469         switch (hdr->type) {
 470         case ACPI_HMAT_TYPE_PROXIMITY:
 471                 return hmat_parse_proximity_domain(header, end);
 472         case ACPI_HMAT_TYPE_LOCALITY:
 473                 return hmat_parse_locality(header, end);
 474         case ACPI_HMAT_TYPE_CACHE:
 475                 return hmat_parse_cache(header, end);
 476         default:
 477                 return -EINVAL;
 478         }
 479 }
 480
 481 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
 482                                           const unsigned long end)
 483 {
 484         struct acpi_srat_mem_affinity *ma = (void *)header;
 485
 486         if (!ma)
 487                 return -EINVAL;
 488         if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
 489                 return 0;
 490         alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
 491         return 0;
 492 }
 493
 494 static u32 hmat_initiator_perf(struct memory_target *target,
 495                                struct memory_initiator *initiator,
 496                                struct acpi_hmat_locality *hmat_loc)
 497 {
 498         unsigned int ipds, tpds, i, idx = 0, tdx = 0;
 499         u32 *inits, *targs;
 500         u16 *entries;
 501
 502         ipds = hmat_loc->number_of_initiator_Pds;
 503         tpds = hmat_loc->number_of_target_Pds;
 504         inits = (u32 *)(hmat_loc + 1);
 505         targs = inits + ipds;
 506         entries = (u16 *)(targs + tpds);
 507
 508         for (i = 0; i < ipds; i++) {
 509                 if (inits[i] == initiator->processor_pxm) {
 510                         idx = i;
 511                         break;
 512                 }
 513         }
 514
 515         if (i == ipds)
 516                 return 0;
 517
 518         for (i = 0; i < tpds; i++) {
 519                 if (targs[i] == target->memory_pxm) {
 520                         tdx = i;
 521                         break;
 522                 }
 523         }
 524         if (i == tpds)
 525                 return 0;
 526
 527         return hmat_normalize(entries[idx * tpds + tdx],
 528                               hmat_loc->entry_base_unit,
 529                               hmat_loc->data_type);
 530 }
 531
 532 static bool hmat_update_best(u8 type, u32 value, u32 *best)
 533 {
 534         bool updated = false;
 535
 536         if (!value)
 537                 return false;
 538
 539         switch (type) {
 540         case ACPI_HMAT_ACCESS_LATENCY:
 541         case ACPI_HMAT_READ_LATENCY:
 542         case ACPI_HMAT_WRITE_LATENCY:
 543                 if (!*best || *best > value) {
 544                         *best = value;
 545                         updated = true;
 546                 }
 547                 break;
 548         case ACPI_HMAT_ACCESS_BANDWIDTH:
 549         case ACPI_HMAT_READ_BANDWIDTH:
 550         case ACPI_HMAT_WRITE_BANDWIDTH:
 551                 if (!*best || *best < value) {
 552                         *best = value;
 553                         updated = true;
 554                 }
 555                 break;
 556         }
 557
 558         return updated;
 559 }
 560
 561 static int initiator_cmp(void *priv, const struct list_head *a,
 562                          const struct list_head *b)
 563 {
 564         struct memory_initiator *ia;
 565         struct memory_initiator *ib;
 566
 567         ia = list_entry(a, struct memory_initiator, node);
 568         ib = list_entry(b, struct memory_initiator, node);
 569
 570         return ia->processor_pxm - ib->processor_pxm;
 571 }
 572
 573 static int initiators_to_nodemask(unsigned long *p_nodes)
 574 {
 575         struct memory_initiator *initiator;
 576
 577         if (list_empty(&initiators))
 578                 return -ENXIO;
 579
 580         list_for_each_entry(initiator, &initiators, node)
 581                 set_bit(initiator->processor_pxm, p_nodes);
 582
 583         return 0;
 584 }
 585
 586 static void hmat_register_target_initiators(struct memory_target *target)
 587 {
 588         static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
 589         struct memory_initiator *initiator;
 590         unsigned int mem_nid, cpu_nid;
 591         struct memory_locality *loc = NULL;
 592         u32 best = 0;
 593         bool access0done = false;
 594         int i;
 595
 596         mem_nid = pxm_to_node(target->memory_pxm);
 597         /*
 598          * If the Address Range Structure provides a local processor pxm, link
 599          * only that one. Otherwise, find the best performance attributes and
 600          * register all initiators that match.
 601          */
 602         if (target->processor_pxm != PXM_INVAL) {
 603                 cpu_nid = pxm_to_node(target->processor_pxm);
 604                 register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
 605                 access0done = true;
 606                 if (node_state(cpu_nid, N_CPU)) {
 607                         register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
 608                         return;
 609                 }
 610         }
 611
 612         if (list_empty(&localities))
 613                 return;
 614
 615         /*
 616          * We need the initiator list sorted so we can use bitmap_clear for
 617          * previously set initiators when we find a better memory accessor.
 618          * We'll also use the sorting to prime the candidate nodes with known
 619          * initiators.
 620          */
 621         bitmap_zero(p_nodes, MAX_NUMNODES);
 622         list_sort(NULL, &initiators, initiator_cmp);
 623         if (initiators_to_nodemask(p_nodes) < 0)
 624                 return;
 625
 626         if (!access0done) {
 627                 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
 628                         loc = localities_types[i];
 629                         if (!loc)
 630                                 continue;
 631
 632                         best = 0;
 633                         list_for_each_entry(initiator, &initiators, node) {
 634                                 u32 value;
 635
 636                                 if (!test_bit(initiator->processor_pxm, p_nodes))
 637                                         continue;
 638
 639                                 value = hmat_initiator_perf(target, initiator,
 640                                                             loc->hmat_loc);
 641                                 if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
 642                                         bitmap_clear(p_nodes, 0, initiator->processor_pxm);
 643                                 if (value != best)
 644                                         clear_bit(initiator->processor_pxm, p_nodes);
 645                         }
 646                         if (best)
 647                                 hmat_update_target_access(target, loc->hmat_loc->data_type,
 648                                                           best, 0);
 649                 }
 650
 651                 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
 652                         cpu_nid = pxm_to_node(i);
 653                         register_memory_node_under_compute_node(mem_nid, cpu_nid, 0);
 654                 }
 655         }
 656
 657         /* Access 1 ignores Generic Initiators */
 658         bitmap_zero(p_nodes, MAX_NUMNODES);
 659         if (initiators_to_nodemask(p_nodes) < 0)
 660                 return;
 661
 662         for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
 663                 loc = localities_types[i];
 664                 if (!loc)
 665                         continue;
 666
 667                 best = 0;
 668                 list_for_each_entry(initiator, &initiators, node) {
 669                         u32 value;
 670
 671                         if (!initiator->has_cpu) {
 672                                 clear_bit(initiator->processor_pxm, p_nodes);
 673                                 continue;
 674                         }
 675                         if (!test_bit(initiator->processor_pxm, p_nodes))
 676                                 continue;
 677
 678                         value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
 679                         if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
 680                                 bitmap_clear(p_nodes, 0, initiator->processor_pxm);
 681                         if (value != best)
 682                                 clear_bit(initiator->processor_pxm, p_nodes);
 683                 }
 684                 if (best)
 685                         hmat_update_target_access(target, loc->hmat_loc->data_type, best, 1);
 686         }
 687         for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
 688                 cpu_nid = pxm_to_node(i);
 689                 register_memory_node_under_compute_node(mem_nid, cpu_nid, 1);
 690         }
 691 }
 692
 693 static void hmat_register_target_cache(struct memory_target *target)
 694 {
 695         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 696         struct target_cache *tcache;
 697
 698         list_for_each_entry(tcache, &target->caches, node)
 699                 node_add_cache(mem_nid, &tcache->cache_attrs);
 700 }
 701
 702 static void hmat_register_target_perf(struct memory_target *target, int access)
 703 {
 704         unsigned mem_nid = pxm_to_node(target->memory_pxm);
 705         node_set_perf_attrs(mem_nid, &target->hmem_attrs[access], access);
 706 }
 707
 708 static void hmat_register_target_devices(struct memory_target *target)
 709 {
 710         struct resource *res;
 711
 712         /*
 713          * Do not bother creating devices if no driver is available to
 714          * consume them.
 715          */
 716         if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
 717                 return;
 718
 719         for (res = target->memregions.child; res; res = res->sibling) {
 720                 int target_nid = pxm_to_node(target->memory_pxm);
 721
 722                 hmem_register_device(target_nid, res);
 723         }
 724 }
 725
 726 static void hmat_register_target(struct memory_target *target)
 727 {
 728         int nid = pxm_to_node(target->memory_pxm);
 729
 730         /*
 731          * Devices may belong to either an offline or online
 732          * node, so unconditionally add them.
 733          */
 734         hmat_register_target_devices(target);
 735
 736         /*
 737          * Skip offline nodes. This can happen when memory
 738          * marked EFI_MEMORY_SP, "specific purpose", is applied
 739          * to all the memory in a promixity domain leading to
 740          * the node being marked offline / unplugged, or if
 741          * memory-only "hotplug" node is offline.
 742          */
 743         if (nid == NUMA_NO_NODE || !node_online(nid))
 744                 return;
 745
 746         mutex_lock(&target_lock);
 747         if (!target->registered) {
 748                 hmat_register_target_initiators(target);
 749                 hmat_register_target_cache(target);
 750                 hmat_register_target_perf(target, 0);
 751                 hmat_register_target_perf(target, 1);
 752                 target->registered = true;
 753         }
 754         mutex_unlock(&target_lock);
 755 }
 756
 757 static void hmat_register_targets(void)
 758 {
 759         struct memory_target *target;
 760
 761         list_for_each_entry(target, &targets, node)
 762                 hmat_register_target(target);
 763 }
 764
 765 static int hmat_callback(struct notifier_block *self,
 766                          unsigned long action, void *arg)
 767 {
 768         struct memory_target *target;
 769         struct memory_notify *mnb = arg;
 770         int pxm, nid = mnb->status_change_nid;
 771
 772         if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
 773                 return NOTIFY_OK;
 774
 775         pxm = node_to_pxm(nid);
 776         target = find_mem_target(pxm);
 777         if (!target)
 778                 return NOTIFY_OK;
 779
 780         hmat_register_target(target);
 781         return NOTIFY_OK;
 782 }
 783
 784 static struct notifier_block hmat_callback_nb = {
 785         .notifier_call = hmat_callback,
 786         .priority = 2,
 787 };
 788
 789 static __init void hmat_free_structures(void)
 790 {
 791         struct memory_target *target, *tnext;
 792         struct memory_locality *loc, *lnext;
 793         struct memory_initiator *initiator, *inext;
 794         struct target_cache *tcache, *cnext;
 795
 796         list_for_each_entry_safe(target, tnext, &targets, node) {
 797                 struct resource *res, *res_next;
 798
 799                 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
 800                         list_del(&tcache->node);
 801                         kfree(tcache);
 802                 }
 803
 804                 list_del(&target->node);
 805                 res = target->memregions.child;
 806                 while (res) {
 807                         res_next = res->sibling;
 808                         __release_region(&target->memregions, res->start,
 809                                         resource_size(res));
 810                         res = res_next;
 811                 }
 812                 kfree(target);
 813         }
 814
 815         list_for_each_entry_safe(initiator, inext, &initiators, node) {
 816                 list_del(&initiator->node);
 817                 kfree(initiator);
 818         }
 819
 820         list_for_each_entry_safe(loc, lnext, &localities, node) {
 821                 list_del(&loc->node);
 822                 kfree(loc);
 823         }
 824 }
 825
 826 static __init int hmat_init(void)
 827 {
 828         struct acpi_table_header *tbl;
 829         enum acpi_hmat_type i;
 830         acpi_status status;
 831
 832         if (srat_disabled() || hmat_disable)
 833                 return 0;
 834
 835         status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
 836         if (ACPI_FAILURE(status))
 837                 return 0;
 838
 839         if (acpi_table_parse_entries(ACPI_SIG_SRAT,
 840                                 sizeof(struct acpi_table_srat),
 841                                 ACPI_SRAT_TYPE_MEMORY_AFFINITY,
 842                                 srat_parse_mem_affinity, 0) < 0)
 843                 goto out_put;
 844         acpi_put_table(tbl);
 845
 846         status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
 847         if (ACPI_FAILURE(status))
 848                 goto out_put;
 849
 850         hmat_revision = tbl->revision;
 851         switch (hmat_revision) {
 852         case 1:
 853         case 2:
 854                 break;
 855         default:
 856                 pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision);
 857                 goto out_put;
 858         }
 859
 860         for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
 861                 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
 862                                              sizeof(struct acpi_table_hmat), i,
 863                                              hmat_parse_subtable, 0) < 0) {
 864                         pr_notice("Ignoring HMAT: Invalid table");
 865                         goto out_put;
 866                 }
 867         }
 868         hmat_register_targets();
 869
 870         /* Keep the table and structures if the notifier may use them */
 871         if (!register_hotmemory_notifier(&hmat_callback_nb))
 872                 return 0;
 873 out_put:
 874         hmat_free_structures();
 875         acpi_put_table(tbl);
 876         return 0;
 877 }
 878 device_initcall(hmat_init);