drivers/cxl/core/hdm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/io-64-nonatomic-hi-lo.h>
   4 #include <linux/device.h>
   5 #include <linux/delay.h>
   6
   7 #include "cxlmem.h"
   8 #include "core.h"
   9
  10 /**
  11  * DOC: cxl core hdm
  12  *
  13  * Compute Express Link Host Managed Device Memory, starting with the
  14  * CXL 2.0 specification, is managed by an array of HDM Decoder register
  15  * instances per CXL port and per CXL endpoint. Define common helpers
  16  * for enumerating these registers and capabilities.
  17  */
  18
  19 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
  20                            int *target_map)
  21 {
  22         int rc;
  23
  24         rc = cxl_decoder_add_locked(cxld, target_map);
  25         if (rc) {
  26                 put_device(&cxld->dev);
  27                 dev_err(&port->dev, "Failed to add decoder\n");
  28                 return rc;
  29         }
  30
  31         rc = cxl_decoder_autoremove(&port->dev, cxld);
  32         if (rc)
  33                 return rc;
  34
  35         dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
  36
  37         return 0;
  38 }
  39
  40 /*
  41  * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)
  42  * single ported host-bridges need not publish a decoder capability when a
  43  * passthrough decode can be assumed, i.e. all transactions that the uport sees
  44  * are claimed and passed to the single dport. Disable the range until the first
  45  * CXL region is enumerated / activated.
  46  */
  47 int devm_cxl_add_passthrough_decoder(struct cxl_port *port)
  48 {
  49         struct cxl_decoder *cxld;
  50         struct cxl_dport *dport;
  51         int single_port_map[1];
  52
  53         cxld = cxl_switch_decoder_alloc(port, 1);
  54         if (IS_ERR(cxld))
  55                 return PTR_ERR(cxld);
  56
  57         device_lock_assert(&port->dev);
  58
  59         dport = list_first_entry(&port->dports, typeof(*dport), list);
  60         single_port_map[0] = dport->port_id;
  61
  62         return add_hdm_decoder(port, cxld, single_port_map);
  63 }
  64 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);
  65
  66 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
  67 {
  68         u32 hdm_cap;
  69
  70         hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
  71         cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
  72         cxlhdm->target_count =
  73                 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
  74         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
  75                 cxlhdm->interleave_mask |= GENMASK(11, 8);
  76         if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))
  77                 cxlhdm->interleave_mask |= GENMASK(14, 12);
  78 }
  79
  80 static void __iomem *map_hdm_decoder_regs(struct cxl_port *port,
  81                                           void __iomem *crb)
  82 {
  83         struct cxl_component_reg_map map;
  84
  85         cxl_probe_component_regs(&port->dev, crb, &map);
  86         if (!map.hdm_decoder.valid) {
  87                 dev_err(&port->dev, "HDM decoder registers invalid\n");
  88                 return IOMEM_ERR_PTR(-ENXIO);
  89         }
  90
  91         return crb + map.hdm_decoder.offset;
  92 }
  93
  94 /**
  95  * devm_cxl_setup_hdm - map HDM decoder component registers
  96  * @port: cxl_port to map
  97  */
  98 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port)
  99 {
 100         struct device *dev = &port->dev;
 101         void __iomem *crb, *hdm;
 102         struct cxl_hdm *cxlhdm;
 103
 104         cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);
 105         if (!cxlhdm)
 106                 return ERR_PTR(-ENOMEM);
 107
 108         cxlhdm->port = port;
 109         crb = devm_cxl_iomap_block(dev, port->component_reg_phys,
 110                                    CXL_COMPONENT_REG_BLOCK_SIZE);
 111         if (!crb) {
 112                 dev_err(dev, "No component registers mapped\n");
 113                 return ERR_PTR(-ENXIO);
 114         }
 115
 116         hdm = map_hdm_decoder_regs(port, crb);
 117         if (IS_ERR(hdm))
 118                 return ERR_CAST(hdm);
 119         cxlhdm->regs.hdm_decoder = hdm;
 120
 121         parse_hdm_decoder_caps(cxlhdm);
 122         if (cxlhdm->decoder_count == 0) {
 123                 dev_err(dev, "Spec violation. Caps invalid\n");
 124                 return ERR_PTR(-ENXIO);
 125         }
 126
 127         return cxlhdm;
 128 }
 129 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);
 130
 131 static int to_interleave_granularity(u32 ctrl)
 132 {
 133         int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl);
 134
 135         return 256 << val;
 136 }
 137
 138 static int to_interleave_ways(u32 ctrl)
 139 {
 140         int val = FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl);
 141
 142         switch (val) {
 143         case 0 ... 4:
 144                 return 1 << val;
 145         case 8 ... 10:
 146                 return 3 << (val - 8);
 147         default:
 148                 return 0;
 149         }
 150 }
 151
 152 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
 153                             int *target_map, void __iomem *hdm, int which)
 154 {
 155         u64 size, base;
 156         u32 ctrl;
 157         int i;
 158         union {
 159                 u64 value;
 160                 unsigned char target_id[8];
 161         } target_list;
 162
 163         ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));
 164         base = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));
 165         size = ioread64_hi_lo(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));
 166
 167         if (!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED))
 168                 size = 0;
 169         if (base == U64_MAX || size == U64_MAX) {
 170                 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",
 171                          port->id, cxld->id);
 172                 return -ENXIO;
 173         }
 174
 175         cxld->decoder_range = (struct range) {
 176                 .start = base,
 177                 .end = base + size - 1,
 178         };
 179
 180         /* switch decoders are always enabled if committed */
 181         if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) {
 182                 cxld->flags |= CXL_DECODER_F_ENABLE;
 183                 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)
 184                         cxld->flags |= CXL_DECODER_F_LOCK;
 185         }
 186         cxld->interleave_ways = to_interleave_ways(ctrl);
 187         if (!cxld->interleave_ways) {
 188                 dev_warn(&port->dev,
 189                          "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",
 190                          port->id, cxld->id, ctrl);
 191                 return -ENXIO;
 192         }
 193         cxld->interleave_granularity = to_interleave_granularity(ctrl);
 194
 195         if (FIELD_GET(CXL_HDM_DECODER0_CTRL_TYPE, ctrl))
 196                 cxld->target_type = CXL_DECODER_EXPANDER;
 197         else
 198                 cxld->target_type = CXL_DECODER_ACCELERATOR;
 199
 200         if (is_endpoint_decoder(&cxld->dev))
 201                 return 0;
 202
 203         target_list.value =
 204                 ioread64_hi_lo(hdm + CXL_HDM_DECODER0_TL_LOW(which));
 205         for (i = 0; i < cxld->interleave_ways; i++)
 206                 target_map[i] = target_list.target_id[i];
 207
 208         return 0;
 209 }
 210
 211 /**
 212  * devm_cxl_enumerate_decoders - add decoder objects per HDM register set
 213  * @cxlhdm: Structure to populate with HDM capabilities
 214  */
 215 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm)
 216 {
 217         void __iomem *hdm = cxlhdm->regs.hdm_decoder;
 218         struct cxl_port *port = cxlhdm->port;
 219         int i, committed, failed;
 220         u32 ctrl;
 221
 222         /*
 223          * Since the register resource was recently claimed via request_region()
 224          * be careful about trusting the "not-committed" status until the commit
 225          * timeout has elapsed.  The commit timeout is 10ms (CXL 2.0
 226          * 8.2.5.12.20), but double it to be tolerant of any clock skew between
 227          * host and target.
 228          */
 229         for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {
 230                 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));
 231                 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)
 232                         committed++;
 233         }
 234
 235         /* ensure that future checks of committed can be trusted */
 236         if (committed != cxlhdm->decoder_count)
 237                 msleep(20);
 238
 239         for (i = 0, failed = 0; i < cxlhdm->decoder_count; i++) {
 240                 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
 241                 int rc, target_count = cxlhdm->target_count;
 242                 struct cxl_decoder *cxld;
 243
 244                 if (is_cxl_endpoint(port))
 245                         cxld = cxl_endpoint_decoder_alloc(port);
 246                 else
 247                         cxld = cxl_switch_decoder_alloc(port, target_count);
 248                 if (IS_ERR(cxld)) {
 249                         dev_warn(&port->dev,
 250                                  "Failed to allocate the decoder\n");
 251                         return PTR_ERR(cxld);
 252                 }
 253
 254                 rc = init_hdm_decoder(port, cxld, target_map,
 255                                       cxlhdm->regs.hdm_decoder, i);
 256                 if (rc) {
 257                         put_device(&cxld->dev);
 258                         failed++;
 259                         continue;
 260                 }
 261                 rc = add_hdm_decoder(port, cxld, target_map);
 262                 if (rc) {
 263                         dev_warn(&port->dev,
 264                                  "Failed to add decoder to port\n");
 265                         return rc;
 266                 }
 267         }
 268
 269         if (failed == cxlhdm->decoder_count) {
 270                 dev_err(&port->dev, "No valid decoders found\n");
 271                 return -ENXIO;
 272         }
 273
 274         return 0;
 275 }
 276 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);