drivers/crypto/nx/nx-842-pseries.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Driver for IBM Power 842 compression accelerator
   4  *
   5  * Copyright (C) IBM Corporation, 2012
   6  *
   7  * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
   8  *          Seth Jennings <sjenning@linux.vnet.ibm.com>
   9  */
  10
  11 #include <asm/vio.h>
  12
  13 #include "nx-842.h"
  14 #include "nx_csbcpb.h" /* struct nx_csbcpb */
  15
  16 MODULE_LICENSE("GPL");
  17 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  18 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  19 MODULE_ALIAS_CRYPTO("842");
  20 MODULE_ALIAS_CRYPTO("842-nx");
  21
  22 static struct nx842_constraints nx842_pseries_constraints = {
  23         .alignment =    DDE_BUFFER_ALIGN,
  24         .multiple =     DDE_BUFFER_LAST_MULT,
  25         .minimum =      DDE_BUFFER_LAST_MULT,
  26         .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  27 };
  28
  29 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  30 {
  31         if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  32                 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  33                          in ? "input" : "output", buf,
  34                          nx842_pseries_constraints.alignment);
  35                 return -EINVAL;
  36         }
  37         if (*len % nx842_pseries_constraints.multiple) {
  38                 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  39                          in ? "input" : "output", *len,
  40                          nx842_pseries_constraints.multiple);
  41                 if (in)
  42                         return -EINVAL;
  43                 *len = round_down(*len, nx842_pseries_constraints.multiple);
  44         }
  45         if (*len < nx842_pseries_constraints.minimum) {
  46                 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  47                          in ? "input" : "output", *len,
  48                          nx842_pseries_constraints.minimum);
  49                 return -EINVAL;
  50         }
  51         if (*len > nx842_pseries_constraints.maximum) {
  52                 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  53                          in ? "input" : "output", *len,
  54                          nx842_pseries_constraints.maximum);
  55                 if (in)
  56                         return -EINVAL;
  57                 *len = nx842_pseries_constraints.maximum;
  58         }
  59         return 0;
  60 }
  61
  62 /* I assume we need to align the CSB? */
  63 #define WORKMEM_ALIGN   (256)
  64
  65 struct nx842_workmem {
  66         /* scatterlist */
  67         char slin[4096];
  68         char slout[4096];
  69         /* coprocessor status/parameter block */
  70         struct nx_csbcpb csbcpb;
  71
  72         char padding[WORKMEM_ALIGN];
  73 } __aligned(WORKMEM_ALIGN);
  74
  75 /* Macros for fields within nx_csbcpb */
  76 /* Check the valid bit within the csbcpb valid field */
  77 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  78
  79 /* CE macros operate on the completion_extension field bits in the csbcpb.
  80  * CE0 0=full completion, 1=partial completion
  81  * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  82  * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  83 #define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  84 #define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  85 #define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  86
  87 /* The NX unit accepts data only on 4K page boundaries */
  88 #define NX842_HW_PAGE_SIZE      (4096)
  89 #define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
  90
  91 struct ibm_nx842_counters {
  92         atomic64_t comp_complete;
  93         atomic64_t comp_failed;
  94         atomic64_t decomp_complete;
  95         atomic64_t decomp_failed;
  96         atomic64_t swdecomp;
  97         atomic64_t comp_times[32];
  98         atomic64_t decomp_times[32];
  99 };
 100
 101 static struct nx842_devdata {
 102         struct vio_dev *vdev;
 103         struct device *dev;
 104         struct ibm_nx842_counters *counters;
 105         unsigned int max_sg_len;
 106         unsigned int max_sync_size;
 107         unsigned int max_sync_sg;
 108 } __rcu *devdata;
 109 static DEFINE_SPINLOCK(devdata_mutex);
 110
 111 #define NX842_COUNTER_INC(_x) \
 112 static inline void nx842_inc_##_x( \
 113         const struct nx842_devdata *dev) { \
 114         if (dev) \
 115                 atomic64_inc(&dev->counters->_x); \
 116 }
 117 NX842_COUNTER_INC(comp_complete);
 118 NX842_COUNTER_INC(comp_failed);
 119 NX842_COUNTER_INC(decomp_complete);
 120 NX842_COUNTER_INC(decomp_failed);
 121 NX842_COUNTER_INC(swdecomp);
 122
 123 #define NX842_HIST_SLOTS 16
 124
 125 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 126 {
 127         int bucket = fls(time);
 128
 129         if (bucket)
 130                 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 131
 132         atomic64_inc(&times[bucket]);
 133 }
 134
 135 /* NX unit operation flags */
 136 #define NX842_OP_COMPRESS       0x0
 137 #define NX842_OP_CRC            0x1
 138 #define NX842_OP_DECOMPRESS     0x2
 139 #define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 140 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 141 #define NX842_OP_ASYNC          (1<<23)
 142 #define NX842_OP_NOTIFY         (1<<22)
 143 #define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 144
 145 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 146 {
 147         /* No use of DMA mappings within the driver. */
 148         return 0;
 149 }
 150
 151 struct nx842_slentry {
 152         __be64 ptr; /* Real address (use __pa()) */
 153         __be64 len;
 154 };
 155
 156 /* pHyp scatterlist entry */
 157 struct nx842_scatterlist {
 158         int entry_nr; /* number of slentries */
 159         struct nx842_slentry *entries; /* ptr to array of slentries */
 160 };
 161
 162 /* Does not include sizeof(entry_nr) in the size */
 163 static inline unsigned long nx842_get_scatterlist_size(
 164                                 struct nx842_scatterlist *sl)
 165 {
 166         return sl->entry_nr * sizeof(struct nx842_slentry);
 167 }
 168
 169 static int nx842_build_scatterlist(unsigned long buf, int len,
 170                         struct nx842_scatterlist *sl)
 171 {
 172         unsigned long entrylen;
 173         struct nx842_slentry *entry;
 174
 175         sl->entry_nr = 0;
 176
 177         entry = sl->entries;
 178         while (len) {
 179                 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 180                 entrylen = min_t(int, len,
 181                                  LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 182                 entry->len = cpu_to_be64(entrylen);
 183
 184                 len -= entrylen;
 185                 buf += entrylen;
 186
 187                 sl->entry_nr++;
 188                 entry++;
 189         }
 190
 191         return 0;
 192 }
 193
 194 static int nx842_validate_result(struct device *dev,
 195         struct cop_status_block *csb)
 196 {
 197         /* The csb must be valid after returning from vio_h_cop_sync */
 198         if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 199                 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 200                                 __func__);
 201                 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 202                                 csb->valid,
 203                                 csb->crb_seq_number,
 204                                 csb->completion_code,
 205                                 csb->completion_extension);
 206                 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 207                                 be32_to_cpu(csb->processed_byte_count),
 208                                 (unsigned long)be64_to_cpu(csb->address));
 209                 return -EIO;
 210         }
 211
 212         /* Check return values from the hardware in the CSB */
 213         switch (csb->completion_code) {
 214         case 0: /* Completed without error */
 215                 break;
 216         case 64: /* Compression ok, but output larger than input */
 217                 dev_dbg(dev, "%s: output size larger than input size\n",
 218                                         __func__);
 219                 break;
 220         case 13: /* Output buffer too small */
 221                 dev_dbg(dev, "%s: Out of space in output buffer\n",
 222                                         __func__);
 223                 return -ENOSPC;
 224         case 65: /* Calculated CRC doesn't match the passed value */
 225                 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
 226                                         __func__);
 227                 return -EINVAL;
 228         case 66: /* Input data contains an illegal template field */
 229         case 67: /* Template indicates data past the end of the input stream */
 230                 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 231                                         __func__, csb->completion_code);
 232                 return -EINVAL;
 233         default:
 234                 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 235                                         __func__, csb->completion_code);
 236                 return -EIO;
 237         }
 238
 239         /* Hardware sanity check */
 240         if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 241                 dev_err(dev, "%s: No error returned by hardware, but "
 242                                 "data returned is unusable, contact support.\n"
 243                                 "(Additional info: csbcbp->processed bytes "
 244                                 "does not specify processed bytes for the "
 245                                 "target buffer.)\n", __func__);
 246                 return -EIO;
 247         }
 248
 249         return 0;
 250 }
 251
 252 /**
 253  * nx842_pseries_compress - Compress data using the 842 algorithm
 254  *
 255  * Compression provide by the NX842 coprocessor on IBM Power systems.
 256  * The input buffer is compressed and the result is stored in the
 257  * provided output buffer.
 258  *
 259  * Upon return from this function @outlen contains the length of the
 260  * compressed data.  If there is an error then @outlen will be 0 and an
 261  * error will be specified by the return code from this function.
 262  *
 263  * @in: Pointer to input buffer
 264  * @inlen: Length of input buffer
 265  * @out: Pointer to output buffer
 266  * @outlen: Length of output buffer
 267  * @wrkmem: ptr to buffer for working memory, size determined by
 268  *          nx842_pseries_driver.workmem_size
 269  *
 270  * Returns:
 271  *   0          Success, output of length @outlen stored in the buffer at @out
 272  *   -ENOMEM    Unable to allocate internal buffers
 273  *   -ENOSPC    Output buffer is to small
 274  *   -EIO       Internal error
 275  *   -ENODEV    Hardware unavailable
 276  */
 277 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 278                                   unsigned char *out, unsigned int *outlen,
 279                                   void *wmem)
 280 {
 281         struct nx842_devdata *local_devdata;
 282         struct device *dev = NULL;
 283         struct nx842_workmem *workmem;
 284         struct nx842_scatterlist slin, slout;
 285         struct nx_csbcpb *csbcpb;
 286         int ret = 0;
 287         unsigned long inbuf, outbuf;
 288         struct vio_pfo_op op = {
 289                 .done = NULL,
 290                 .handle = 0,
 291                 .timeout = 0,
 292         };
 293         unsigned long start = get_tb();
 294
 295         inbuf = (unsigned long)in;
 296         if (check_constraints(inbuf, &inlen, true))
 297                 return -EINVAL;
 298
 299         outbuf = (unsigned long)out;
 300         if (check_constraints(outbuf, outlen, false))
 301                 return -EINVAL;
 302
 303         rcu_read_lock();
 304         local_devdata = rcu_dereference(devdata);
 305         if (!local_devdata || !local_devdata->dev) {
 306                 rcu_read_unlock();
 307                 return -ENODEV;
 308         }
 309         dev = local_devdata->dev;
 310
 311         /* Init scatterlist */
 312         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 313         slin.entries = (struct nx842_slentry *)workmem->slin;
 314         slout.entries = (struct nx842_slentry *)workmem->slout;
 315
 316         /* Init operation */
 317         op.flags = NX842_OP_COMPRESS_CRC;
 318         csbcpb = &workmem->csbcpb;
 319         memset(csbcpb, 0, sizeof(*csbcpb));
 320         op.csbcpb = nx842_get_pa(csbcpb);
 321
 322         if ((inbuf & NX842_HW_PAGE_MASK) ==
 323             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 324                 /* Create direct DDE */
 325                 op.in = nx842_get_pa((void *)inbuf);
 326                 op.inlen = inlen;
 327         } else {
 328                 /* Create indirect DDE (scatterlist) */
 329                 nx842_build_scatterlist(inbuf, inlen, &slin);
 330                 op.in = nx842_get_pa(slin.entries);
 331                 op.inlen = -nx842_get_scatterlist_size(&slin);
 332         }
 333
 334         if ((outbuf & NX842_HW_PAGE_MASK) ==
 335             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 336                 /* Create direct DDE */
 337                 op.out = nx842_get_pa((void *)outbuf);
 338                 op.outlen = *outlen;
 339         } else {
 340                 /* Create indirect DDE (scatterlist) */
 341                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 342                 op.out = nx842_get_pa(slout.entries);
 343                 op.outlen = -nx842_get_scatterlist_size(&slout);
 344         }
 345
 346         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 347                 __func__, (unsigned long)op.in, (long)op.inlen,
 348                 (unsigned long)op.out, (long)op.outlen);
 349
 350         /* Send request to pHyp */
 351         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 352
 353         /* Check for pHyp error */
 354         if (ret) {
 355                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 356                         __func__, ret, op.hcall_err);
 357                 ret = -EIO;
 358                 goto unlock;
 359         }
 360
 361         /* Check for hardware error */
 362         ret = nx842_validate_result(dev, &csbcpb->csb);
 363         if (ret)
 364                 goto unlock;
 365
 366         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 367         dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 368
 369 unlock:
 370         if (ret)
 371                 nx842_inc_comp_failed(local_devdata);
 372         else {
 373                 nx842_inc_comp_complete(local_devdata);
 374                 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 375                         (get_tb() - start) / tb_ticks_per_usec);
 376         }
 377         rcu_read_unlock();
 378         return ret;
 379 }
 380
 381 /**
 382  * nx842_pseries_decompress - Decompress data using the 842 algorithm
 383  *
 384  * Decompression provide by the NX842 coprocessor on IBM Power systems.
 385  * The input buffer is decompressed and the result is stored in the
 386  * provided output buffer.  The size allocated to the output buffer is
 387  * provided by the caller of this function in @outlen.  Upon return from
 388  * this function @outlen contains the length of the decompressed data.
 389  * If there is an error then @outlen will be 0 and an error will be
 390  * specified by the return code from this function.
 391  *
 392  * @in: Pointer to input buffer
 393  * @inlen: Length of input buffer
 394  * @out: Pointer to output buffer
 395  * @outlen: Length of output buffer
 396  * @wrkmem: ptr to buffer for working memory, size determined by
 397  *          nx842_pseries_driver.workmem_size
 398  *
 399  * Returns:
 400  *   0          Success, output of length @outlen stored in the buffer at @out
 401  *   -ENODEV    Hardware decompression device is unavailable
 402  *   -ENOMEM    Unable to allocate internal buffers
 403  *   -ENOSPC    Output buffer is to small
 404  *   -EINVAL    Bad input data encountered when attempting decompress
 405  *   -EIO       Internal error
 406  */
 407 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 408                                     unsigned char *out, unsigned int *outlen,
 409                                     void *wmem)
 410 {
 411         struct nx842_devdata *local_devdata;
 412         struct device *dev = NULL;
 413         struct nx842_workmem *workmem;
 414         struct nx842_scatterlist slin, slout;
 415         struct nx_csbcpb *csbcpb;
 416         int ret = 0;
 417         unsigned long inbuf, outbuf;
 418         struct vio_pfo_op op = {
 419                 .done = NULL,
 420                 .handle = 0,
 421                 .timeout = 0,
 422         };
 423         unsigned long start = get_tb();
 424
 425         /* Ensure page alignment and size */
 426         inbuf = (unsigned long)in;
 427         if (check_constraints(inbuf, &inlen, true))
 428                 return -EINVAL;
 429
 430         outbuf = (unsigned long)out;
 431         if (check_constraints(outbuf, outlen, false))
 432                 return -EINVAL;
 433
 434         rcu_read_lock();
 435         local_devdata = rcu_dereference(devdata);
 436         if (!local_devdata || !local_devdata->dev) {
 437                 rcu_read_unlock();
 438                 return -ENODEV;
 439         }
 440         dev = local_devdata->dev;
 441
 442         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 443
 444         /* Init scatterlist */
 445         slin.entries = (struct nx842_slentry *)workmem->slin;
 446         slout.entries = (struct nx842_slentry *)workmem->slout;
 447
 448         /* Init operation */
 449         op.flags = NX842_OP_DECOMPRESS_CRC;
 450         csbcpb = &workmem->csbcpb;
 451         memset(csbcpb, 0, sizeof(*csbcpb));
 452         op.csbcpb = nx842_get_pa(csbcpb);
 453
 454         if ((inbuf & NX842_HW_PAGE_MASK) ==
 455             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 456                 /* Create direct DDE */
 457                 op.in = nx842_get_pa((void *)inbuf);
 458                 op.inlen = inlen;
 459         } else {
 460                 /* Create indirect DDE (scatterlist) */
 461                 nx842_build_scatterlist(inbuf, inlen, &slin);
 462                 op.in = nx842_get_pa(slin.entries);
 463                 op.inlen = -nx842_get_scatterlist_size(&slin);
 464         }
 465
 466         if ((outbuf & NX842_HW_PAGE_MASK) ==
 467             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 468                 /* Create direct DDE */
 469                 op.out = nx842_get_pa((void *)outbuf);
 470                 op.outlen = *outlen;
 471         } else {
 472                 /* Create indirect DDE (scatterlist) */
 473                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 474                 op.out = nx842_get_pa(slout.entries);
 475                 op.outlen = -nx842_get_scatterlist_size(&slout);
 476         }
 477
 478         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 479                 __func__, (unsigned long)op.in, (long)op.inlen,
 480                 (unsigned long)op.out, (long)op.outlen);
 481
 482         /* Send request to pHyp */
 483         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 484
 485         /* Check for pHyp error */
 486         if (ret) {
 487                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 488                         __func__, ret, op.hcall_err);
 489                 goto unlock;
 490         }
 491
 492         /* Check for hardware error */
 493         ret = nx842_validate_result(dev, &csbcpb->csb);
 494         if (ret)
 495                 goto unlock;
 496
 497         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 498
 499 unlock:
 500         if (ret)
 501                 /* decompress fail */
 502                 nx842_inc_decomp_failed(local_devdata);
 503         else {
 504                 nx842_inc_decomp_complete(local_devdata);
 505                 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 506                         (get_tb() - start) / tb_ticks_per_usec);
 507         }
 508
 509         rcu_read_unlock();
 510         return ret;
 511 }
 512
 513 /**
 514  * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 515  *
 516  * @devdata - struct nx842_devdata to update
 517  *
 518  * Returns:
 519  *  0 on success
 520  *  -ENOENT if @devdata ptr is NULL
 521  */
 522 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 523 {
 524         if (devdata) {
 525                 devdata->max_sync_size = 0;
 526                 devdata->max_sync_sg = 0;
 527                 devdata->max_sg_len = 0;
 528                 return 0;
 529         } else
 530                 return -ENOENT;
 531 }
 532
 533 /**
 534  * nx842_OF_upd_status -- Check the device info from OF status prop
 535  *
 536  * The status property indicates if the accelerator is enabled.  If the
 537  * device is in the OF tree it indicates that the hardware is present.
 538  * The status field indicates if the device is enabled when the status
 539  * is 'okay'.  Otherwise the device driver will be disabled.
 540  *
 541  * @devdata: struct nx842_devdata to use for dev_info
 542  * @prop: struct property point containing the maxsyncop for the update
 543  *
 544  * Returns:
 545  *  0 - Device is available
 546  *  -ENODEV - Device is not available
 547  */
 548 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
 549                                struct property *prop)
 550 {
 551         const char *status = (const char *)prop->value;
 552
 553         if (!strncmp(status, "okay", (size_t)prop->length))
 554                 return 0;
 555         if (!strncmp(status, "disabled", (size_t)prop->length))
 556                 return -ENODEV;
 557         dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
 558
 559         return -EINVAL;
 560 }
 561
 562 /**
 563  * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 564  *
 565  * Definition of the 'ibm,max-sg-len' OF property:
 566  *  This field indicates the maximum byte length of a scatter list
 567  *  for the platform facility. It is a single cell encoded as with encode-int.
 568  *
 569  * Example:
 570  *  # od -x ibm,max-sg-len
 571  *  0000000 0000 0ff0
 572  *
 573  *  In this example, the maximum byte length of a scatter list is
 574  *  0x0ff0 (4,080).
 575  *
 576  * @devdata - struct nx842_devdata to update
 577  * @prop - struct property point containing the maxsyncop for the update
 578  *
 579  * Returns:
 580  *  0 on success
 581  *  -EINVAL on failure
 582  */
 583 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 584                                         struct property *prop) {
 585         int ret = 0;
 586         const unsigned int maxsglen = of_read_number(prop->value, 1);
 587
 588         if (prop->length != sizeof(maxsglen)) {
 589                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 590                 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 591                                 prop->length, sizeof(maxsglen));
 592                 ret = -EINVAL;
 593         } else {
 594                 devdata->max_sg_len = min_t(unsigned int,
 595                                             maxsglen, NX842_HW_PAGE_SIZE);
 596         }
 597
 598         return ret;
 599 }
 600
 601 /**
 602  * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 603  *
 604  * Definition of the 'ibm,max-sync-cop' OF property:
 605  *  Two series of cells.  The first series of cells represents the maximums
 606  *  that can be synchronously compressed. The second series of cells
 607  *  represents the maximums that can be synchronously decompressed.
 608  *  1. The first cell in each series contains the count of the number of
 609  *     data length, scatter list elements pairs that follow – each being
 610  *     of the form
 611  *    a. One cell data byte length
 612  *    b. One cell total number of scatter list elements
 613  *
 614  * Example:
 615  *  # od -x ibm,max-sync-cop
 616  *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 617  *  0000020 0000 1000 0000 01fe
 618  *
 619  *  In this example, compression supports 0x1000 (4,096) data byte length
 620  *  and 0x1fe (510) total scatter list elements.  Decompression supports
 621  *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 622  *  elements.
 623  *
 624  * @devdata - struct nx842_devdata to update
 625  * @prop - struct property point containing the maxsyncop for the update
 626  *
 627  * Returns:
 628  *  0 on success
 629  *  -EINVAL on failure
 630  */
 631 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 632                                         struct property *prop) {
 633         int ret = 0;
 634         unsigned int comp_data_limit, decomp_data_limit;
 635         unsigned int comp_sg_limit, decomp_sg_limit;
 636         const struct maxsynccop_t {
 637                 __be32 comp_elements;
 638                 __be32 comp_data_limit;
 639                 __be32 comp_sg_limit;
 640                 __be32 decomp_elements;
 641                 __be32 decomp_data_limit;
 642                 __be32 decomp_sg_limit;
 643         } *maxsynccop;
 644
 645         if (prop->length != sizeof(*maxsynccop)) {
 646                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 647                 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 648                                 sizeof(*maxsynccop));
 649                 ret = -EINVAL;
 650                 goto out;
 651         }
 652
 653         maxsynccop = (const struct maxsynccop_t *)prop->value;
 654         comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 655         comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 656         decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 657         decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 658
 659         /* Use one limit rather than separate limits for compression and
 660          * decompression. Set a maximum for this so as not to exceed the
 661          * size that the header can support and round the value down to
 662          * the hardware page size (4K) */
 663         devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 664
 665         devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 666                                         65536);
 667
 668         if (devdata->max_sync_size < 4096) {
 669                 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 670                                 "less than the driver minimum, unable to use "
 671                                 "the hardware device\n",
 672                                 __func__, devdata->max_sync_size);
 673                 ret = -EINVAL;
 674                 goto out;
 675         }
 676
 677         nx842_pseries_constraints.maximum = devdata->max_sync_size;
 678
 679         devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 680         if (devdata->max_sync_sg < 1) {
 681                 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 682                                 "less than the driver minimum, unable to use "
 683                                 "the hardware device\n",
 684                                 __func__, devdata->max_sync_sg);
 685                 ret = -EINVAL;
 686                 goto out;
 687         }
 688
 689 out:
 690         return ret;
 691 }
 692
 693 /**
 694  *
 695  * nx842_OF_upd -- Handle OF properties updates for the device.
 696  *
 697  * Set all properties from the OF tree.  Optionally, a new property
 698  * can be provided by the @new_prop pointer to overwrite an existing value.
 699  * The device will remain disabled until all values are valid, this function
 700  * will return an error for updates unless all values are valid.
 701  *
 702  * @new_prop: If not NULL, this property is being updated.  If NULL, update
 703  *  all properties from the current values in the OF tree.
 704  *
 705  * Returns:
 706  *  0 - Success
 707  *  -ENOMEM - Could not allocate memory for new devdata structure
 708  *  -EINVAL - property value not found, new_prop is not a recognized
 709  *      property for the device or property value is not valid.
 710  *  -ENODEV - Device is not available
 711  */
 712 static int nx842_OF_upd(struct property *new_prop)
 713 {
 714         struct nx842_devdata *old_devdata = NULL;
 715         struct nx842_devdata *new_devdata = NULL;
 716         struct device_node *of_node = NULL;
 717         struct property *status = NULL;
 718         struct property *maxsglen = NULL;
 719         struct property *maxsyncop = NULL;
 720         int ret = 0;
 721         unsigned long flags;
 722
 723         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 724         if (!new_devdata)
 725                 return -ENOMEM;
 726
 727         spin_lock_irqsave(&devdata_mutex, flags);
 728         old_devdata = rcu_dereference_check(devdata,
 729                         lockdep_is_held(&devdata_mutex));
 730         if (old_devdata)
 731                 of_node = old_devdata->dev->of_node;
 732
 733         if (!old_devdata || !of_node) {
 734                 pr_err("%s: device is not available\n", __func__);
 735                 spin_unlock_irqrestore(&devdata_mutex, flags);
 736                 kfree(new_devdata);
 737                 return -ENODEV;
 738         }
 739
 740         memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 741         new_devdata->counters = old_devdata->counters;
 742
 743         /* Set ptrs for existing properties */
 744         status = of_find_property(of_node, "status", NULL);
 745         maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 746         maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 747         if (!status || !maxsglen || !maxsyncop) {
 748                 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 749                 ret = -EINVAL;
 750                 goto error_out;
 751         }
 752
 753         /*
 754          * If this is a property update, there are only certain properties that
 755          * we care about. Bail if it isn't in the below list
 756          */
 757         if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 758                          strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 759                          strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 760                 goto out;
 761
 762         /* Perform property updates */
 763         ret = nx842_OF_upd_status(new_devdata, status);
 764         if (ret)
 765                 goto error_out;
 766
 767         ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 768         if (ret)
 769                 goto error_out;
 770
 771         ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 772         if (ret)
 773                 goto error_out;
 774
 775 out:
 776         dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 777                         __func__, new_devdata->max_sync_size,
 778                         old_devdata->max_sync_size);
 779         dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 780                         __func__, new_devdata->max_sync_sg,
 781                         old_devdata->max_sync_sg);
 782         dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 783                         __func__, new_devdata->max_sg_len,
 784                         old_devdata->max_sg_len);
 785
 786         rcu_assign_pointer(devdata, new_devdata);
 787         spin_unlock_irqrestore(&devdata_mutex, flags);
 788         synchronize_rcu();
 789         dev_set_drvdata(new_devdata->dev, new_devdata);
 790         kfree(old_devdata);
 791         return 0;
 792
 793 error_out:
 794         if (new_devdata) {
 795                 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 796                 nx842_OF_set_defaults(new_devdata);
 797                 rcu_assign_pointer(devdata, new_devdata);
 798                 spin_unlock_irqrestore(&devdata_mutex, flags);
 799                 synchronize_rcu();
 800                 dev_set_drvdata(new_devdata->dev, new_devdata);
 801                 kfree(old_devdata);
 802         } else {
 803                 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 804                 spin_unlock_irqrestore(&devdata_mutex, flags);
 805         }
 806
 807         if (!ret)
 808                 ret = -EINVAL;
 809         return ret;
 810 }
 811
 812 /**
 813  * nx842_OF_notifier - Process updates to OF properties for the device
 814  *
 815  * @np: notifier block
 816  * @action: notifier action
 817  * @update: struct pSeries_reconfig_prop_update pointer if action is
 818  *      PSERIES_UPDATE_PROPERTY
 819  *
 820  * Returns:
 821  *      NOTIFY_OK on success
 822  *      NOTIFY_BAD encoded with error number on failure, use
 823  *              notifier_to_errno() to decode this value
 824  */
 825 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 826                              void *data)
 827 {
 828         struct of_reconfig_data *upd = data;
 829         struct nx842_devdata *local_devdata;
 830         struct device_node *node = NULL;
 831
 832         rcu_read_lock();
 833         local_devdata = rcu_dereference(devdata);
 834         if (local_devdata)
 835                 node = local_devdata->dev->of_node;
 836
 837         if (local_devdata &&
 838                         action == OF_RECONFIG_UPDATE_PROPERTY &&
 839                         !strcmp(upd->dn->name, node->name)) {
 840                 rcu_read_unlock();
 841                 nx842_OF_upd(upd->prop);
 842         } else
 843                 rcu_read_unlock();
 844
 845         return NOTIFY_OK;
 846 }
 847
 848 static struct notifier_block nx842_of_nb = {
 849         .notifier_call = nx842_OF_notifier,
 850 };
 851
 852 #define nx842_counter_read(_name)                                       \
 853 static ssize_t nx842_##_name##_show(struct device *dev,         \
 854                 struct device_attribute *attr,                          \
 855                 char *buf) {                                            \
 856         struct nx842_devdata *local_devdata;                    \
 857         int p = 0;                                                      \
 858         rcu_read_lock();                                                \
 859         local_devdata = rcu_dereference(devdata);                       \
 860         if (local_devdata)                                              \
 861                 p = snprintf(buf, PAGE_SIZE, "%lld\n",                  \
 862                        atomic64_read(&local_devdata->counters->_name)); \
 863         rcu_read_unlock();                                              \
 864         return p;                                                       \
 865 }
 866
 867 #define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 868         nx842_counter_read(_name);                                      \
 869         static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 870                                                 0444,                   \
 871                                                 nx842_##_name##_show,\
 872                                                 NULL);
 873
 874 NX842DEV_COUNTER_ATTR_RO(comp_complete);
 875 NX842DEV_COUNTER_ATTR_RO(comp_failed);
 876 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 877 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 878 NX842DEV_COUNTER_ATTR_RO(swdecomp);
 879
 880 static ssize_t nx842_timehist_show(struct device *,
 881                 struct device_attribute *, char *);
 882
 883 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 884                 nx842_timehist_show, NULL);
 885 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 886                 0444, nx842_timehist_show, NULL);
 887
 888 static ssize_t nx842_timehist_show(struct device *dev,
 889                 struct device_attribute *attr, char *buf) {
 890         char *p = buf;
 891         struct nx842_devdata *local_devdata;
 892         atomic64_t *times;
 893         int bytes_remain = PAGE_SIZE;
 894         int bytes;
 895         int i;
 896
 897         rcu_read_lock();
 898         local_devdata = rcu_dereference(devdata);
 899         if (!local_devdata) {
 900                 rcu_read_unlock();
 901                 return 0;
 902         }
 903
 904         if (attr == &dev_attr_comp_times)
 905                 times = local_devdata->counters->comp_times;
 906         else if (attr == &dev_attr_decomp_times)
 907                 times = local_devdata->counters->decomp_times;
 908         else {
 909                 rcu_read_unlock();
 910                 return 0;
 911         }
 912
 913         for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 914                 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
 915                                i ? (2<<(i-1)) : 0, (2<<i)-1,
 916                                atomic64_read(&times[i]));
 917                 bytes_remain -= bytes;
 918                 p += bytes;
 919         }
 920         /* The last bucket holds everything over
 921          * 2<<(NX842_HIST_SLOTS - 2) us */
 922         bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
 923                         2<<(NX842_HIST_SLOTS - 2),
 924                         atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 925         p += bytes;
 926
 927         rcu_read_unlock();
 928         return p - buf;
 929 }
 930
 931 static struct attribute *nx842_sysfs_entries[] = {
 932         &dev_attr_comp_complete.attr,
 933         &dev_attr_comp_failed.attr,
 934         &dev_attr_decomp_complete.attr,
 935         &dev_attr_decomp_failed.attr,
 936         &dev_attr_swdecomp.attr,
 937         &dev_attr_comp_times.attr,
 938         &dev_attr_decomp_times.attr,
 939         NULL,
 940 };
 941
 942 static struct attribute_group nx842_attribute_group = {
 943         .name = NULL,           /* put in device directory */
 944         .attrs = nx842_sysfs_entries,
 945 };
 946
 947 static struct nx842_driver nx842_pseries_driver = {
 948         .name =         KBUILD_MODNAME,
 949         .owner =        THIS_MODULE,
 950         .workmem_size = sizeof(struct nx842_workmem),
 951         .constraints =  &nx842_pseries_constraints,
 952         .compress =     nx842_pseries_compress,
 953         .decompress =   nx842_pseries_decompress,
 954 };
 955
 956 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
 957 {
 958         return nx842_crypto_init(tfm, &nx842_pseries_driver);
 959 }
 960
 961 static struct crypto_alg nx842_pseries_alg = {
 962         .cra_name               = "842",
 963         .cra_driver_name        = "842-nx",
 964         .cra_priority           = 300,
 965         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 966         .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
 967         .cra_module             = THIS_MODULE,
 968         .cra_init               = nx842_pseries_crypto_init,
 969         .cra_exit               = nx842_crypto_exit,
 970         .cra_u                  = { .compress = {
 971         .coa_compress           = nx842_crypto_compress,
 972         .coa_decompress         = nx842_crypto_decompress } }
 973 };
 974
 975 static int nx842_probe(struct vio_dev *viodev,
 976                        const struct vio_device_id *id)
 977 {
 978         struct nx842_devdata *old_devdata, *new_devdata = NULL;
 979         unsigned long flags;
 980         int ret = 0;
 981
 982         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 983         if (!new_devdata)
 984                 return -ENOMEM;
 985
 986         new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
 987                         GFP_NOFS);
 988         if (!new_devdata->counters) {
 989                 kfree(new_devdata);
 990                 return -ENOMEM;
 991         }
 992
 993         spin_lock_irqsave(&devdata_mutex, flags);
 994         old_devdata = rcu_dereference_check(devdata,
 995                         lockdep_is_held(&devdata_mutex));
 996
 997         if (old_devdata && old_devdata->vdev != NULL) {
 998                 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
 999                 ret = -1;
1000                 goto error_unlock;
1001         }
1002
1003         dev_set_drvdata(&viodev->dev, NULL);
1004
1005         new_devdata->vdev = viodev;
1006         new_devdata->dev = &viodev->dev;
1007         nx842_OF_set_defaults(new_devdata);
1008
1009         rcu_assign_pointer(devdata, new_devdata);
1010         spin_unlock_irqrestore(&devdata_mutex, flags);
1011         synchronize_rcu();
1012         kfree(old_devdata);
1013
1014         of_reconfig_notifier_register(&nx842_of_nb);
1015
1016         ret = nx842_OF_upd(NULL);
1017         if (ret)
1018                 goto error;
1019
1020         ret = crypto_register_alg(&nx842_pseries_alg);
1021         if (ret) {
1022                 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1023                 goto error;
1024         }
1025
1026         rcu_read_lock();
1027         dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1028         rcu_read_unlock();
1029
1030         if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1031                 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1032                 ret = -1;
1033                 goto error;
1034         }
1035
1036         return 0;
1037
1038 error_unlock:
1039         spin_unlock_irqrestore(&devdata_mutex, flags);
1040         if (new_devdata)
1041                 kfree(new_devdata->counters);
1042         kfree(new_devdata);
1043 error:
1044         return ret;
1045 }
1046
1047 static int nx842_remove(struct vio_dev *viodev)
1048 {
1049         struct nx842_devdata *old_devdata;
1050         unsigned long flags;
1051
1052         pr_info("Removing IBM Power 842 compression device\n");
1053         sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1054
1055         crypto_unregister_alg(&nx842_pseries_alg);
1056
1057         spin_lock_irqsave(&devdata_mutex, flags);
1058         old_devdata = rcu_dereference_check(devdata,
1059                         lockdep_is_held(&devdata_mutex));
1060         of_reconfig_notifier_unregister(&nx842_of_nb);
1061         RCU_INIT_POINTER(devdata, NULL);
1062         spin_unlock_irqrestore(&devdata_mutex, flags);
1063         synchronize_rcu();
1064         dev_set_drvdata(&viodev->dev, NULL);
1065         if (old_devdata)
1066                 kfree(old_devdata->counters);
1067         kfree(old_devdata);
1068
1069         return 0;
1070 }
1071
1072 static const struct vio_device_id nx842_vio_driver_ids[] = {
1073         {"ibm,compression-v1", "ibm,compression"},
1074         {"", ""},
1075 };
1076 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1077
1078 static struct vio_driver nx842_vio_driver = {
1079         .name = KBUILD_MODNAME,
1080         .probe = nx842_probe,
1081         .remove = nx842_remove,
1082         .get_desired_dma = nx842_get_desired_dma,
1083         .id_table = nx842_vio_driver_ids,
1084 };
1085
1086 static int __init nx842_pseries_init(void)
1087 {
1088         struct nx842_devdata *new_devdata;
1089         int ret;
1090
1091         if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1092                 return -ENODEV;
1093
1094         RCU_INIT_POINTER(devdata, NULL);
1095         new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1096         if (!new_devdata)
1097                 return -ENOMEM;
1098
1099         RCU_INIT_POINTER(devdata, new_devdata);
1100
1101         ret = vio_register_driver(&nx842_vio_driver);
1102         if (ret) {
1103                 pr_err("Could not register VIO driver %d\n", ret);
1104
1105                 kfree(new_devdata);
1106                 return ret;
1107         }
1108
1109         return 0;
1110 }
1111
1112 module_init(nx842_pseries_init);
1113
1114 static void __exit nx842_pseries_exit(void)
1115 {
1116         struct nx842_devdata *old_devdata;
1117         unsigned long flags;
1118
1119         crypto_unregister_alg(&nx842_pseries_alg);
1120
1121         spin_lock_irqsave(&devdata_mutex, flags);
1122         old_devdata = rcu_dereference_check(devdata,
1123                         lockdep_is_held(&devdata_mutex));
1124         RCU_INIT_POINTER(devdata, NULL);
1125         spin_unlock_irqrestore(&devdata_mutex, flags);
1126         synchronize_rcu();
1127         if (old_devdata && old_devdata->dev)
1128                 dev_set_drvdata(old_devdata->dev, NULL);
1129         kfree(old_devdata);
1130         vio_unregister_driver(&nx842_vio_driver);
1131 }
1132
1133 module_exit(nx842_pseries_exit);
1134