drivers/crypto/nx/nx-842-pseries.c

   1 /*
   2  * Driver for IBM Power 842 compression accelerator
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2012
  19  *
  20  * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
  21  *          Seth Jennings <sjenning@linux.vnet.ibm.com>
  22  */
  23
  24 #include <asm/vio.h>
  25
  26 #include "nx-842.h"
  27 #include "nx_csbcpb.h" /* struct nx_csbcpb */
  28
  29 MODULE_LICENSE("GPL");
  30 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  31 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  32
  33 static struct nx842_constraints nx842_pseries_constraints = {
  34         .alignment =    DDE_BUFFER_ALIGN,
  35         .multiple =     DDE_BUFFER_LAST_MULT,
  36         .minimum =      DDE_BUFFER_LAST_MULT,
  37         .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  38 };
  39
  40 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  41 {
  42         if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  43                 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  44                          in ? "input" : "output", buf,
  45                          nx842_pseries_constraints.alignment);
  46                 return -EINVAL;
  47         }
  48         if (*len % nx842_pseries_constraints.multiple) {
  49                 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  50                          in ? "input" : "output", *len,
  51                          nx842_pseries_constraints.multiple);
  52                 if (in)
  53                         return -EINVAL;
  54                 *len = round_down(*len, nx842_pseries_constraints.multiple);
  55         }
  56         if (*len < nx842_pseries_constraints.minimum) {
  57                 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  58                          in ? "input" : "output", *len,
  59                          nx842_pseries_constraints.minimum);
  60                 return -EINVAL;
  61         }
  62         if (*len > nx842_pseries_constraints.maximum) {
  63                 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  64                          in ? "input" : "output", *len,
  65                          nx842_pseries_constraints.maximum);
  66                 if (in)
  67                         return -EINVAL;
  68                 *len = nx842_pseries_constraints.maximum;
  69         }
  70         return 0;
  71 }
  72
  73 /* I assume we need to align the CSB? */
  74 #define WORKMEM_ALIGN   (256)
  75
  76 struct nx842_workmem {
  77         /* scatterlist */
  78         char slin[4096];
  79         char slout[4096];
  80         /* coprocessor status/parameter block */
  81         struct nx_csbcpb csbcpb;
  82
  83         char padding[WORKMEM_ALIGN];
  84 } __aligned(WORKMEM_ALIGN);
  85
  86 /* Macros for fields within nx_csbcpb */
  87 /* Check the valid bit within the csbcpb valid field */
  88 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  89
  90 /* CE macros operate on the completion_extension field bits in the csbcpb.
  91  * CE0 0=full completion, 1=partial completion
  92  * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  93  * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  94 #define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  95 #define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  96 #define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  97
  98 /* The NX unit accepts data only on 4K page boundaries */
  99 #define NX842_HW_PAGE_SIZE      (4096)
 100 #define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
 101
 102 enum nx842_status {
 103         UNAVAILABLE,
 104         AVAILABLE
 105 };
 106
 107 struct ibm_nx842_counters {
 108         atomic64_t comp_complete;
 109         atomic64_t comp_failed;
 110         atomic64_t decomp_complete;
 111         atomic64_t decomp_failed;
 112         atomic64_t swdecomp;
 113         atomic64_t comp_times[32];
 114         atomic64_t decomp_times[32];
 115 };
 116
 117 static struct nx842_devdata {
 118         struct vio_dev *vdev;
 119         struct device *dev;
 120         struct ibm_nx842_counters *counters;
 121         unsigned int max_sg_len;
 122         unsigned int max_sync_size;
 123         unsigned int max_sync_sg;
 124         enum nx842_status status;
 125 } __rcu *devdata;
 126 static DEFINE_SPINLOCK(devdata_mutex);
 127
 128 #define NX842_COUNTER_INC(_x) \
 129 static inline void nx842_inc_##_x( \
 130         const struct nx842_devdata *dev) { \
 131         if (dev) \
 132                 atomic64_inc(&dev->counters->_x); \
 133 }
 134 NX842_COUNTER_INC(comp_complete);
 135 NX842_COUNTER_INC(comp_failed);
 136 NX842_COUNTER_INC(decomp_complete);
 137 NX842_COUNTER_INC(decomp_failed);
 138 NX842_COUNTER_INC(swdecomp);
 139
 140 #define NX842_HIST_SLOTS 16
 141
 142 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 143 {
 144         int bucket = fls(time);
 145
 146         if (bucket)
 147                 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 148
 149         atomic64_inc(&times[bucket]);
 150 }
 151
 152 /* NX unit operation flags */
 153 #define NX842_OP_COMPRESS       0x0
 154 #define NX842_OP_CRC            0x1
 155 #define NX842_OP_DECOMPRESS     0x2
 156 #define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 157 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 158 #define NX842_OP_ASYNC          (1<<23)
 159 #define NX842_OP_NOTIFY         (1<<22)
 160 #define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 161
 162 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 163 {
 164         /* No use of DMA mappings within the driver. */
 165         return 0;
 166 }
 167
 168 struct nx842_slentry {
 169         __be64 ptr; /* Real address (use __pa()) */
 170         __be64 len;
 171 };
 172
 173 /* pHyp scatterlist entry */
 174 struct nx842_scatterlist {
 175         int entry_nr; /* number of slentries */
 176         struct nx842_slentry *entries; /* ptr to array of slentries */
 177 };
 178
 179 /* Does not include sizeof(entry_nr) in the size */
 180 static inline unsigned long nx842_get_scatterlist_size(
 181                                 struct nx842_scatterlist *sl)
 182 {
 183         return sl->entry_nr * sizeof(struct nx842_slentry);
 184 }
 185
 186 static int nx842_build_scatterlist(unsigned long buf, int len,
 187                         struct nx842_scatterlist *sl)
 188 {
 189         unsigned long entrylen;
 190         struct nx842_slentry *entry;
 191
 192         sl->entry_nr = 0;
 193
 194         entry = sl->entries;
 195         while (len) {
 196                 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 197                 entrylen = min_t(int, len,
 198                                  LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 199                 entry->len = cpu_to_be64(entrylen);
 200
 201                 len -= entrylen;
 202                 buf += entrylen;
 203
 204                 sl->entry_nr++;
 205                 entry++;
 206         }
 207
 208         return 0;
 209 }
 210
 211 static int nx842_validate_result(struct device *dev,
 212         struct cop_status_block *csb)
 213 {
 214         /* The csb must be valid after returning from vio_h_cop_sync */
 215         if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 216                 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 217                                 __func__);
 218                 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 219                                 csb->valid,
 220                                 csb->crb_seq_number,
 221                                 csb->completion_code,
 222                                 csb->completion_extension);
 223                 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 224                                 be32_to_cpu(csb->processed_byte_count),
 225                                 (unsigned long)be64_to_cpu(csb->address));
 226                 return -EIO;
 227         }
 228
 229         /* Check return values from the hardware in the CSB */
 230         switch (csb->completion_code) {
 231         case 0: /* Completed without error */
 232                 break;
 233         case 64: /* Target bytes > Source bytes during compression */
 234         case 13: /* Output buffer too small */
 235                 dev_dbg(dev, "%s: Compression output larger than input\n",
 236                                         __func__);
 237                 return -ENOSPC;
 238         case 66: /* Input data contains an illegal template field */
 239         case 67: /* Template indicates data past the end of the input stream */
 240                 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 241                                         __func__, csb->completion_code);
 242                 return -EINVAL;
 243         default:
 244                 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 245                                         __func__, csb->completion_code);
 246                 return -EIO;
 247         }
 248
 249         /* Hardware sanity check */
 250         if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 251                 dev_err(dev, "%s: No error returned by hardware, but "
 252                                 "data returned is unusable, contact support.\n"
 253                                 "(Additional info: csbcbp->processed bytes "
 254                                 "does not specify processed bytes for the "
 255                                 "target buffer.)\n", __func__);
 256                 return -EIO;
 257         }
 258
 259         return 0;
 260 }
 261
 262 /**
 263  * nx842_pseries_compress - Compress data using the 842 algorithm
 264  *
 265  * Compression provide by the NX842 coprocessor on IBM Power systems.
 266  * The input buffer is compressed and the result is stored in the
 267  * provided output buffer.
 268  *
 269  * Upon return from this function @outlen contains the length of the
 270  * compressed data.  If there is an error then @outlen will be 0 and an
 271  * error will be specified by the return code from this function.
 272  *
 273  * @in: Pointer to input buffer
 274  * @inlen: Length of input buffer
 275  * @out: Pointer to output buffer
 276  * @outlen: Length of output buffer
 277  * @wrkmem: ptr to buffer for working memory, size determined by
 278  *          nx842_pseries_driver.workmem_size
 279  *
 280  * Returns:
 281  *   0          Success, output of length @outlen stored in the buffer at @out
 282  *   -ENOMEM    Unable to allocate internal buffers
 283  *   -ENOSPC    Output buffer is to small
 284  *   -EIO       Internal error
 285  *   -ENODEV    Hardware unavailable
 286  */
 287 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 288                                   unsigned char *out, unsigned int *outlen,
 289                                   void *wmem)
 290 {
 291         struct nx842_devdata *local_devdata;
 292         struct device *dev = NULL;
 293         struct nx842_workmem *workmem;
 294         struct nx842_scatterlist slin, slout;
 295         struct nx_csbcpb *csbcpb;
 296         int ret = 0, max_sync_size;
 297         unsigned long inbuf, outbuf;
 298         struct vio_pfo_op op = {
 299                 .done = NULL,
 300                 .handle = 0,
 301                 .timeout = 0,
 302         };
 303         unsigned long start = get_tb();
 304
 305         inbuf = (unsigned long)in;
 306         if (check_constraints(inbuf, &inlen, true))
 307                 return -EINVAL;
 308
 309         outbuf = (unsigned long)out;
 310         if (check_constraints(outbuf, outlen, false))
 311                 return -EINVAL;
 312
 313         rcu_read_lock();
 314         local_devdata = rcu_dereference(devdata);
 315         if (!local_devdata || !local_devdata->dev) {
 316                 rcu_read_unlock();
 317                 return -ENODEV;
 318         }
 319         max_sync_size = local_devdata->max_sync_size;
 320         dev = local_devdata->dev;
 321
 322         /* Init scatterlist */
 323         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 324         slin.entries = (struct nx842_slentry *)workmem->slin;
 325         slout.entries = (struct nx842_slentry *)workmem->slout;
 326
 327         /* Init operation */
 328         op.flags = NX842_OP_COMPRESS;
 329         csbcpb = &workmem->csbcpb;
 330         memset(csbcpb, 0, sizeof(*csbcpb));
 331         op.csbcpb = nx842_get_pa(csbcpb);
 332
 333         if ((inbuf & NX842_HW_PAGE_MASK) ==
 334             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 335                 /* Create direct DDE */
 336                 op.in = nx842_get_pa((void *)inbuf);
 337                 op.inlen = inlen;
 338         } else {
 339                 /* Create indirect DDE (scatterlist) */
 340                 nx842_build_scatterlist(inbuf, inlen, &slin);
 341                 op.in = nx842_get_pa(slin.entries);
 342                 op.inlen = -nx842_get_scatterlist_size(&slin);
 343         }
 344
 345         if ((outbuf & NX842_HW_PAGE_MASK) ==
 346             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 347                 /* Create direct DDE */
 348                 op.out = nx842_get_pa((void *)outbuf);
 349                 op.outlen = *outlen;
 350         } else {
 351                 /* Create indirect DDE (scatterlist) */
 352                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 353                 op.out = nx842_get_pa(slout.entries);
 354                 op.outlen = -nx842_get_scatterlist_size(&slout);
 355         }
 356
 357         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 358                 __func__, (unsigned long)op.in, (long)op.inlen,
 359                 (unsigned long)op.out, (long)op.outlen);
 360
 361         /* Send request to pHyp */
 362         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 363
 364         /* Check for pHyp error */
 365         if (ret) {
 366                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 367                         __func__, ret, op.hcall_err);
 368                 ret = -EIO;
 369                 goto unlock;
 370         }
 371
 372         /* Check for hardware error */
 373         ret = nx842_validate_result(dev, &csbcpb->csb);
 374         if (ret)
 375                 goto unlock;
 376
 377         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 378         dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 379
 380 unlock:
 381         if (ret)
 382                 nx842_inc_comp_failed(local_devdata);
 383         else {
 384                 nx842_inc_comp_complete(local_devdata);
 385                 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 386                         (get_tb() - start) / tb_ticks_per_usec);
 387         }
 388         rcu_read_unlock();
 389         return ret;
 390 }
 391
 392 /**
 393  * nx842_pseries_decompress - Decompress data using the 842 algorithm
 394  *
 395  * Decompression provide by the NX842 coprocessor on IBM Power systems.
 396  * The input buffer is decompressed and the result is stored in the
 397  * provided output buffer.  The size allocated to the output buffer is
 398  * provided by the caller of this function in @outlen.  Upon return from
 399  * this function @outlen contains the length of the decompressed data.
 400  * If there is an error then @outlen will be 0 and an error will be
 401  * specified by the return code from this function.
 402  *
 403  * @in: Pointer to input buffer
 404  * @inlen: Length of input buffer
 405  * @out: Pointer to output buffer
 406  * @outlen: Length of output buffer
 407  * @wrkmem: ptr to buffer for working memory, size determined by
 408  *          nx842_pseries_driver.workmem_size
 409  *
 410  * Returns:
 411  *   0          Success, output of length @outlen stored in the buffer at @out
 412  *   -ENODEV    Hardware decompression device is unavailable
 413  *   -ENOMEM    Unable to allocate internal buffers
 414  *   -ENOSPC    Output buffer is to small
 415  *   -EINVAL    Bad input data encountered when attempting decompress
 416  *   -EIO       Internal error
 417  */
 418 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 419                                     unsigned char *out, unsigned int *outlen,
 420                                     void *wmem)
 421 {
 422         struct nx842_devdata *local_devdata;
 423         struct device *dev = NULL;
 424         struct nx842_workmem *workmem;
 425         struct nx842_scatterlist slin, slout;
 426         struct nx_csbcpb *csbcpb;
 427         int ret = 0, max_sync_size;
 428         unsigned long inbuf, outbuf;
 429         struct vio_pfo_op op = {
 430                 .done = NULL,
 431                 .handle = 0,
 432                 .timeout = 0,
 433         };
 434         unsigned long start = get_tb();
 435
 436         /* Ensure page alignment and size */
 437         inbuf = (unsigned long)in;
 438         if (check_constraints(inbuf, &inlen, true))
 439                 return -EINVAL;
 440
 441         outbuf = (unsigned long)out;
 442         if (check_constraints(outbuf, outlen, false))
 443                 return -EINVAL;
 444
 445         rcu_read_lock();
 446         local_devdata = rcu_dereference(devdata);
 447         if (!local_devdata || !local_devdata->dev) {
 448                 rcu_read_unlock();
 449                 return -ENODEV;
 450         }
 451         max_sync_size = local_devdata->max_sync_size;
 452         dev = local_devdata->dev;
 453
 454         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 455
 456         /* Init scatterlist */
 457         slin.entries = (struct nx842_slentry *)workmem->slin;
 458         slout.entries = (struct nx842_slentry *)workmem->slout;
 459
 460         /* Init operation */
 461         op.flags = NX842_OP_DECOMPRESS;
 462         csbcpb = &workmem->csbcpb;
 463         memset(csbcpb, 0, sizeof(*csbcpb));
 464         op.csbcpb = nx842_get_pa(csbcpb);
 465
 466         if ((inbuf & NX842_HW_PAGE_MASK) ==
 467             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 468                 /* Create direct DDE */
 469                 op.in = nx842_get_pa((void *)inbuf);
 470                 op.inlen = inlen;
 471         } else {
 472                 /* Create indirect DDE (scatterlist) */
 473                 nx842_build_scatterlist(inbuf, inlen, &slin);
 474                 op.in = nx842_get_pa(slin.entries);
 475                 op.inlen = -nx842_get_scatterlist_size(&slin);
 476         }
 477
 478         if ((outbuf & NX842_HW_PAGE_MASK) ==
 479             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 480                 /* Create direct DDE */
 481                 op.out = nx842_get_pa((void *)outbuf);
 482                 op.outlen = *outlen;
 483         } else {
 484                 /* Create indirect DDE (scatterlist) */
 485                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 486                 op.out = nx842_get_pa(slout.entries);
 487                 op.outlen = -nx842_get_scatterlist_size(&slout);
 488         }
 489
 490         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 491                 __func__, (unsigned long)op.in, (long)op.inlen,
 492                 (unsigned long)op.out, (long)op.outlen);
 493
 494         /* Send request to pHyp */
 495         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 496
 497         /* Check for pHyp error */
 498         if (ret) {
 499                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 500                         __func__, ret, op.hcall_err);
 501                 goto unlock;
 502         }
 503
 504         /* Check for hardware error */
 505         ret = nx842_validate_result(dev, &csbcpb->csb);
 506         if (ret)
 507                 goto unlock;
 508
 509         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 510
 511 unlock:
 512         if (ret)
 513                 /* decompress fail */
 514                 nx842_inc_decomp_failed(local_devdata);
 515         else {
 516                 nx842_inc_decomp_complete(local_devdata);
 517                 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 518                         (get_tb() - start) / tb_ticks_per_usec);
 519         }
 520
 521         rcu_read_unlock();
 522         return ret;
 523 }
 524
 525 /**
 526  * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 527  *
 528  * @devdata - struct nx842_devdata to update
 529  *
 530  * Returns:
 531  *  0 on success
 532  *  -ENOENT if @devdata ptr is NULL
 533  */
 534 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 535 {
 536         if (devdata) {
 537                 devdata->max_sync_size = 0;
 538                 devdata->max_sync_sg = 0;
 539                 devdata->max_sg_len = 0;
 540                 devdata->status = UNAVAILABLE;
 541                 return 0;
 542         } else
 543                 return -ENOENT;
 544 }
 545
 546 /**
 547  * nx842_OF_upd_status -- Update the device info from OF status prop
 548  *
 549  * The status property indicates if the accelerator is enabled.  If the
 550  * device is in the OF tree it indicates that the hardware is present.
 551  * The status field indicates if the device is enabled when the status
 552  * is 'okay'.  Otherwise the device driver will be disabled.
 553  *
 554  * @devdata - struct nx842_devdata to update
 555  * @prop - struct property point containing the maxsyncop for the update
 556  *
 557  * Returns:
 558  *  0 - Device is available
 559  *  -EINVAL - Device is not available
 560  */
 561 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
 562                                         struct property *prop) {
 563         int ret = 0;
 564         const char *status = (const char *)prop->value;
 565
 566         if (!strncmp(status, "okay", (size_t)prop->length)) {
 567                 devdata->status = AVAILABLE;
 568         } else {
 569                 dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
 570                                 __func__, status);
 571                 devdata->status = UNAVAILABLE;
 572         }
 573
 574         return ret;
 575 }
 576
 577 /**
 578  * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 579  *
 580  * Definition of the 'ibm,max-sg-len' OF property:
 581  *  This field indicates the maximum byte length of a scatter list
 582  *  for the platform facility. It is a single cell encoded as with encode-int.
 583  *
 584  * Example:
 585  *  # od -x ibm,max-sg-len
 586  *  0000000 0000 0ff0
 587  *
 588  *  In this example, the maximum byte length of a scatter list is
 589  *  0x0ff0 (4,080).
 590  *
 591  * @devdata - struct nx842_devdata to update
 592  * @prop - struct property point containing the maxsyncop for the update
 593  *
 594  * Returns:
 595  *  0 on success
 596  *  -EINVAL on failure
 597  */
 598 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 599                                         struct property *prop) {
 600         int ret = 0;
 601         const unsigned int maxsglen = of_read_number(prop->value, 1);
 602
 603         if (prop->length != sizeof(maxsglen)) {
 604                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 605                 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 606                                 prop->length, sizeof(maxsglen));
 607                 ret = -EINVAL;
 608         } else {
 609                 devdata->max_sg_len = min_t(unsigned int,
 610                                             maxsglen, NX842_HW_PAGE_SIZE);
 611         }
 612
 613         return ret;
 614 }
 615
 616 /**
 617  * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 618  *
 619  * Definition of the 'ibm,max-sync-cop' OF property:
 620  *  Two series of cells.  The first series of cells represents the maximums
 621  *  that can be synchronously compressed. The second series of cells
 622  *  represents the maximums that can be synchronously decompressed.
 623  *  1. The first cell in each series contains the count of the number of
 624  *     data length, scatter list elements pairs that follow – each being
 625  *     of the form
 626  *    a. One cell data byte length
 627  *    b. One cell total number of scatter list elements
 628  *
 629  * Example:
 630  *  # od -x ibm,max-sync-cop
 631  *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 632  *  0000020 0000 1000 0000 01fe
 633  *
 634  *  In this example, compression supports 0x1000 (4,096) data byte length
 635  *  and 0x1fe (510) total scatter list elements.  Decompression supports
 636  *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 637  *  elements.
 638  *
 639  * @devdata - struct nx842_devdata to update
 640  * @prop - struct property point containing the maxsyncop for the update
 641  *
 642  * Returns:
 643  *  0 on success
 644  *  -EINVAL on failure
 645  */
 646 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 647                                         struct property *prop) {
 648         int ret = 0;
 649         unsigned int comp_data_limit, decomp_data_limit;
 650         unsigned int comp_sg_limit, decomp_sg_limit;
 651         const struct maxsynccop_t {
 652                 __be32 comp_elements;
 653                 __be32 comp_data_limit;
 654                 __be32 comp_sg_limit;
 655                 __be32 decomp_elements;
 656                 __be32 decomp_data_limit;
 657                 __be32 decomp_sg_limit;
 658         } *maxsynccop;
 659
 660         if (prop->length != sizeof(*maxsynccop)) {
 661                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 662                 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 663                                 sizeof(*maxsynccop));
 664                 ret = -EINVAL;
 665                 goto out;
 666         }
 667
 668         maxsynccop = (const struct maxsynccop_t *)prop->value;
 669         comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 670         comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 671         decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 672         decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 673
 674         /* Use one limit rather than separate limits for compression and
 675          * decompression. Set a maximum for this so as not to exceed the
 676          * size that the header can support and round the value down to
 677          * the hardware page size (4K) */
 678         devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 679
 680         devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 681                                         65536);
 682
 683         if (devdata->max_sync_size < 4096) {
 684                 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 685                                 "less than the driver minimum, unable to use "
 686                                 "the hardware device\n",
 687                                 __func__, devdata->max_sync_size);
 688                 ret = -EINVAL;
 689                 goto out;
 690         }
 691
 692         nx842_pseries_constraints.maximum = devdata->max_sync_size;
 693
 694         devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 695         if (devdata->max_sync_sg < 1) {
 696                 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 697                                 "less than the driver minimum, unable to use "
 698                                 "the hardware device\n",
 699                                 __func__, devdata->max_sync_sg);
 700                 ret = -EINVAL;
 701                 goto out;
 702         }
 703
 704 out:
 705         return ret;
 706 }
 707
 708 /**
 709  *
 710  * nx842_OF_upd -- Handle OF properties updates for the device.
 711  *
 712  * Set all properties from the OF tree.  Optionally, a new property
 713  * can be provided by the @new_prop pointer to overwrite an existing value.
 714  * The device will remain disabled until all values are valid, this function
 715  * will return an error for updates unless all values are valid.
 716  *
 717  * @new_prop: If not NULL, this property is being updated.  If NULL, update
 718  *  all properties from the current values in the OF tree.
 719  *
 720  * Returns:
 721  *  0 - Success
 722  *  -ENOMEM - Could not allocate memory for new devdata structure
 723  *  -EINVAL - property value not found, new_prop is not a recognized
 724  *      property for the device or property value is not valid.
 725  *  -ENODEV - Device is not available
 726  */
 727 static int nx842_OF_upd(struct property *new_prop)
 728 {
 729         struct nx842_devdata *old_devdata = NULL;
 730         struct nx842_devdata *new_devdata = NULL;
 731         struct device_node *of_node = NULL;
 732         struct property *status = NULL;
 733         struct property *maxsglen = NULL;
 734         struct property *maxsyncop = NULL;
 735         int ret = 0;
 736         unsigned long flags;
 737
 738         spin_lock_irqsave(&devdata_mutex, flags);
 739         old_devdata = rcu_dereference_check(devdata,
 740                         lockdep_is_held(&devdata_mutex));
 741         if (old_devdata)
 742                 of_node = old_devdata->dev->of_node;
 743
 744         if (!old_devdata || !of_node) {
 745                 pr_err("%s: device is not available\n", __func__);
 746                 spin_unlock_irqrestore(&devdata_mutex, flags);
 747                 return -ENODEV;
 748         }
 749
 750         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 751         if (!new_devdata) {
 752                 dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
 753                 ret = -ENOMEM;
 754                 goto error_out;
 755         }
 756
 757         memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 758         new_devdata->counters = old_devdata->counters;
 759
 760         /* Set ptrs for existing properties */
 761         status = of_find_property(of_node, "status", NULL);
 762         maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 763         maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 764         if (!status || !maxsglen || !maxsyncop) {
 765                 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 766                 ret = -EINVAL;
 767                 goto error_out;
 768         }
 769
 770         /*
 771          * If this is a property update, there are only certain properties that
 772          * we care about. Bail if it isn't in the below list
 773          */
 774         if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 775                          strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 776                          strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 777                 goto out;
 778
 779         /* Perform property updates */
 780         ret = nx842_OF_upd_status(new_devdata, status);
 781         if (ret)
 782                 goto error_out;
 783
 784         ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 785         if (ret)
 786                 goto error_out;
 787
 788         ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 789         if (ret)
 790                 goto error_out;
 791
 792 out:
 793         dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 794                         __func__, new_devdata->max_sync_size,
 795                         old_devdata->max_sync_size);
 796         dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 797                         __func__, new_devdata->max_sync_sg,
 798                         old_devdata->max_sync_sg);
 799         dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 800                         __func__, new_devdata->max_sg_len,
 801                         old_devdata->max_sg_len);
 802
 803         rcu_assign_pointer(devdata, new_devdata);
 804         spin_unlock_irqrestore(&devdata_mutex, flags);
 805         synchronize_rcu();
 806         dev_set_drvdata(new_devdata->dev, new_devdata);
 807         kfree(old_devdata);
 808         return 0;
 809
 810 error_out:
 811         if (new_devdata) {
 812                 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 813                 nx842_OF_set_defaults(new_devdata);
 814                 rcu_assign_pointer(devdata, new_devdata);
 815                 spin_unlock_irqrestore(&devdata_mutex, flags);
 816                 synchronize_rcu();
 817                 dev_set_drvdata(new_devdata->dev, new_devdata);
 818                 kfree(old_devdata);
 819         } else {
 820                 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 821                 spin_unlock_irqrestore(&devdata_mutex, flags);
 822         }
 823
 824         if (!ret)
 825                 ret = -EINVAL;
 826         return ret;
 827 }
 828
 829 /**
 830  * nx842_OF_notifier - Process updates to OF properties for the device
 831  *
 832  * @np: notifier block
 833  * @action: notifier action
 834  * @update: struct pSeries_reconfig_prop_update pointer if action is
 835  *      PSERIES_UPDATE_PROPERTY
 836  *
 837  * Returns:
 838  *      NOTIFY_OK on success
 839  *      NOTIFY_BAD encoded with error number on failure, use
 840  *              notifier_to_errno() to decode this value
 841  */
 842 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 843                              void *data)
 844 {
 845         struct of_reconfig_data *upd = data;
 846         struct nx842_devdata *local_devdata;
 847         struct device_node *node = NULL;
 848
 849         rcu_read_lock();
 850         local_devdata = rcu_dereference(devdata);
 851         if (local_devdata)
 852                 node = local_devdata->dev->of_node;
 853
 854         if (local_devdata &&
 855                         action == OF_RECONFIG_UPDATE_PROPERTY &&
 856                         !strcmp(upd->dn->name, node->name)) {
 857                 rcu_read_unlock();
 858                 nx842_OF_upd(upd->prop);
 859         } else
 860                 rcu_read_unlock();
 861
 862         return NOTIFY_OK;
 863 }
 864
 865 static struct notifier_block nx842_of_nb = {
 866         .notifier_call = nx842_OF_notifier,
 867 };
 868
 869 #define nx842_counter_read(_name)                                       \
 870 static ssize_t nx842_##_name##_show(struct device *dev,         \
 871                 struct device_attribute *attr,                          \
 872                 char *buf) {                                            \
 873         struct nx842_devdata *local_devdata;                    \
 874         int p = 0;                                                      \
 875         rcu_read_lock();                                                \
 876         local_devdata = rcu_dereference(devdata);                       \
 877         if (local_devdata)                                              \
 878                 p = snprintf(buf, PAGE_SIZE, "%ld\n",                   \
 879                        atomic64_read(&local_devdata->counters->_name)); \
 880         rcu_read_unlock();                                              \
 881         return p;                                                       \
 882 }
 883
 884 #define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 885         nx842_counter_read(_name);                                      \
 886         static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 887                                                 0444,                   \
 888                                                 nx842_##_name##_show,\
 889                                                 NULL);
 890
 891 NX842DEV_COUNTER_ATTR_RO(comp_complete);
 892 NX842DEV_COUNTER_ATTR_RO(comp_failed);
 893 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 894 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 895 NX842DEV_COUNTER_ATTR_RO(swdecomp);
 896
 897 static ssize_t nx842_timehist_show(struct device *,
 898                 struct device_attribute *, char *);
 899
 900 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 901                 nx842_timehist_show, NULL);
 902 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 903                 0444, nx842_timehist_show, NULL);
 904
 905 static ssize_t nx842_timehist_show(struct device *dev,
 906                 struct device_attribute *attr, char *buf) {
 907         char *p = buf;
 908         struct nx842_devdata *local_devdata;
 909         atomic64_t *times;
 910         int bytes_remain = PAGE_SIZE;
 911         int bytes;
 912         int i;
 913
 914         rcu_read_lock();
 915         local_devdata = rcu_dereference(devdata);
 916         if (!local_devdata) {
 917                 rcu_read_unlock();
 918                 return 0;
 919         }
 920
 921         if (attr == &dev_attr_comp_times)
 922                 times = local_devdata->counters->comp_times;
 923         else if (attr == &dev_attr_decomp_times)
 924                 times = local_devdata->counters->decomp_times;
 925         else {
 926                 rcu_read_unlock();
 927                 return 0;
 928         }
 929
 930         for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 931                 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
 932                                i ? (2<<(i-1)) : 0, (2<<i)-1,
 933                                atomic64_read(&times[i]));
 934                 bytes_remain -= bytes;
 935                 p += bytes;
 936         }
 937         /* The last bucket holds everything over
 938          * 2<<(NX842_HIST_SLOTS - 2) us */
 939         bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
 940                         2<<(NX842_HIST_SLOTS - 2),
 941                         atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 942         p += bytes;
 943
 944         rcu_read_unlock();
 945         return p - buf;
 946 }
 947
 948 static struct attribute *nx842_sysfs_entries[] = {
 949         &dev_attr_comp_complete.attr,
 950         &dev_attr_comp_failed.attr,
 951         &dev_attr_decomp_complete.attr,
 952         &dev_attr_decomp_failed.attr,
 953         &dev_attr_swdecomp.attr,
 954         &dev_attr_comp_times.attr,
 955         &dev_attr_decomp_times.attr,
 956         NULL,
 957 };
 958
 959 static struct attribute_group nx842_attribute_group = {
 960         .name = NULL,           /* put in device directory */
 961         .attrs = nx842_sysfs_entries,
 962 };
 963
 964 static struct nx842_driver nx842_pseries_driver = {
 965         .name =         KBUILD_MODNAME,
 966         .owner =        THIS_MODULE,
 967         .workmem_size = sizeof(struct nx842_workmem),
 968         .constraints =  &nx842_pseries_constraints,
 969         .compress =     nx842_pseries_compress,
 970         .decompress =   nx842_pseries_decompress,
 971 };
 972
 973 static int __init nx842_probe(struct vio_dev *viodev,
 974                                   const struct vio_device_id *id)
 975 {
 976         struct nx842_devdata *old_devdata, *new_devdata = NULL;
 977         unsigned long flags;
 978         int ret = 0;
 979
 980         spin_lock_irqsave(&devdata_mutex, flags);
 981         old_devdata = rcu_dereference_check(devdata,
 982                         lockdep_is_held(&devdata_mutex));
 983
 984         if (old_devdata && old_devdata->vdev != NULL) {
 985                 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
 986                 ret = -1;
 987                 goto error_unlock;
 988         }
 989
 990         dev_set_drvdata(&viodev->dev, NULL);
 991
 992         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 993         if (!new_devdata) {
 994                 dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
 995                 ret = -ENOMEM;
 996                 goto error_unlock;
 997         }
 998
 999         new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1000                         GFP_NOFS);
1001         if (!new_devdata->counters) {
1002                 dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
1003                 ret = -ENOMEM;
1004                 goto error_unlock;
1005         }
1006
1007         new_devdata->vdev = viodev;
1008         new_devdata->dev = &viodev->dev;
1009         nx842_OF_set_defaults(new_devdata);
1010
1011         rcu_assign_pointer(devdata, new_devdata);
1012         spin_unlock_irqrestore(&devdata_mutex, flags);
1013         synchronize_rcu();
1014         kfree(old_devdata);
1015
1016         of_reconfig_notifier_register(&nx842_of_nb);
1017
1018         ret = nx842_OF_upd(NULL);
1019         if (ret && ret != -ENODEV) {
1020                 dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
1021                 ret = -1;
1022                 goto error;
1023         }
1024
1025         rcu_read_lock();
1026         dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1027         rcu_read_unlock();
1028
1029         if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1030                 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1031                 ret = -1;
1032                 goto error;
1033         }
1034
1035         return 0;
1036
1037 error_unlock:
1038         spin_unlock_irqrestore(&devdata_mutex, flags);
1039         if (new_devdata)
1040                 kfree(new_devdata->counters);
1041         kfree(new_devdata);
1042 error:
1043         return ret;
1044 }
1045
1046 static int __exit nx842_remove(struct vio_dev *viodev)
1047 {
1048         struct nx842_devdata *old_devdata;
1049         unsigned long flags;
1050
1051         pr_info("Removing IBM Power 842 compression device\n");
1052         sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1053
1054         spin_lock_irqsave(&devdata_mutex, flags);
1055         old_devdata = rcu_dereference_check(devdata,
1056                         lockdep_is_held(&devdata_mutex));
1057         of_reconfig_notifier_unregister(&nx842_of_nb);
1058         RCU_INIT_POINTER(devdata, NULL);
1059         spin_unlock_irqrestore(&devdata_mutex, flags);
1060         synchronize_rcu();
1061         dev_set_drvdata(&viodev->dev, NULL);
1062         if (old_devdata)
1063                 kfree(old_devdata->counters);
1064         kfree(old_devdata);
1065
1066         return 0;
1067 }
1068
1069 static struct vio_device_id nx842_vio_driver_ids[] = {
1070         {"ibm,compression-v1", "ibm,compression"},
1071         {"", ""},
1072 };
1073
1074 static struct vio_driver nx842_vio_driver = {
1075         .name = KBUILD_MODNAME,
1076         .probe = nx842_probe,
1077         .remove = __exit_p(nx842_remove),
1078         .get_desired_dma = nx842_get_desired_dma,
1079         .id_table = nx842_vio_driver_ids,
1080 };
1081
1082 static int __init nx842_init(void)
1083 {
1084         struct nx842_devdata *new_devdata;
1085         int ret;
1086
1087         pr_info("Registering IBM Power 842 compression driver\n");
1088
1089         if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1090                 return -ENODEV;
1091
1092         RCU_INIT_POINTER(devdata, NULL);
1093         new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1094         if (!new_devdata) {
1095                 pr_err("Could not allocate memory for device data\n");
1096                 return -ENOMEM;
1097         }
1098         new_devdata->status = UNAVAILABLE;
1099         RCU_INIT_POINTER(devdata, new_devdata);
1100
1101         ret = vio_register_driver(&nx842_vio_driver);
1102         if (ret) {
1103                 pr_err("Could not register VIO driver %d\n", ret);
1104
1105                 kfree(new_devdata);
1106                 return ret;
1107         }
1108
1109         if (!nx842_platform_driver_set(&nx842_pseries_driver)) {
1110                 vio_unregister_driver(&nx842_vio_driver);
1111                 kfree(new_devdata);
1112                 return -EEXIST;
1113         }
1114
1115         return 0;
1116 }
1117
1118 module_init(nx842_init);
1119
1120 static void __exit nx842_exit(void)
1121 {
1122         struct nx842_devdata *old_devdata;
1123         unsigned long flags;
1124
1125         pr_info("Exiting IBM Power 842 compression driver\n");
1126         nx842_platform_driver_unset(&nx842_pseries_driver);
1127         spin_lock_irqsave(&devdata_mutex, flags);
1128         old_devdata = rcu_dereference_check(devdata,
1129                         lockdep_is_held(&devdata_mutex));
1130         RCU_INIT_POINTER(devdata, NULL);
1131         spin_unlock_irqrestore(&devdata_mutex, flags);
1132         synchronize_rcu();
1133         if (old_devdata && old_devdata->dev)
1134                 dev_set_drvdata(old_devdata->dev, NULL);
1135         kfree(old_devdata);
1136         vio_unregister_driver(&nx842_vio_driver);
1137 }
1138
1139 module_exit(nx842_exit);
1140