2 * Driver for IBM PowerNV 842 compression accelerator
4 * Copyright (C) 2015 Dan Streetman, IBM Corp
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 #include <linux/timer.h>
24 #include <asm/icswx.h>
26 MODULE_LICENSE("GPL");
27 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
28 MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
30 #define WORKMEM_ALIGN (CRB_ALIGN)
31 #define CSB_WAIT_MAX (5000) /* ms */
33 struct nx842_workmem {
34 /* Below fields must be properly aligned */
35 struct coprocessor_request_block crb; /* CRB_ALIGN align */
36 struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
37 struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
38 /* Above fields must be properly aligned */
42 char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
43 } __packed __aligned(WORKMEM_ALIGN);
49 struct list_head list;
52 /* no cpu hotplug on powernv, so this list never changes after init */
53 static LIST_HEAD(nx842_coprocs);
54 static unsigned int nx842_ct;
57 * setup_indirect_dde - Setup an indirect DDE
59 * The DDE is setup with the the DDE count, byte count, and address of
60 * first direct DDE in the list.
62 static void setup_indirect_dde(struct data_descriptor_entry *dde,
63 struct data_descriptor_entry *ddl,
64 unsigned int dde_count, unsigned int byte_count)
67 dde->count = dde_count;
69 dde->length = cpu_to_be32(byte_count);
70 dde->address = cpu_to_be64(nx842_get_pa(ddl));
74 * setup_direct_dde - Setup single DDE from buffer
76 * The DDE is setup with the buffer and length. The buffer must be properly
77 * aligned. The used length is returned.
79 * N Successfully set up DDE with N bytes
81 static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
82 unsigned long pa, unsigned int len)
84 unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
89 dde->length = cpu_to_be32(l);
90 dde->address = cpu_to_be64(pa);
96 * setup_ddl - Setup DDL from buffer
99 * 0 Successfully set up DDL
101 static int setup_ddl(struct data_descriptor_entry *dde,
102 struct data_descriptor_entry *ddl,
103 unsigned char *buf, unsigned int len,
106 unsigned long pa = nx842_get_pa(buf);
107 int i, ret, total_len = len;
109 if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
110 pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
111 in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
115 /* only need to check last mult; since buffer must be
116 * DDE_BUFFER_ALIGN aligned, and that is a multiple of
117 * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
118 * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
120 if (len % DDE_BUFFER_LAST_MULT) {
121 pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
122 in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
125 len = round_down(len, DDE_BUFFER_LAST_MULT);
128 /* use a single direct DDE */
129 if (len <= LEN_ON_PAGE(pa)) {
130 ret = setup_direct_dde(dde, pa, len);
136 for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
137 ret = setup_direct_dde(&ddl[i], pa, len);
140 pa = nx842_get_pa(buf);
144 pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
145 total_len, in ? "input" : "output", len);
150 setup_indirect_dde(dde, ddl, i, total_len);
155 #define CSB_ERR(csb, msg, ...) \
156 pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
157 ##__VA_ARGS__, (csb)->flags, \
158 (csb)->cs, (csb)->cc, (csb)->ce, \
159 be32_to_cpu((csb)->count))
161 #define CSB_ERR_ADDR(csb, msg, ...) \
162 CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
163 (unsigned long)be64_to_cpu((csb)->address))
168 static int wait_for_csb(struct nx842_workmem *wmem,
169 struct coprocessor_status_block *csb)
171 ktime_t start = wmem->start, now = ktime_get();
172 ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
174 while (!(ACCESS_ONCE(csb->flags) & CSB_V)) {
177 if (ktime_after(now, timeout))
181 /* hw has updated csb and output buffer */
184 /* check CSB flags */
185 if (!(csb->flags & CSB_V)) {
186 CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
187 (long)ktime_us_delta(now, start));
190 if (csb->flags & CSB_F) {
191 CSB_ERR(csb, "Invalid CSB format");
194 if (csb->flags & CSB_CH) {
195 CSB_ERR(csb, "Invalid CSB chaining state");
199 /* verify CSB completion sequence is 0 */
201 CSB_ERR(csb, "Invalid CSB completion sequence");
205 /* check CSB Completion Code */
210 case CSB_CC_TPBC_GT_SPBC:
211 /* not an error, but the compressed data is
212 * larger than the uncompressed data :(
216 /* input data errors */
217 case CSB_CC_OPERAND_OVERLAP:
218 /* input and output buffers overlap */
219 CSB_ERR(csb, "Operand Overlap error");
221 case CSB_CC_INVALID_OPERAND:
222 CSB_ERR(csb, "Invalid operand");
225 /* output buffer too small */
228 CSB_ERR(csb, "Function aborted");
230 case CSB_CC_CRC_MISMATCH:
231 CSB_ERR(csb, "CRC mismatch");
233 case CSB_CC_TEMPL_INVALID:
234 CSB_ERR(csb, "Compressed data template invalid");
236 case CSB_CC_TEMPL_OVERFLOW:
237 CSB_ERR(csb, "Compressed data template shows data past end");
240 /* these should not happen */
241 case CSB_CC_INVALID_ALIGN:
242 /* setup_ddl should have detected this */
243 CSB_ERR_ADDR(csb, "Invalid alignment");
245 case CSB_CC_DATA_LENGTH:
246 /* setup_ddl should have detected this */
247 CSB_ERR(csb, "Invalid data length");
249 case CSB_CC_WR_TRANSLATION:
250 case CSB_CC_TRANSLATION:
251 case CSB_CC_TRANSLATION_DUP1:
252 case CSB_CC_TRANSLATION_DUP2:
253 case CSB_CC_TRANSLATION_DUP3:
254 case CSB_CC_TRANSLATION_DUP4:
255 case CSB_CC_TRANSLATION_DUP5:
256 case CSB_CC_TRANSLATION_DUP6:
257 /* should not happen, we use physical addrs */
258 CSB_ERR_ADDR(csb, "Translation error");
260 case CSB_CC_WR_PROTECTION:
261 case CSB_CC_PROTECTION:
262 case CSB_CC_PROTECTION_DUP1:
263 case CSB_CC_PROTECTION_DUP2:
264 case CSB_CC_PROTECTION_DUP3:
265 case CSB_CC_PROTECTION_DUP4:
266 case CSB_CC_PROTECTION_DUP5:
267 case CSB_CC_PROTECTION_DUP6:
268 /* should not happen, we use physical addrs */
269 CSB_ERR_ADDR(csb, "Protection error");
271 case CSB_CC_PRIVILEGE:
272 /* shouldn't happen, we're in HYP mode */
273 CSB_ERR(csb, "Insufficient Privilege error");
275 case CSB_CC_EXCESSIVE_DDE:
276 /* shouldn't happen, setup_ddl doesn't use many dde's */
277 CSB_ERR(csb, "Too many DDEs in DDL");
279 case CSB_CC_TRANSPORT:
280 /* shouldn't happen, we setup CRB correctly */
281 CSB_ERR(csb, "Invalid CRB");
283 case CSB_CC_SEGMENTED_DDL:
284 /* shouldn't happen, setup_ddl creates DDL right */
285 CSB_ERR(csb, "Segmented DDL error");
287 case CSB_CC_DDE_OVERFLOW:
288 /* shouldn't happen, setup_ddl creates DDL right */
289 CSB_ERR(csb, "DDE overflow error");
292 /* should not happen with ICSWX */
293 CSB_ERR(csb, "Session violation error");
296 /* should not happen, we don't use chained CRBs */
297 CSB_ERR(csb, "Chained CRB error");
299 case CSB_CC_SEQUENCE:
300 /* should not happen, we don't use chained CRBs */
301 CSB_ERR(csb, "CRB seqeunce number error");
303 case CSB_CC_UNKNOWN_CODE:
304 CSB_ERR(csb, "Unknown subfunction code");
307 /* hardware errors */
308 case CSB_CC_RD_EXTERNAL:
309 case CSB_CC_RD_EXTERNAL_DUP1:
310 case CSB_CC_RD_EXTERNAL_DUP2:
311 case CSB_CC_RD_EXTERNAL_DUP3:
312 CSB_ERR_ADDR(csb, "Read error outside coprocessor");
314 case CSB_CC_WR_EXTERNAL:
315 CSB_ERR_ADDR(csb, "Write error outside coprocessor");
317 case CSB_CC_INTERNAL:
318 CSB_ERR(csb, "Internal error in coprocessor");
320 case CSB_CC_PROVISION:
321 CSB_ERR(csb, "Storage provision error");
324 CSB_ERR(csb, "Correctable hardware error");
328 CSB_ERR(csb, "Invalid CC %d", csb->cc);
332 /* check Completion Extension state */
333 if (csb->ce & CSB_CE_TERMINATION) {
334 CSB_ERR(csb, "CSB request was terminated");
337 if (csb->ce & CSB_CE_INCOMPLETE) {
338 CSB_ERR(csb, "CSB request not complete");
341 if (!(csb->ce & CSB_CE_TPBC)) {
342 CSB_ERR(csb, "TPBC not provided, unknown target length");
346 /* successful completion */
347 pr_debug_ratelimited("Processed %u bytes in %lu us\n", csb->count,
348 (unsigned long)ktime_us_delta(now, start));
354 * nx842_powernv_function - compress/decompress data using the 842 algorithm
356 * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
357 * This compresses or decompresses the provided input buffer into the provided
360 * Upon return from this function @outlen contains the length of the
361 * output data. If there is an error then @outlen will be 0 and an
362 * error will be specified by the return code from this function.
364 * The @workmem buffer should only be used by one function call at a time.
366 * @in: input buffer pointer
367 * @inlen: input buffer size
368 * @out: output buffer pointer
369 * @outlenp: output buffer size pointer
370 * @workmem: working memory buffer pointer, size determined by
371 * nx842_powernv_driver.workmem_size
372 * @fc: function code, see CCW Function Codes in nx-842.h
375 * 0 Success, output of length @outlenp stored in the buffer at @out
376 * -ENODEV Hardware unavailable
377 * -ENOSPC Output buffer is to small
378 * -EMSGSIZE Input buffer too large
379 * -EINVAL buffer constraints do not fix nx842_constraints
380 * -EPROTO hardware error during operation
381 * -ETIMEDOUT hardware did not complete operation in reasonable time
382 * -EINTR operation was aborted
384 static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
385 unsigned char *out, unsigned int *outlenp,
386 void *workmem, int fc)
388 struct coprocessor_request_block *crb;
389 struct coprocessor_status_block *csb;
390 struct nx842_workmem *wmem;
394 unsigned int outlen = *outlenp;
396 wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
400 /* shoudn't happen, we don't load without a coproc */
402 pr_err_ratelimited("coprocessor CT is 0");
409 /* Clear any previous values */
410 memset(crb, 0, sizeof(*crb));
413 ret = setup_ddl(&crb->source, wmem->ddl_in,
414 (unsigned char *)in, inlen, true);
417 ret = setup_ddl(&crb->target, wmem->ddl_out,
424 ccw = SET_FIELD(ccw, CCW_CT, nx842_ct);
425 ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */
426 ccw = SET_FIELD(ccw, CCW_FC_842, fc);
428 /* set up CRB's CSB addr */
429 csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
430 csb_addr |= CRB_CSB_AT; /* Addrs are phys */
431 crb->csb_addr = cpu_to_be64(csb_addr);
433 wmem->start = ktime_get();
436 ret = icswx(cpu_to_be32(ccw), crb);
438 pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
440 (unsigned int)be32_to_cpu(crb->ccw));
443 case ICSWX_INITIATED:
444 ret = wait_for_csb(wmem, csb);
447 pr_debug_ratelimited("842 Coprocessor busy\n");
451 pr_err_ratelimited("ICSWX rejected\n");
455 pr_err_ratelimited("Invalid ICSWX return code %x\n", ret);
461 *outlenp = be32_to_cpu(csb->count);
467 * nx842_powernv_compress - Compress data using the 842 algorithm
469 * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
470 * The input buffer is compressed and the result is stored in the
471 * provided output buffer.
473 * Upon return from this function @outlen contains the length of the
474 * compressed data. If there is an error then @outlen will be 0 and an
475 * error will be specified by the return code from this function.
477 * @in: input buffer pointer
478 * @inlen: input buffer size
479 * @out: output buffer pointer
480 * @outlenp: output buffer size pointer
481 * @workmem: working memory buffer pointer, size determined by
482 * nx842_powernv_driver.workmem_size
484 * Returns: see @nx842_powernv_function()
486 static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
487 unsigned char *out, unsigned int *outlenp,
490 return nx842_powernv_function(in, inlen, out, outlenp,
491 wmem, CCW_FC_842_COMP_NOCRC);
495 * nx842_powernv_decompress - Decompress data using the 842 algorithm
497 * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
498 * The input buffer is decompressed and the result is stored in the
499 * provided output buffer.
501 * Upon return from this function @outlen contains the length of the
502 * decompressed data. If there is an error then @outlen will be 0 and an
503 * error will be specified by the return code from this function.
505 * @in: input buffer pointer
506 * @inlen: input buffer size
507 * @out: output buffer pointer
508 * @outlenp: output buffer size pointer
509 * @workmem: working memory buffer pointer, size determined by
510 * nx842_powernv_driver.workmem_size
512 * Returns: see @nx842_powernv_function()
514 static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
515 unsigned char *out, unsigned int *outlenp,
518 return nx842_powernv_function(in, inlen, out, outlenp,
519 wmem, CCW_FC_842_DECOMP_NOCRC);
522 static int __init nx842_powernv_probe(struct device_node *dn)
524 struct nx842_coproc *coproc;
525 struct property *ct_prop, *ci_prop;
529 chip_id = of_get_ibm_chip_id(dn);
531 pr_err("ibm,chip-id missing\n");
534 ct_prop = of_find_property(dn, "ibm,842-coprocessor-type", NULL);
536 pr_err("ibm,842-coprocessor-type missing\n");
539 ct = be32_to_cpu(*(unsigned int *)ct_prop->value);
540 ci_prop = of_find_property(dn, "ibm,842-coprocessor-instance", NULL);
542 pr_err("ibm,842-coprocessor-instance missing\n");
545 ci = be32_to_cpu(*(unsigned int *)ci_prop->value);
547 coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
551 coproc->chip_id = chip_id;
554 INIT_LIST_HEAD(&coproc->list);
555 list_add(&coproc->list, &nx842_coprocs);
557 pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
561 else if (nx842_ct != ct)
562 pr_err("NX842 chip %d, CT %d != first found CT %d\n",
563 chip_id, ct, nx842_ct);
568 static struct nx842_constraints nx842_powernv_constraints = {
569 .alignment = DDE_BUFFER_ALIGN,
570 .multiple = DDE_BUFFER_LAST_MULT,
571 .minimum = DDE_BUFFER_LAST_MULT,
572 .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
575 static struct nx842_driver nx842_powernv_driver = {
576 .name = KBUILD_MODNAME,
577 .owner = THIS_MODULE,
578 .workmem_size = sizeof(struct nx842_workmem),
579 .constraints = &nx842_powernv_constraints,
580 .compress = nx842_powernv_compress,
581 .decompress = nx842_powernv_decompress,
584 static __init int nx842_powernv_init(void)
586 struct device_node *dn;
588 /* verify workmem size/align restrictions */
589 BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
590 BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
591 BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
592 /* verify buffer size/align restrictions */
593 BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
594 BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
595 BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
597 pr_info("loading\n");
599 for_each_compatible_node(dn, NULL, "ibm,power-nx")
600 nx842_powernv_probe(dn);
603 pr_err("no coprocessors found\n");
607 if (!nx842_platform_driver_set(&nx842_powernv_driver)) {
608 struct nx842_coproc *coproc, *n;
610 list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
611 list_del(&coproc->list);
622 module_init(nx842_powernv_init);
624 static void __exit nx842_powernv_exit(void)
626 struct nx842_coproc *coproc, *n;
628 nx842_platform_driver_unset(&nx842_powernv_driver);
630 list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
631 list_del(&coproc->list);
635 pr_info("unloaded\n");
637 module_exit(nx842_powernv_exit);