mm/zswap.c

   1 /*
   2  * zswap.c - zswap driver file
   3  *
   4  * zswap is a backend for frontswap that takes pages that are in the process
   5  * of being swapped out and attempts to compress and store them in a
   6  * RAM-based memory pool.  This can result in a significant I/O reduction on
   7  * the swap device and, in the case where decompressing from RAM is faster
   8  * than reading from the swap device, can also improve workload performance.
   9  *
  10  * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20  * GNU General Public License for more details.
  21 */
  22
  23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25 #include <linux/module.h>
  26 #include <linux/cpu.h>
  27 #include <linux/highmem.h>
  28 #include <linux/slab.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/types.h>
  31 #include <linux/atomic.h>
  32 #include <linux/frontswap.h>
  33 #include <linux/rbtree.h>
  34 #include <linux/swap.h>
  35 #include <linux/crypto.h>
  36 #include <linux/mempool.h>
  37 #include <linux/zpool.h>
  38
  39 #include <linux/mm_types.h>
  40 #include <linux/page-flags.h>
  41 #include <linux/swapops.h>
  42 #include <linux/writeback.h>
  43 #include <linux/pagemap.h>
  44
  45 /*********************************
  46 * statistics
  47 **********************************/
  48 /* Total bytes used by the compressed storage */
  49 static u64 zswap_pool_total_size;
  50 /* The number of compressed pages currently stored in zswap */
  51 static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
  52
  53 /*
  54  * The statistics below are not protected from concurrent access for
  55  * performance reasons so they may not be a 100% accurate.  However,
  56  * they do provide useful information on roughly how many times a
  57  * certain event is occurring.
  58 */
  59
  60 /* Pool limit was hit (see zswap_max_pool_percent) */
  61 static u64 zswap_pool_limit_hit;
  62 /* Pages written back when pool limit was reached */
  63 static u64 zswap_written_back_pages;
  64 /* Store failed due to a reclaim failure after pool limit was reached */
  65 static u64 zswap_reject_reclaim_fail;
  66 /* Compressed page was too big for the allocator to (optimally) store */
  67 static u64 zswap_reject_compress_poor;
  68 /* Store failed because underlying allocator could not get memory */
  69 static u64 zswap_reject_alloc_fail;
  70 /* Store failed because the entry metadata could not be allocated (rare) */
  71 static u64 zswap_reject_kmemcache_fail;
  72 /* Duplicate store was encountered (rare) */
  73 static u64 zswap_duplicate_entry;
  74
  75 /*********************************
  76 * tunables
  77 **********************************/
  78
  79 /* Enable/disable zswap (disabled by default) */
  80 static bool zswap_enabled;
  81 module_param_named(enabled, zswap_enabled, bool, 0644);
  82
  83 /* Crypto compressor to use */
  84 #define ZSWAP_COMPRESSOR_DEFAULT "lzo"
  85 static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
  86 static int zswap_compressor_param_set(const char *,
  87                                       const struct kernel_param *);
  88 static struct kernel_param_ops zswap_compressor_param_ops = {
  89         .set =          zswap_compressor_param_set,
  90         .get =          param_get_charp,
  91         .free =         param_free_charp,
  92 };
  93 module_param_cb(compressor, &zswap_compressor_param_ops,
  94                 &zswap_compressor, 0644);
  95
  96 /* Compressed storage zpool to use */
  97 #define ZSWAP_ZPOOL_DEFAULT "zbud"
  98 static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
  99 static int zswap_zpool_param_set(const char *, const struct kernel_param *);
 100 static struct kernel_param_ops zswap_zpool_param_ops = {
 101         .set =          zswap_zpool_param_set,
 102         .get =          param_get_charp,
 103         .free =         param_free_charp,
 104 };
 105 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
 106
 107 /* The maximum percentage of memory that the compressed pool can occupy */
 108 static unsigned int zswap_max_pool_percent = 20;
 109 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
 110
 111 /*********************************
 112 * data structures
 113 **********************************/
 114
 115 struct zswap_pool {
 116         struct zpool *zpool;
 117         struct crypto_comp * __percpu *tfm;
 118         struct kref kref;
 119         struct list_head list;
 120         struct rcu_head rcu_head;
 121         struct notifier_block notifier;
 122         char tfm_name[CRYPTO_MAX_ALG_NAME];
 123 };
 124
 125 /*
 126  * struct zswap_entry
 127  *
 128  * This structure contains the metadata for tracking a single compressed
 129  * page within zswap.
 130  *
 131  * rbnode - links the entry into red-black tree for the appropriate swap type
 132  * offset - the swap offset for the entry.  Index into the red-black tree.
 133  * refcount - the number of outstanding reference to the entry. This is needed
 134  *            to protect against premature freeing of the entry by code
 135  *            concurrent calls to load, invalidate, and writeback.  The lock
 136  *            for the zswap_tree structure that contains the entry must
 137  *            be held while changing the refcount.  Since the lock must
 138  *            be held, there is no reason to also make refcount atomic.
 139  * length - the length in bytes of the compressed page data.  Needed during
 140  *          decompression
 141  * pool - the zswap_pool the entry's data is in
 142  * handle - zpool allocation handle that stores the compressed page data
 143  */
 144 struct zswap_entry {
 145         struct rb_node rbnode;
 146         pgoff_t offset;
 147         int refcount;
 148         unsigned int length;
 149         struct zswap_pool *pool;
 150         unsigned long handle;
 151 };
 152
 153 struct zswap_header {
 154         swp_entry_t swpentry;
 155 };
 156
 157 /*
 158  * The tree lock in the zswap_tree struct protects a few things:
 159  * - the rbtree
 160  * - the refcount field of each entry in the tree
 161  */
 162 struct zswap_tree {
 163         struct rb_root rbroot;
 164         spinlock_t lock;
 165 };
 166
 167 static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
 168
 169 /* RCU-protected iteration */
 170 static LIST_HEAD(zswap_pools);
 171 /* protects zswap_pools list modification */
 172 static DEFINE_SPINLOCK(zswap_pools_lock);
 173 /* pool counter to provide unique names to zpool */
 174 static atomic_t zswap_pools_count = ATOMIC_INIT(0);
 175
 176 /* used by param callback function */
 177 static bool zswap_init_started;
 178
 179 /*********************************
 180 * helpers and fwd declarations
 181 **********************************/
 182
 183 #define zswap_pool_debug(msg, p)                                \
 184         pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,         \
 185                  zpool_get_type((p)->zpool))
 186
 187 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
 188 static int zswap_pool_get(struct zswap_pool *pool);
 189 static void zswap_pool_put(struct zswap_pool *pool);
 190
 191 static const struct zpool_ops zswap_zpool_ops = {
 192         .evict = zswap_writeback_entry
 193 };
 194
 195 static bool zswap_is_full(void)
 196 {
 197         return totalram_pages * zswap_max_pool_percent / 100 <
 198                 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
 199 }
 200
 201 static void zswap_update_total_size(void)
 202 {
 203         struct zswap_pool *pool;
 204         u64 total = 0;
 205
 206         rcu_read_lock();
 207
 208         list_for_each_entry_rcu(pool, &zswap_pools, list)
 209                 total += zpool_get_total_size(pool->zpool);
 210
 211         rcu_read_unlock();
 212
 213         zswap_pool_total_size = total;
 214 }
 215
 216 /*********************************
 217 * zswap entry functions
 218 **********************************/
 219 static struct kmem_cache *zswap_entry_cache;
 220
 221 static int __init zswap_entry_cache_create(void)
 222 {
 223         zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
 224         return zswap_entry_cache == NULL;
 225 }
 226
 227 static void __init zswap_entry_cache_destroy(void)
 228 {
 229         kmem_cache_destroy(zswap_entry_cache);
 230 }
 231
 232 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
 233 {
 234         struct zswap_entry *entry;
 235         entry = kmem_cache_alloc(zswap_entry_cache, gfp);
 236         if (!entry)
 237                 return NULL;
 238         entry->refcount = 1;
 239         RB_CLEAR_NODE(&entry->rbnode);
 240         return entry;
 241 }
 242
 243 static void zswap_entry_cache_free(struct zswap_entry *entry)
 244 {
 245         kmem_cache_free(zswap_entry_cache, entry);
 246 }
 247
 248 /*********************************
 249 * rbtree functions
 250 **********************************/
 251 static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
 252 {
 253         struct rb_node *node = root->rb_node;
 254         struct zswap_entry *entry;
 255
 256         while (node) {
 257                 entry = rb_entry(node, struct zswap_entry, rbnode);
 258                 if (entry->offset > offset)
 259                         node = node->rb_left;
 260                 else if (entry->offset < offset)
 261                         node = node->rb_right;
 262                 else
 263                         return entry;
 264         }
 265         return NULL;
 266 }
 267
 268 /*
 269  * In the case that a entry with the same offset is found, a pointer to
 270  * the existing entry is stored in dupentry and the function returns -EEXIST
 271  */
 272 static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
 273                         struct zswap_entry **dupentry)
 274 {
 275         struct rb_node **link = &root->rb_node, *parent = NULL;
 276         struct zswap_entry *myentry;
 277
 278         while (*link) {
 279                 parent = *link;
 280                 myentry = rb_entry(parent, struct zswap_entry, rbnode);
 281                 if (myentry->offset > entry->offset)
 282                         link = &(*link)->rb_left;
 283                 else if (myentry->offset < entry->offset)
 284                         link = &(*link)->rb_right;
 285                 else {
 286                         *dupentry = myentry;
 287                         return -EEXIST;
 288                 }
 289         }
 290         rb_link_node(&entry->rbnode, parent, link);
 291         rb_insert_color(&entry->rbnode, root);
 292         return 0;
 293 }
 294
 295 static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
 296 {
 297         if (!RB_EMPTY_NODE(&entry->rbnode)) {
 298                 rb_erase(&entry->rbnode, root);
 299                 RB_CLEAR_NODE(&entry->rbnode);
 300         }
 301 }
 302
 303 /*
 304  * Carries out the common pattern of freeing and entry's zpool allocation,
 305  * freeing the entry itself, and decrementing the number of stored pages.
 306  */
 307 static void zswap_free_entry(struct zswap_entry *entry)
 308 {
 309         zpool_free(entry->pool->zpool, entry->handle);
 310         zswap_pool_put(entry->pool);
 311         zswap_entry_cache_free(entry);
 312         atomic_dec(&zswap_stored_pages);
 313         zswap_update_total_size();
 314 }
 315
 316 /* caller must hold the tree lock */
 317 static void zswap_entry_get(struct zswap_entry *entry)
 318 {
 319         entry->refcount++;
 320 }
 321
 322 /* caller must hold the tree lock
 323 * remove from the tree and free it, if nobody reference the entry
 324 */
 325 static void zswap_entry_put(struct zswap_tree *tree,
 326                         struct zswap_entry *entry)
 327 {
 328         int refcount = --entry->refcount;
 329
 330         BUG_ON(refcount < 0);
 331         if (refcount == 0) {
 332                 zswap_rb_erase(&tree->rbroot, entry);
 333                 zswap_free_entry(entry);
 334         }
 335 }
 336
 337 /* caller must hold the tree lock */
 338 static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
 339                                 pgoff_t offset)
 340 {
 341         struct zswap_entry *entry;
 342
 343         entry = zswap_rb_search(root, offset);
 344         if (entry)
 345                 zswap_entry_get(entry);
 346
 347         return entry;
 348 }
 349
 350 /*********************************
 351 * per-cpu code
 352 **********************************/
 353 static DEFINE_PER_CPU(u8 *, zswap_dstmem);
 354
 355 static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu)
 356 {
 357         u8 *dst;
 358
 359         switch (action) {
 360         case CPU_UP_PREPARE:
 361                 dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
 362                 if (!dst) {
 363                         pr_err("can't allocate compressor buffer\n");
 364                         return NOTIFY_BAD;
 365                 }
 366                 per_cpu(zswap_dstmem, cpu) = dst;
 367                 break;
 368         case CPU_DEAD:
 369         case CPU_UP_CANCELED:
 370                 dst = per_cpu(zswap_dstmem, cpu);
 371                 kfree(dst);
 372                 per_cpu(zswap_dstmem, cpu) = NULL;
 373                 break;
 374         default:
 375                 break;
 376         }
 377         return NOTIFY_OK;
 378 }
 379
 380 static int zswap_cpu_dstmem_notifier(struct notifier_block *nb,
 381                                      unsigned long action, void *pcpu)
 382 {
 383         return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu);
 384 }
 385
 386 static struct notifier_block zswap_dstmem_notifier = {
 387         .notifier_call =        zswap_cpu_dstmem_notifier,
 388 };
 389
 390 static int __init zswap_cpu_dstmem_init(void)
 391 {
 392         unsigned long cpu;
 393
 394         cpu_notifier_register_begin();
 395         for_each_online_cpu(cpu)
 396                 if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) ==
 397                     NOTIFY_BAD)
 398                         goto cleanup;
 399         __register_cpu_notifier(&zswap_dstmem_notifier);
 400         cpu_notifier_register_done();
 401         return 0;
 402
 403 cleanup:
 404         for_each_online_cpu(cpu)
 405                 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
 406         cpu_notifier_register_done();
 407         return -ENOMEM;
 408 }
 409
 410 static void zswap_cpu_dstmem_destroy(void)
 411 {
 412         unsigned long cpu;
 413
 414         cpu_notifier_register_begin();
 415         for_each_online_cpu(cpu)
 416                 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
 417         __unregister_cpu_notifier(&zswap_dstmem_notifier);
 418         cpu_notifier_register_done();
 419 }
 420
 421 static int __zswap_cpu_comp_notifier(struct zswap_pool *pool,
 422                                      unsigned long action, unsigned long cpu)
 423 {
 424         struct crypto_comp *tfm;
 425
 426         switch (action) {
 427         case CPU_UP_PREPARE:
 428                 if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
 429                         break;
 430                 tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
 431                 if (IS_ERR_OR_NULL(tfm)) {
 432                         pr_err("could not alloc crypto comp %s : %ld\n",
 433                                pool->tfm_name, PTR_ERR(tfm));
 434                         return NOTIFY_BAD;
 435                 }
 436                 *per_cpu_ptr(pool->tfm, cpu) = tfm;
 437                 break;
 438         case CPU_DEAD:
 439         case CPU_UP_CANCELED:
 440                 tfm = *per_cpu_ptr(pool->tfm, cpu);
 441                 if (!IS_ERR_OR_NULL(tfm))
 442                         crypto_free_comp(tfm);
 443                 *per_cpu_ptr(pool->tfm, cpu) = NULL;
 444                 break;
 445         default:
 446                 break;
 447         }
 448         return NOTIFY_OK;
 449 }
 450
 451 static int zswap_cpu_comp_notifier(struct notifier_block *nb,
 452                                    unsigned long action, void *pcpu)
 453 {
 454         unsigned long cpu = (unsigned long)pcpu;
 455         struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier);
 456
 457         return __zswap_cpu_comp_notifier(pool, action, cpu);
 458 }
 459
 460 static int zswap_cpu_comp_init(struct zswap_pool *pool)
 461 {
 462         unsigned long cpu;
 463
 464         memset(&pool->notifier, 0, sizeof(pool->notifier));
 465         pool->notifier.notifier_call = zswap_cpu_comp_notifier;
 466
 467         cpu_notifier_register_begin();
 468         for_each_online_cpu(cpu)
 469                 if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) ==
 470                     NOTIFY_BAD)
 471                         goto cleanup;
 472         __register_cpu_notifier(&pool->notifier);
 473         cpu_notifier_register_done();
 474         return 0;
 475
 476 cleanup:
 477         for_each_online_cpu(cpu)
 478                 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
 479         cpu_notifier_register_done();
 480         return -ENOMEM;
 481 }
 482
 483 static void zswap_cpu_comp_destroy(struct zswap_pool *pool)
 484 {
 485         unsigned long cpu;
 486
 487         cpu_notifier_register_begin();
 488         for_each_online_cpu(cpu)
 489                 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
 490         __unregister_cpu_notifier(&pool->notifier);
 491         cpu_notifier_register_done();
 492 }
 493
 494 /*********************************
 495 * pool functions
 496 **********************************/
 497
 498 static struct zswap_pool *__zswap_pool_current(void)
 499 {
 500         struct zswap_pool *pool;
 501
 502         pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
 503         WARN_ON(!pool);
 504
 505         return pool;
 506 }
 507
 508 static struct zswap_pool *zswap_pool_current(void)
 509 {
 510         assert_spin_locked(&zswap_pools_lock);
 511
 512         return __zswap_pool_current();
 513 }
 514
 515 static struct zswap_pool *zswap_pool_current_get(void)
 516 {
 517         struct zswap_pool *pool;
 518
 519         rcu_read_lock();
 520
 521         pool = __zswap_pool_current();
 522         if (!pool || !zswap_pool_get(pool))
 523                 pool = NULL;
 524
 525         rcu_read_unlock();
 526
 527         return pool;
 528 }
 529
 530 static struct zswap_pool *zswap_pool_last_get(void)
 531 {
 532         struct zswap_pool *pool, *last = NULL;
 533
 534         rcu_read_lock();
 535
 536         list_for_each_entry_rcu(pool, &zswap_pools, list)
 537                 last = pool;
 538         if (!WARN_ON(!last) && !zswap_pool_get(last))
 539                 last = NULL;
 540
 541         rcu_read_unlock();
 542
 543         return last;
 544 }
 545
 546 /* type and compressor must be null-terminated */
 547 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
 548 {
 549         struct zswap_pool *pool;
 550
 551         assert_spin_locked(&zswap_pools_lock);
 552
 553         list_for_each_entry_rcu(pool, &zswap_pools, list) {
 554                 if (strcmp(pool->tfm_name, compressor))
 555                         continue;
 556                 if (strcmp(zpool_get_type(pool->zpool), type))
 557                         continue;
 558                 /* if we can't get it, it's about to be destroyed */
 559                 if (!zswap_pool_get(pool))
 560                         continue;
 561                 return pool;
 562         }
 563
 564         return NULL;
 565 }
 566
 567 static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
 568 {
 569         struct zswap_pool *pool;
 570         char name[38]; /* 'zswap' + 32 char (max) num + \0 */
 571         gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
 572
 573         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 574         if (!pool) {
 575                 pr_err("pool alloc failed\n");
 576                 return NULL;
 577         }
 578
 579         /* unique name for each pool specifically required by zsmalloc */
 580         snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
 581
 582         pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
 583         if (!pool->zpool) {
 584                 pr_err("%s zpool not available\n", type);
 585                 goto error;
 586         }
 587         pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
 588
 589         strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
 590         pool->tfm = alloc_percpu(struct crypto_comp *);
 591         if (!pool->tfm) {
 592                 pr_err("percpu alloc failed\n");
 593                 goto error;
 594         }
 595
 596         if (zswap_cpu_comp_init(pool))
 597                 goto error;
 598         pr_debug("using %s compressor\n", pool->tfm_name);
 599
 600         /* being the current pool takes 1 ref; this func expects the
 601          * caller to always add the new pool as the current pool
 602          */
 603         kref_init(&pool->kref);
 604         INIT_LIST_HEAD(&pool->list);
 605
 606         zswap_pool_debug("created", pool);
 607
 608         return pool;
 609
 610 error:
 611         free_percpu(pool->tfm);
 612         if (pool->zpool)
 613                 zpool_destroy_pool(pool->zpool);
 614         kfree(pool);
 615         return NULL;
 616 }
 617
 618 static __init struct zswap_pool *__zswap_pool_create_fallback(void)
 619 {
 620         if (!crypto_has_comp(zswap_compressor, 0, 0)) {
 621                 if (!strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
 622                         pr_err("default compressor %s not available\n",
 623                                zswap_compressor);
 624                         return NULL;
 625                 }
 626                 pr_err("compressor %s not available, using default %s\n",
 627                        zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
 628                 param_free_charp(&zswap_compressor);
 629                 zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
 630         }
 631         if (!zpool_has_pool(zswap_zpool_type)) {
 632                 if (!strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
 633                         pr_err("default zpool %s not available\n",
 634                                zswap_zpool_type);
 635                         return NULL;
 636                 }
 637                 pr_err("zpool %s not available, using default %s\n",
 638                        zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
 639                 param_free_charp(&zswap_zpool_type);
 640                 zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
 641         }
 642
 643         return zswap_pool_create(zswap_zpool_type, zswap_compressor);
 644 }
 645
 646 static void zswap_pool_destroy(struct zswap_pool *pool)
 647 {
 648         zswap_pool_debug("destroying", pool);
 649
 650         zswap_cpu_comp_destroy(pool);
 651         free_percpu(pool->tfm);
 652         zpool_destroy_pool(pool->zpool);
 653         kfree(pool);
 654 }
 655
 656 static int __must_check zswap_pool_get(struct zswap_pool *pool)
 657 {
 658         return kref_get_unless_zero(&pool->kref);
 659 }
 660
 661 static void __zswap_pool_release(struct rcu_head *head)
 662 {
 663         struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head);
 664
 665         /* nobody should have been able to get a kref... */
 666         WARN_ON(kref_get_unless_zero(&pool->kref));
 667
 668         /* pool is now off zswap_pools list and has no references. */
 669         zswap_pool_destroy(pool);
 670 }
 671
 672 static void __zswap_pool_empty(struct kref *kref)
 673 {
 674         struct zswap_pool *pool;
 675
 676         pool = container_of(kref, typeof(*pool), kref);
 677
 678         spin_lock(&zswap_pools_lock);
 679
 680         WARN_ON(pool == zswap_pool_current());
 681
 682         list_del_rcu(&pool->list);
 683         call_rcu(&pool->rcu_head, __zswap_pool_release);
 684
 685         spin_unlock(&zswap_pools_lock);
 686 }
 687
 688 static void zswap_pool_put(struct zswap_pool *pool)
 689 {
 690         kref_put(&pool->kref, __zswap_pool_empty);
 691 }
 692
 693 /*********************************
 694 * param callbacks
 695 **********************************/
 696
 697 /* val must be a null-terminated string */
 698 static int __zswap_param_set(const char *val, const struct kernel_param *kp,
 699                              char *type, char *compressor)
 700 {
 701         struct zswap_pool *pool, *put_pool = NULL;
 702         char *s = strstrip((char *)val);
 703         int ret;
 704
 705         /* no change required */
 706         if (!strcmp(s, *(char **)kp->arg))
 707                 return 0;
 708
 709         /* if this is load-time (pre-init) param setting,
 710          * don't create a pool; that's done during init.
 711          */
 712         if (!zswap_init_started)
 713                 return param_set_charp(s, kp);
 714
 715         if (!type) {
 716                 if (!zpool_has_pool(s)) {
 717                         pr_err("zpool %s not available\n", s);
 718                         return -ENOENT;
 719                 }
 720                 type = s;
 721         } else if (!compressor) {
 722                 if (!crypto_has_comp(s, 0, 0)) {
 723                         pr_err("compressor %s not available\n", s);
 724                         return -ENOENT;
 725                 }
 726                 compressor = s;
 727         } else {
 728                 WARN_ON(1);
 729                 return -EINVAL;
 730         }
 731
 732         spin_lock(&zswap_pools_lock);
 733
 734         pool = zswap_pool_find_get(type, compressor);
 735         if (pool) {
 736                 zswap_pool_debug("using existing", pool);
 737                 list_del_rcu(&pool->list);
 738         } else {
 739                 spin_unlock(&zswap_pools_lock);
 740                 pool = zswap_pool_create(type, compressor);
 741                 spin_lock(&zswap_pools_lock);
 742         }
 743
 744         if (pool)
 745                 ret = param_set_charp(s, kp);
 746         else
 747                 ret = -EINVAL;
 748
 749         if (!ret) {
 750                 put_pool = zswap_pool_current();
 751                 list_add_rcu(&pool->list, &zswap_pools);
 752         } else if (pool) {
 753                 /* add the possibly pre-existing pool to the end of the pools
 754                  * list; if it's new (and empty) then it'll be removed and
 755                  * destroyed by the put after we drop the lock
 756                  */
 757                 list_add_tail_rcu(&pool->list, &zswap_pools);
 758                 put_pool = pool;
 759         }
 760
 761         spin_unlock(&zswap_pools_lock);
 762
 763         /* drop the ref from either the old current pool,
 764          * or the new pool we failed to add
 765          */
 766         if (put_pool)
 767                 zswap_pool_put(put_pool);
 768
 769         return ret;
 770 }
 771
 772 static int zswap_compressor_param_set(const char *val,
 773                                       const struct kernel_param *kp)
 774 {
 775         return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
 776 }
 777
 778 static int zswap_zpool_param_set(const char *val,
 779                                  const struct kernel_param *kp)
 780 {
 781         return __zswap_param_set(val, kp, NULL, zswap_compressor);
 782 }
 783
 784 /*********************************
 785 * writeback code
 786 **********************************/
 787 /* return enum for zswap_get_swap_cache_page */
 788 enum zswap_get_swap_ret {
 789         ZSWAP_SWAPCACHE_NEW,
 790         ZSWAP_SWAPCACHE_EXIST,
 791         ZSWAP_SWAPCACHE_FAIL,
 792 };
 793
 794 /*
 795  * zswap_get_swap_cache_page
 796  *
 797  * This is an adaption of read_swap_cache_async()
 798  *
 799  * This function tries to find a page with the given swap entry
 800  * in the swapper_space address space (the swap cache).  If the page
 801  * is found, it is returned in retpage.  Otherwise, a page is allocated,
 802  * added to the swap cache, and returned in retpage.
 803  *
 804  * If success, the swap cache page is returned in retpage
 805  * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
 806  * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
 807  *     the new page is added to swapcache and locked
 808  * Returns ZSWAP_SWAPCACHE_FAIL on error
 809  */
 810 static int zswap_get_swap_cache_page(swp_entry_t entry,
 811                                 struct page **retpage)
 812 {
 813         bool page_was_allocated;
 814
 815         *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
 816                         NULL, 0, &page_was_allocated);
 817         if (page_was_allocated)
 818                 return ZSWAP_SWAPCACHE_NEW;
 819         if (!*retpage)
 820                 return ZSWAP_SWAPCACHE_FAIL;
 821         return ZSWAP_SWAPCACHE_EXIST;
 822 }
 823
 824 /*
 825  * Attempts to free an entry by adding a page to the swap cache,
 826  * decompressing the entry data into the page, and issuing a
 827  * bio write to write the page back to the swap device.
 828  *
 829  * This can be thought of as a "resumed writeback" of the page
 830  * to the swap device.  We are basically resuming the same swap
 831  * writeback path that was intercepted with the frontswap_store()
 832  * in the first place.  After the page has been decompressed into
 833  * the swap cache, the compressed version stored by zswap can be
 834  * freed.
 835  */
 836 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
 837 {
 838         struct zswap_header *zhdr;
 839         swp_entry_t swpentry;
 840         struct zswap_tree *tree;
 841         pgoff_t offset;
 842         struct zswap_entry *entry;
 843         struct page *page;
 844         struct crypto_comp *tfm;
 845         u8 *src, *dst;
 846         unsigned int dlen;
 847         int ret;
 848         struct writeback_control wbc = {
 849                 .sync_mode = WB_SYNC_NONE,
 850         };
 851
 852         /* extract swpentry from data */
 853         zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
 854         swpentry = zhdr->swpentry; /* here */
 855         zpool_unmap_handle(pool, handle);
 856         tree = zswap_trees[swp_type(swpentry)];
 857         offset = swp_offset(swpentry);
 858
 859         /* find and ref zswap entry */
 860         spin_lock(&tree->lock);
 861         entry = zswap_entry_find_get(&tree->rbroot, offset);
 862         if (!entry) {
 863                 /* entry was invalidated */
 864                 spin_unlock(&tree->lock);
 865                 return 0;
 866         }
 867         spin_unlock(&tree->lock);
 868         BUG_ON(offset != entry->offset);
 869
 870         /* try to allocate swap cache page */
 871         switch (zswap_get_swap_cache_page(swpentry, &page)) {
 872         case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
 873                 ret = -ENOMEM;
 874                 goto fail;
 875
 876         case ZSWAP_SWAPCACHE_EXIST:
 877                 /* page is already in the swap cache, ignore for now */
 878                 page_cache_release(page);
 879                 ret = -EEXIST;
 880                 goto fail;
 881
 882         case ZSWAP_SWAPCACHE_NEW: /* page is locked */
 883                 /* decompress */
 884                 dlen = PAGE_SIZE;
 885                 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
 886                                 ZPOOL_MM_RO) + sizeof(struct zswap_header);
 887                 dst = kmap_atomic(page);
 888                 tfm = *get_cpu_ptr(entry->pool->tfm);
 889                 ret = crypto_comp_decompress(tfm, src, entry->length,
 890                                              dst, &dlen);
 891                 put_cpu_ptr(entry->pool->tfm);
 892                 kunmap_atomic(dst);
 893                 zpool_unmap_handle(entry->pool->zpool, entry->handle);
 894                 BUG_ON(ret);
 895                 BUG_ON(dlen != PAGE_SIZE);
 896
 897                 /* page is up to date */
 898                 SetPageUptodate(page);
 899         }
 900
 901         /* move it to the tail of the inactive list after end_writeback */
 902         SetPageReclaim(page);
 903
 904         /* start writeback */
 905         __swap_writepage(page, &wbc, end_swap_bio_write);
 906         page_cache_release(page);
 907         zswap_written_back_pages++;
 908
 909         spin_lock(&tree->lock);
 910         /* drop local reference */
 911         zswap_entry_put(tree, entry);
 912
 913         /*
 914         * There are two possible situations for entry here:
 915         * (1) refcount is 1(normal case),  entry is valid and on the tree
 916         * (2) refcount is 0, entry is freed and not on the tree
 917         *     because invalidate happened during writeback
 918         *  search the tree and free the entry if find entry
 919         */
 920         if (entry == zswap_rb_search(&tree->rbroot, offset))
 921                 zswap_entry_put(tree, entry);
 922         spin_unlock(&tree->lock);
 923
 924         goto end;
 925
 926         /*
 927         * if we get here due to ZSWAP_SWAPCACHE_EXIST
 928         * a load may happening concurrently
 929         * it is safe and okay to not free the entry
 930         * if we free the entry in the following put
 931         * it it either okay to return !0
 932         */
 933 fail:
 934         spin_lock(&tree->lock);
 935         zswap_entry_put(tree, entry);
 936         spin_unlock(&tree->lock);
 937
 938 end:
 939         return ret;
 940 }
 941
 942 static int zswap_shrink(void)
 943 {
 944         struct zswap_pool *pool;
 945         int ret;
 946
 947         pool = zswap_pool_last_get();
 948         if (!pool)
 949                 return -ENOENT;
 950
 951         ret = zpool_shrink(pool->zpool, 1, NULL);
 952
 953         zswap_pool_put(pool);
 954
 955         return ret;
 956 }
 957
 958 /*********************************
 959 * frontswap hooks
 960 **********************************/
 961 /* attempts to compress and store an single page */
 962 static int zswap_frontswap_store(unsigned type, pgoff_t offset,
 963                                 struct page *page)
 964 {
 965         struct zswap_tree *tree = zswap_trees[type];
 966         struct zswap_entry *entry, *dupentry;
 967         struct crypto_comp *tfm;
 968         int ret;
 969         unsigned int dlen = PAGE_SIZE, len;
 970         unsigned long handle;
 971         char *buf;
 972         u8 *src, *dst;
 973         struct zswap_header *zhdr;
 974
 975         if (!zswap_enabled || !tree) {
 976                 ret = -ENODEV;
 977                 goto reject;
 978         }
 979
 980         /* reclaim space if needed */
 981         if (zswap_is_full()) {
 982                 zswap_pool_limit_hit++;
 983                 if (zswap_shrink()) {
 984                         zswap_reject_reclaim_fail++;
 985                         ret = -ENOMEM;
 986                         goto reject;
 987                 }
 988         }
 989
 990         /* allocate entry */
 991         entry = zswap_entry_cache_alloc(GFP_KERNEL);
 992         if (!entry) {
 993                 zswap_reject_kmemcache_fail++;
 994                 ret = -ENOMEM;
 995                 goto reject;
 996         }
 997
 998         /* if entry is successfully added, it keeps the reference */
 999         entry->pool = zswap_pool_current_get();
1000         if (!entry->pool) {
1001                 ret = -EINVAL;
1002                 goto freepage;
1003         }
1004
1005         /* compress */
1006         dst = get_cpu_var(zswap_dstmem);
1007         tfm = *get_cpu_ptr(entry->pool->tfm);
1008         src = kmap_atomic(page);
1009         ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
1010         kunmap_atomic(src);
1011         put_cpu_ptr(entry->pool->tfm);
1012         if (ret) {
1013                 ret = -EINVAL;
1014                 goto put_dstmem;
1015         }
1016
1017         /* store */
1018         len = dlen + sizeof(struct zswap_header);
1019         ret = zpool_malloc(entry->pool->zpool, len,
1020                            __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
1021                            &handle);
1022         if (ret == -ENOSPC) {
1023                 zswap_reject_compress_poor++;
1024                 goto put_dstmem;
1025         }
1026         if (ret) {
1027                 zswap_reject_alloc_fail++;
1028                 goto put_dstmem;
1029         }
1030         zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
1031         zhdr->swpentry = swp_entry(type, offset);
1032         buf = (u8 *)(zhdr + 1);
1033         memcpy(buf, dst, dlen);
1034         zpool_unmap_handle(entry->pool->zpool, handle);
1035         put_cpu_var(zswap_dstmem);
1036
1037         /* populate entry */
1038         entry->offset = offset;
1039         entry->handle = handle;
1040         entry->length = dlen;
1041
1042         /* map */
1043         spin_lock(&tree->lock);
1044         do {
1045                 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
1046                 if (ret == -EEXIST) {
1047                         zswap_duplicate_entry++;
1048                         /* remove from rbtree */
1049                         zswap_rb_erase(&tree->rbroot, dupentry);
1050                         zswap_entry_put(tree, dupentry);
1051                 }
1052         } while (ret == -EEXIST);
1053         spin_unlock(&tree->lock);
1054
1055         /* update stats */
1056         atomic_inc(&zswap_stored_pages);
1057         zswap_update_total_size();
1058
1059         return 0;
1060
1061 put_dstmem:
1062         put_cpu_var(zswap_dstmem);
1063         zswap_pool_put(entry->pool);
1064 freepage:
1065         zswap_entry_cache_free(entry);
1066 reject:
1067         return ret;
1068 }
1069
1070 /*
1071  * returns 0 if the page was successfully decompressed
1072  * return -1 on entry not found or error
1073 */
1074 static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1075                                 struct page *page)
1076 {
1077         struct zswap_tree *tree = zswap_trees[type];
1078         struct zswap_entry *entry;
1079         struct crypto_comp *tfm;
1080         u8 *src, *dst;
1081         unsigned int dlen;
1082         int ret;
1083
1084         /* find */
1085         spin_lock(&tree->lock);
1086         entry = zswap_entry_find_get(&tree->rbroot, offset);
1087         if (!entry) {
1088                 /* entry was written back */
1089                 spin_unlock(&tree->lock);
1090                 return -1;
1091         }
1092         spin_unlock(&tree->lock);
1093
1094         /* decompress */
1095         dlen = PAGE_SIZE;
1096         src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
1097                         ZPOOL_MM_RO) + sizeof(struct zswap_header);
1098         dst = kmap_atomic(page);
1099         tfm = *get_cpu_ptr(entry->pool->tfm);
1100         ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
1101         put_cpu_ptr(entry->pool->tfm);
1102         kunmap_atomic(dst);
1103         zpool_unmap_handle(entry->pool->zpool, entry->handle);
1104         BUG_ON(ret);
1105
1106         spin_lock(&tree->lock);
1107         zswap_entry_put(tree, entry);
1108         spin_unlock(&tree->lock);
1109
1110         return 0;
1111 }
1112
1113 /* frees an entry in zswap */
1114 static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1115 {
1116         struct zswap_tree *tree = zswap_trees[type];
1117         struct zswap_entry *entry;
1118
1119         /* find */
1120         spin_lock(&tree->lock);
1121         entry = zswap_rb_search(&tree->rbroot, offset);
1122         if (!entry) {
1123                 /* entry was written back */
1124                 spin_unlock(&tree->lock);
1125                 return;
1126         }
1127
1128         /* remove from rbtree */
1129         zswap_rb_erase(&tree->rbroot, entry);
1130
1131         /* drop the initial reference from entry creation */
1132         zswap_entry_put(tree, entry);
1133
1134         spin_unlock(&tree->lock);
1135 }
1136
1137 /* frees all zswap entries for the given swap type */
1138 static void zswap_frontswap_invalidate_area(unsigned type)
1139 {
1140         struct zswap_tree *tree = zswap_trees[type];
1141         struct zswap_entry *entry, *n;
1142
1143         if (!tree)
1144                 return;
1145
1146         /* walk the tree and free everything */
1147         spin_lock(&tree->lock);
1148         rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1149                 zswap_free_entry(entry);
1150         tree->rbroot = RB_ROOT;
1151         spin_unlock(&tree->lock);
1152         kfree(tree);
1153         zswap_trees[type] = NULL;
1154 }
1155
1156 static void zswap_frontswap_init(unsigned type)
1157 {
1158         struct zswap_tree *tree;
1159
1160         tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
1161         if (!tree) {
1162                 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1163                 return;
1164         }
1165
1166         tree->rbroot = RB_ROOT;
1167         spin_lock_init(&tree->lock);
1168         zswap_trees[type] = tree;
1169 }
1170
1171 static struct frontswap_ops zswap_frontswap_ops = {
1172         .store = zswap_frontswap_store,
1173         .load = zswap_frontswap_load,
1174         .invalidate_page = zswap_frontswap_invalidate_page,
1175         .invalidate_area = zswap_frontswap_invalidate_area,
1176         .init = zswap_frontswap_init
1177 };
1178
1179 /*********************************
1180 * debugfs functions
1181 **********************************/
1182 #ifdef CONFIG_DEBUG_FS
1183 #include <linux/debugfs.h>
1184
1185 static struct dentry *zswap_debugfs_root;
1186
1187 static int __init zswap_debugfs_init(void)
1188 {
1189         if (!debugfs_initialized())
1190                 return -ENODEV;
1191
1192         zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1193         if (!zswap_debugfs_root)
1194                 return -ENOMEM;
1195
1196         debugfs_create_u64("pool_limit_hit", S_IRUGO,
1197                         zswap_debugfs_root, &zswap_pool_limit_hit);
1198         debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
1199                         zswap_debugfs_root, &zswap_reject_reclaim_fail);
1200         debugfs_create_u64("reject_alloc_fail", S_IRUGO,
1201                         zswap_debugfs_root, &zswap_reject_alloc_fail);
1202         debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
1203                         zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1204         debugfs_create_u64("reject_compress_poor", S_IRUGO,
1205                         zswap_debugfs_root, &zswap_reject_compress_poor);
1206         debugfs_create_u64("written_back_pages", S_IRUGO,
1207                         zswap_debugfs_root, &zswap_written_back_pages);
1208         debugfs_create_u64("duplicate_entry", S_IRUGO,
1209                         zswap_debugfs_root, &zswap_duplicate_entry);
1210         debugfs_create_u64("pool_total_size", S_IRUGO,
1211                         zswap_debugfs_root, &zswap_pool_total_size);
1212         debugfs_create_atomic_t("stored_pages", S_IRUGO,
1213                         zswap_debugfs_root, &zswap_stored_pages);
1214
1215         return 0;
1216 }
1217
1218 static void __exit zswap_debugfs_exit(void)
1219 {
1220         debugfs_remove_recursive(zswap_debugfs_root);
1221 }
1222 #else
1223 static int __init zswap_debugfs_init(void)
1224 {
1225         return 0;
1226 }
1227
1228 static void __exit zswap_debugfs_exit(void) { }
1229 #endif
1230
1231 /*********************************
1232 * module init and exit
1233 **********************************/
1234 static int __init init_zswap(void)
1235 {
1236         struct zswap_pool *pool;
1237
1238         zswap_init_started = true;
1239
1240         if (zswap_entry_cache_create()) {
1241                 pr_err("entry cache creation failed\n");
1242                 goto cache_fail;
1243         }
1244
1245         if (zswap_cpu_dstmem_init()) {
1246                 pr_err("dstmem alloc failed\n");
1247                 goto dstmem_fail;
1248         }
1249
1250         pool = __zswap_pool_create_fallback();
1251         if (!pool) {
1252                 pr_err("pool creation failed\n");
1253                 goto pool_fail;
1254         }
1255         pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1256                 zpool_get_type(pool->zpool));
1257
1258         list_add(&pool->list, &zswap_pools);
1259
1260         frontswap_register_ops(&zswap_frontswap_ops);
1261         if (zswap_debugfs_init())
1262                 pr_warn("debugfs initialization failed\n");
1263         return 0;
1264
1265 pool_fail:
1266         zswap_cpu_dstmem_destroy();
1267 dstmem_fail:
1268         zswap_entry_cache_destroy();
1269 cache_fail:
1270         return -ENOMEM;
1271 }
1272 /* must be late so crypto has time to come up */
1273 late_initcall(init_zswap);
1274
1275 MODULE_LICENSE("GPL");
1276 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1277 MODULE_DESCRIPTION("Compressed cache for swap pages");