fs/cachefiles/rdwr.c

   1 /* Storage object read/write
   2  *
   3  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   4  * Written by David Howells (dhowells@redhat.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public Licence
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the Licence, or (at your option) any later version.
  10  */
  11
  12 #include <linux/mount.h>
  13 #include <linux/slab.h>
  14 #include <linux/file.h>
  15 #include "internal.h"
  16
  17 /*
  18  * detect wake up events generated by the unlocking of pages in which we're
  19  * interested
  20  * - we use this to detect read completion of backing pages
  21  * - the caller holds the waitqueue lock
  22  */
  23 static int cachefiles_read_waiter(wait_queue_t *wait, unsigned mode,
  24                                   int sync, void *_key)
  25 {
  26         struct cachefiles_one_read *monitor =
  27                 container_of(wait, struct cachefiles_one_read, monitor);
  28         struct cachefiles_object *object;
  29         struct wait_bit_key *key = _key;
  30         struct page *page = wait->private;
  31
  32         ASSERT(key);
  33
  34         _enter("{%lu},%u,%d,{%p,%u}",
  35                monitor->netfs_page->index, mode, sync,
  36                key->flags, key->bit_nr);
  37
  38         if (key->flags != &page->flags ||
  39             key->bit_nr != PG_locked)
  40                 return 0;
  41
  42         _debug("--- monitor %p %lx ---", page, page->flags);
  43
  44         if (!PageUptodate(page) && !PageError(page)) {
  45                 /* unlocked, not uptodate and not erronous? */
  46                 _debug("page probably truncated");
  47         }
  48
  49         /* remove from the waitqueue */
  50         list_del(&wait->task_list);
  51
  52         /* move onto the action list and queue for FS-Cache thread pool */
  53         ASSERT(monitor->op);
  54
  55         object = container_of(monitor->op->op.object,
  56                               struct cachefiles_object, fscache);
  57
  58         spin_lock(&object->work_lock);
  59         list_add_tail(&monitor->op_link, &monitor->op->to_do);
  60         spin_unlock(&object->work_lock);
  61
  62         fscache_enqueue_retrieval(monitor->op);
  63         return 0;
  64 }
  65
  66 /*
  67  * handle a probably truncated page
  68  * - check to see if the page is still relevant and reissue the read if
  69  *   possible
  70  * - return -EIO on error, -ENODATA if the page is gone, -EINPROGRESS if we
  71  *   must wait again and 0 if successful
  72  */
  73 static int cachefiles_read_reissue(struct cachefiles_object *object,
  74                                    struct cachefiles_one_read *monitor)
  75 {
  76         struct address_space *bmapping = object->backer->d_inode->i_mapping;
  77         struct page *backpage = monitor->back_page, *backpage2;
  78         int ret;
  79
  80         _enter("{ino=%lx},{%lx,%lx}",
  81                object->backer->d_inode->i_ino,
  82                backpage->index, backpage->flags);
  83
  84         /* skip if the page was truncated away completely */
  85         if (backpage->mapping != bmapping) {
  86                 _leave(" = -ENODATA [mapping]");
  87                 return -ENODATA;
  88         }
  89
  90         backpage2 = find_get_page(bmapping, backpage->index);
  91         if (!backpage2) {
  92                 _leave(" = -ENODATA [gone]");
  93                 return -ENODATA;
  94         }
  95
  96         if (backpage != backpage2) {
  97                 put_page(backpage2);
  98                 _leave(" = -ENODATA [different]");
  99                 return -ENODATA;
 100         }
 101
 102         /* the page is still there and we already have a ref on it, so we don't
 103          * need a second */
 104         put_page(backpage2);
 105
 106         INIT_LIST_HEAD(&monitor->op_link);
 107         add_page_wait_queue(backpage, &monitor->monitor);
 108
 109         if (trylock_page(backpage)) {
 110                 ret = -EIO;
 111                 if (PageError(backpage))
 112                         goto unlock_discard;
 113                 ret = 0;
 114                 if (PageUptodate(backpage))
 115                         goto unlock_discard;
 116
 117                 _debug("reissue read");
 118                 ret = bmapping->a_ops->readpage(NULL, backpage);
 119                 if (ret < 0)
 120                         goto unlock_discard;
 121         }
 122
 123         /* but the page may have been read before the monitor was installed, so
 124          * the monitor may miss the event - so we have to ensure that we do get
 125          * one in such a case */
 126         if (trylock_page(backpage)) {
 127                 _debug("jumpstart %p {%lx}", backpage, backpage->flags);
 128                 unlock_page(backpage);
 129         }
 130
 131         /* it'll reappear on the todo list */
 132         _leave(" = -EINPROGRESS");
 133         return -EINPROGRESS;
 134
 135 unlock_discard:
 136         unlock_page(backpage);
 137         spin_lock_irq(&object->work_lock);
 138         list_del(&monitor->op_link);
 139         spin_unlock_irq(&object->work_lock);
 140         _leave(" = %d", ret);
 141         return ret;
 142 }
 143
 144 /*
 145  * copy data from backing pages to netfs pages to complete a read operation
 146  * - driven by FS-Cache's thread pool
 147  */
 148 static void cachefiles_read_copier(struct fscache_operation *_op)
 149 {
 150         struct cachefiles_one_read *monitor;
 151         struct cachefiles_object *object;
 152         struct fscache_retrieval *op;
 153         struct pagevec pagevec;
 154         int error, max;
 155
 156         op = container_of(_op, struct fscache_retrieval, op);
 157         object = container_of(op->op.object,
 158                               struct cachefiles_object, fscache);
 159
 160         _enter("{ino=%lu}", object->backer->d_inode->i_ino);
 161
 162         pagevec_init(&pagevec, 0);
 163
 164         max = 8;
 165         spin_lock_irq(&object->work_lock);
 166
 167         while (!list_empty(&op->to_do)) {
 168                 monitor = list_entry(op->to_do.next,
 169                                      struct cachefiles_one_read, op_link);
 170                 list_del(&monitor->op_link);
 171
 172                 spin_unlock_irq(&object->work_lock);
 173
 174                 _debug("- copy {%lu}", monitor->back_page->index);
 175
 176         recheck:
 177                 if (test_bit(FSCACHE_COOKIE_INVALIDATING,
 178                              &object->fscache.cookie->flags)) {
 179                         error = -ESTALE;
 180                 } else if (PageUptodate(monitor->back_page)) {
 181                         copy_highpage(monitor->netfs_page, monitor->back_page);
 182                         fscache_mark_page_cached(monitor->op,
 183                                                  monitor->netfs_page);
 184                         error = 0;
 185                 } else if (!PageError(monitor->back_page)) {
 186                         /* the page has probably been truncated */
 187                         error = cachefiles_read_reissue(object, monitor);
 188                         if (error == -EINPROGRESS)
 189                                 goto next;
 190                         goto recheck;
 191                 } else {
 192                         cachefiles_io_error_obj(
 193                                 object,
 194                                 "Readpage failed on backing file %lx",
 195                                 (unsigned long) monitor->back_page->flags);
 196                         error = -EIO;
 197                 }
 198
 199                 page_cache_release(monitor->back_page);
 200
 201                 fscache_end_io(op, monitor->netfs_page, error);
 202                 page_cache_release(monitor->netfs_page);
 203                 fscache_retrieval_complete(op, 1);
 204                 fscache_put_retrieval(op);
 205                 kfree(monitor);
 206
 207         next:
 208                 /* let the thread pool have some air occasionally */
 209                 max--;
 210                 if (max < 0 || need_resched()) {
 211                         if (!list_empty(&op->to_do))
 212                                 fscache_enqueue_retrieval(op);
 213                         _leave(" [maxed out]");
 214                         return;
 215                 }
 216
 217                 spin_lock_irq(&object->work_lock);
 218         }
 219
 220         spin_unlock_irq(&object->work_lock);
 221         _leave("");
 222 }
 223
 224 /*
 225  * read the corresponding page to the given set from the backing file
 226  * - an uncertain page is simply discarded, to be tried again another time
 227  */
 228 static int cachefiles_read_backing_file_one(struct cachefiles_object *object,
 229                                             struct fscache_retrieval *op,
 230                                             struct page *netpage,
 231                                             struct pagevec *pagevec)
 232 {
 233         struct cachefiles_one_read *monitor;
 234         struct address_space *bmapping;
 235         struct page *newpage, *backpage;
 236         int ret;
 237
 238         _enter("");
 239
 240         pagevec_reinit(pagevec);
 241
 242         _debug("read back %p{%lu,%d}",
 243                netpage, netpage->index, page_count(netpage));
 244
 245         monitor = kzalloc(sizeof(*monitor), cachefiles_gfp);
 246         if (!monitor)
 247                 goto nomem;
 248
 249         monitor->netfs_page = netpage;
 250         monitor->op = fscache_get_retrieval(op);
 251
 252         init_waitqueue_func_entry(&monitor->monitor, cachefiles_read_waiter);
 253
 254         /* attempt to get hold of the backing page */
 255         bmapping = object->backer->d_inode->i_mapping;
 256         newpage = NULL;
 257
 258         for (;;) {
 259                 backpage = find_get_page(bmapping, netpage->index);
 260                 if (backpage)
 261                         goto backing_page_already_present;
 262
 263                 if (!newpage) {
 264                         newpage = __page_cache_alloc(cachefiles_gfp |
 265                                                      __GFP_COLD);
 266                         if (!newpage)
 267                                 goto nomem_monitor;
 268                 }
 269
 270                 ret = add_to_page_cache(newpage, bmapping,
 271                                         netpage->index, cachefiles_gfp);
 272                 if (ret == 0)
 273                         goto installed_new_backing_page;
 274                 if (ret != -EEXIST)
 275                         goto nomem_page;
 276         }
 277
 278         /* we've installed a new backing page, so now we need to add it
 279          * to the LRU list and start it reading */
 280 installed_new_backing_page:
 281         _debug("- new %p", newpage);
 282
 283         backpage = newpage;
 284         newpage = NULL;
 285
 286         page_cache_get(backpage);
 287         pagevec_add(pagevec, backpage);
 288         __pagevec_lru_add_file(pagevec);
 289
 290 read_backing_page:
 291         ret = bmapping->a_ops->readpage(NULL, backpage);
 292         if (ret < 0)
 293                 goto read_error;
 294
 295         /* set the monitor to transfer the data across */
 296 monitor_backing_page:
 297         _debug("- monitor add");
 298
 299         /* install the monitor */
 300         page_cache_get(monitor->netfs_page);
 301         page_cache_get(backpage);
 302         monitor->back_page = backpage;
 303         monitor->monitor.private = backpage;
 304         add_page_wait_queue(backpage, &monitor->monitor);
 305         monitor = NULL;
 306
 307         /* but the page may have been read before the monitor was installed, so
 308          * the monitor may miss the event - so we have to ensure that we do get
 309          * one in such a case */
 310         if (trylock_page(backpage)) {
 311                 _debug("jumpstart %p {%lx}", backpage, backpage->flags);
 312                 unlock_page(backpage);
 313         }
 314         goto success;
 315
 316         /* if the backing page is already present, it can be in one of
 317          * three states: read in progress, read failed or read okay */
 318 backing_page_already_present:
 319         _debug("- present");
 320
 321         if (newpage) {
 322                 page_cache_release(newpage);
 323                 newpage = NULL;
 324         }
 325
 326         if (PageError(backpage))
 327                 goto io_error;
 328
 329         if (PageUptodate(backpage))
 330                 goto backing_page_already_uptodate;
 331
 332         if (!trylock_page(backpage))
 333                 goto monitor_backing_page;
 334         _debug("read %p {%lx}", backpage, backpage->flags);
 335         goto read_backing_page;
 336
 337         /* the backing page is already up to date, attach the netfs
 338          * page to the pagecache and LRU and copy the data across */
 339 backing_page_already_uptodate:
 340         _debug("- uptodate");
 341
 342         fscache_mark_page_cached(op, netpage);
 343
 344         copy_highpage(netpage, backpage);
 345         fscache_end_io(op, netpage, 0);
 346         fscache_retrieval_complete(op, 1);
 347
 348 success:
 349         _debug("success");
 350         ret = 0;
 351
 352 out:
 353         if (backpage)
 354                 page_cache_release(backpage);
 355         if (monitor) {
 356                 fscache_put_retrieval(monitor->op);
 357                 kfree(monitor);
 358         }
 359         _leave(" = %d", ret);
 360         return ret;
 361
 362 read_error:
 363         _debug("read error %d", ret);
 364         if (ret == -ENOMEM)
 365                 goto out;
 366 io_error:
 367         cachefiles_io_error_obj(object, "Page read error on backing file");
 368         fscache_retrieval_complete(op, 1);
 369         ret = -ENOBUFS;
 370         goto out;
 371
 372 nomem_page:
 373         page_cache_release(newpage);
 374 nomem_monitor:
 375         fscache_put_retrieval(monitor->op);
 376         kfree(monitor);
 377 nomem:
 378         fscache_retrieval_complete(op, 1);
 379         _leave(" = -ENOMEM");
 380         return -ENOMEM;
 381 }
 382
 383 /*
 384  * read a page from the cache or allocate a block in which to store it
 385  * - cache withdrawal is prevented by the caller
 386  * - returns -EINTR if interrupted
 387  * - returns -ENOMEM if ran out of memory
 388  * - returns -ENOBUFS if no buffers can be made available
 389  * - returns -ENOBUFS if page is beyond EOF
 390  * - if the page is backed by a block in the cache:
 391  *   - a read will be started which will call the callback on completion
 392  *   - 0 will be returned
 393  * - else if the page is unbacked:
 394  *   - the metadata will be retained
 395  *   - -ENODATA will be returned
 396  */
 397 int cachefiles_read_or_alloc_page(struct fscache_retrieval *op,
 398                                   struct page *page,
 399                                   gfp_t gfp)
 400 {
 401         struct cachefiles_object *object;
 402         struct cachefiles_cache *cache;
 403         struct pagevec pagevec;
 404         struct inode *inode;
 405         sector_t block0, block;
 406         unsigned shift;
 407         int ret;
 408
 409         object = container_of(op->op.object,
 410                               struct cachefiles_object, fscache);
 411         cache = container_of(object->fscache.cache,
 412                              struct cachefiles_cache, cache);
 413
 414         _enter("{%p},{%lx},,,", object, page->index);
 415
 416         if (!object->backer)
 417                 goto enobufs;
 418
 419         inode = object->backer->d_inode;
 420         ASSERT(S_ISREG(inode->i_mode));
 421         ASSERT(inode->i_mapping->a_ops->bmap);
 422         ASSERT(inode->i_mapping->a_ops->readpages);
 423
 424         /* calculate the shift required to use bmap */
 425         if (inode->i_sb->s_blocksize > PAGE_SIZE)
 426                 goto enobufs;
 427
 428         shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
 429
 430         op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
 431         op->op.flags |= FSCACHE_OP_ASYNC;
 432         op->op.processor = cachefiles_read_copier;
 433
 434         pagevec_init(&pagevec, 0);
 435
 436         /* we assume the absence or presence of the first block is a good
 437          * enough indication for the page as a whole
 438          * - TODO: don't use bmap() for this as it is _not_ actually good
 439          *   enough for this as it doesn't indicate errors, but it's all we've
 440          *   got for the moment
 441          */
 442         block0 = page->index;
 443         block0 <<= shift;
 444
 445         block = inode->i_mapping->a_ops->bmap(inode->i_mapping, block0);
 446         _debug("%llx -> %llx",
 447                (unsigned long long) block0,
 448                (unsigned long long) block);
 449
 450         if (block) {
 451                 /* submit the apparently valid page to the backing fs to be
 452                  * read from disk */
 453                 ret = cachefiles_read_backing_file_one(object, op, page,
 454                                                        &pagevec);
 455         } else if (cachefiles_has_space(cache, 0, 1) == 0) {
 456                 /* there's space in the cache we can use */
 457                 fscache_mark_page_cached(op, page);
 458                 fscache_retrieval_complete(op, 1);
 459                 ret = -ENODATA;
 460         } else {
 461                 goto enobufs;
 462         }
 463
 464         _leave(" = %d", ret);
 465         return ret;
 466
 467 enobufs:
 468         fscache_retrieval_complete(op, 1);
 469         _leave(" = -ENOBUFS");
 470         return -ENOBUFS;
 471 }
 472
 473 /*
 474  * read the corresponding pages to the given set from the backing file
 475  * - any uncertain pages are simply discarded, to be tried again another time
 476  */
 477 static int cachefiles_read_backing_file(struct cachefiles_object *object,
 478                                         struct fscache_retrieval *op,
 479                                         struct list_head *list)
 480 {
 481         struct cachefiles_one_read *monitor = NULL;
 482         struct address_space *bmapping = object->backer->d_inode->i_mapping;
 483         struct pagevec lru_pvec;
 484         struct page *newpage = NULL, *netpage, *_n, *backpage = NULL;
 485         int ret = 0;
 486
 487         _enter("");
 488
 489         pagevec_init(&lru_pvec, 0);
 490
 491         list_for_each_entry_safe(netpage, _n, list, lru) {
 492                 list_del(&netpage->lru);
 493
 494                 _debug("read back %p{%lu,%d}",
 495                        netpage, netpage->index, page_count(netpage));
 496
 497                 if (!monitor) {
 498                         monitor = kzalloc(sizeof(*monitor), cachefiles_gfp);
 499                         if (!monitor)
 500                                 goto nomem;
 501
 502                         monitor->op = fscache_get_retrieval(op);
 503                         init_waitqueue_func_entry(&monitor->monitor,
 504                                                   cachefiles_read_waiter);
 505                 }
 506
 507                 for (;;) {
 508                         backpage = find_get_page(bmapping, netpage->index);
 509                         if (backpage)
 510                                 goto backing_page_already_present;
 511
 512                         if (!newpage) {
 513                                 newpage = __page_cache_alloc(cachefiles_gfp |
 514                                                              __GFP_COLD);
 515                                 if (!newpage)
 516                                         goto nomem;
 517                         }
 518
 519                         ret = add_to_page_cache(newpage, bmapping,
 520                                                 netpage->index, cachefiles_gfp);
 521                         if (ret == 0)
 522                                 goto installed_new_backing_page;
 523                         if (ret != -EEXIST)
 524                                 goto nomem;
 525                 }
 526
 527                 /* we've installed a new backing page, so now we need to add it
 528                  * to the LRU list and start it reading */
 529         installed_new_backing_page:
 530                 _debug("- new %p", newpage);
 531
 532                 backpage = newpage;
 533                 newpage = NULL;
 534
 535                 page_cache_get(backpage);
 536                 if (!pagevec_add(&lru_pvec, backpage))
 537                         __pagevec_lru_add_file(&lru_pvec);
 538
 539         reread_backing_page:
 540                 ret = bmapping->a_ops->readpage(NULL, backpage);
 541                 if (ret < 0)
 542                         goto read_error;
 543
 544                 /* add the netfs page to the pagecache and LRU, and set the
 545                  * monitor to transfer the data across */
 546         monitor_backing_page:
 547                 _debug("- monitor add");
 548
 549                 ret = add_to_page_cache(netpage, op->mapping, netpage->index,
 550                                         cachefiles_gfp);
 551                 if (ret < 0) {
 552                         if (ret == -EEXIST) {
 553                                 page_cache_release(netpage);
 554                                 continue;
 555                         }
 556                         goto nomem;
 557                 }
 558
 559                 page_cache_get(netpage);
 560                 if (!pagevec_add(&lru_pvec, netpage))
 561                         __pagevec_lru_add_file(&lru_pvec);
 562
 563                 /* install a monitor */
 564                 page_cache_get(netpage);
 565                 monitor->netfs_page = netpage;
 566
 567                 page_cache_get(backpage);
 568                 monitor->back_page = backpage;
 569                 monitor->monitor.private = backpage;
 570                 add_page_wait_queue(backpage, &monitor->monitor);
 571                 monitor = NULL;
 572
 573                 /* but the page may have been read before the monitor was
 574                  * installed, so the monitor may miss the event - so we have to
 575                  * ensure that we do get one in such a case */
 576                 if (trylock_page(backpage)) {
 577                         _debug("2unlock %p {%lx}", backpage, backpage->flags);
 578                         unlock_page(backpage);
 579                 }
 580
 581                 page_cache_release(backpage);
 582                 backpage = NULL;
 583
 584                 page_cache_release(netpage);
 585                 netpage = NULL;
 586                 continue;
 587
 588                 /* if the backing page is already present, it can be in one of
 589                  * three states: read in progress, read failed or read okay */
 590         backing_page_already_present:
 591                 _debug("- present %p", backpage);
 592
 593                 if (PageError(backpage))
 594                         goto io_error;
 595
 596                 if (PageUptodate(backpage))
 597                         goto backing_page_already_uptodate;
 598
 599                 _debug("- not ready %p{%lx}", backpage, backpage->flags);
 600
 601                 if (!trylock_page(backpage))
 602                         goto monitor_backing_page;
 603
 604                 if (PageError(backpage)) {
 605                         _debug("error %lx", backpage->flags);
 606                         unlock_page(backpage);
 607                         goto io_error;
 608                 }
 609
 610                 if (PageUptodate(backpage))
 611                         goto backing_page_already_uptodate_unlock;
 612
 613                 /* we've locked a page that's neither up to date nor erroneous,
 614                  * so we need to attempt to read it again */
 615                 goto reread_backing_page;
 616
 617                 /* the backing page is already up to date, attach the netfs
 618                  * page to the pagecache and LRU and copy the data across */
 619         backing_page_already_uptodate_unlock:
 620                 _debug("uptodate %lx", backpage->flags);
 621                 unlock_page(backpage);
 622         backing_page_already_uptodate:
 623                 _debug("- uptodate");
 624
 625                 ret = add_to_page_cache(netpage, op->mapping, netpage->index,
 626                                         cachefiles_gfp);
 627                 if (ret < 0) {
 628                         if (ret == -EEXIST) {
 629                                 page_cache_release(netpage);
 630                                 continue;
 631                         }
 632                         goto nomem;
 633                 }
 634
 635                 copy_highpage(netpage, backpage);
 636
 637                 page_cache_release(backpage);
 638                 backpage = NULL;
 639
 640                 fscache_mark_page_cached(op, netpage);
 641
 642                 page_cache_get(netpage);
 643                 if (!pagevec_add(&lru_pvec, netpage))
 644                         __pagevec_lru_add_file(&lru_pvec);
 645
 646                 /* the netpage is unlocked and marked up to date here */
 647                 fscache_end_io(op, netpage, 0);
 648                 fscache_retrieval_complete(op, 1);
 649                 page_cache_release(netpage);
 650                 netpage = NULL;
 651                 continue;
 652         }
 653
 654         netpage = NULL;
 655
 656         _debug("out");
 657
 658 out:
 659         /* tidy up */
 660         pagevec_lru_add_file(&lru_pvec);
 661
 662         if (newpage)
 663                 page_cache_release(newpage);
 664         if (netpage)
 665                 page_cache_release(netpage);
 666         if (backpage)
 667                 page_cache_release(backpage);
 668         if (monitor) {
 669                 fscache_put_retrieval(op);
 670                 kfree(monitor);
 671         }
 672
 673         list_for_each_entry_safe(netpage, _n, list, lru) {
 674                 list_del(&netpage->lru);
 675                 page_cache_release(netpage);
 676                 fscache_retrieval_complete(op, 1);
 677         }
 678
 679         _leave(" = %d", ret);
 680         return ret;
 681
 682 nomem:
 683         _debug("nomem");
 684         ret = -ENOMEM;
 685         goto out;
 686
 687 read_error:
 688         _debug("read error %d", ret);
 689         if (ret == -ENOMEM)
 690                 goto out;
 691 io_error:
 692         cachefiles_io_error_obj(object, "Page read error on backing file");
 693         ret = -ENOBUFS;
 694         goto out;
 695 }
 696
 697 /*
 698  * read a list of pages from the cache or allocate blocks in which to store
 699  * them
 700  */
 701 int cachefiles_read_or_alloc_pages(struct fscache_retrieval *op,
 702                                    struct list_head *pages,
 703                                    unsigned *nr_pages,
 704                                    gfp_t gfp)
 705 {
 706         struct cachefiles_object *object;
 707         struct cachefiles_cache *cache;
 708         struct list_head backpages;
 709         struct pagevec pagevec;
 710         struct inode *inode;
 711         struct page *page, *_n;
 712         unsigned shift, nrbackpages;
 713         int ret, ret2, space;
 714
 715         object = container_of(op->op.object,
 716                               struct cachefiles_object, fscache);
 717         cache = container_of(object->fscache.cache,
 718                              struct cachefiles_cache, cache);
 719
 720         _enter("{OBJ%x,%d},,%d,,",
 721                object->fscache.debug_id, atomic_read(&op->op.usage),
 722                *nr_pages);
 723
 724         if (!object->backer)
 725                 goto all_enobufs;
 726
 727         space = 1;
 728         if (cachefiles_has_space(cache, 0, *nr_pages) < 0)
 729                 space = 0;
 730
 731         inode = object->backer->d_inode;
 732         ASSERT(S_ISREG(inode->i_mode));
 733         ASSERT(inode->i_mapping->a_ops->bmap);
 734         ASSERT(inode->i_mapping->a_ops->readpages);
 735
 736         /* calculate the shift required to use bmap */
 737         if (inode->i_sb->s_blocksize > PAGE_SIZE)
 738                 goto all_enobufs;
 739
 740         shift = PAGE_SHIFT - inode->i_sb->s_blocksize_bits;
 741
 742         pagevec_init(&pagevec, 0);
 743
 744         op->op.flags &= FSCACHE_OP_KEEP_FLAGS;
 745         op->op.flags |= FSCACHE_OP_ASYNC;
 746         op->op.processor = cachefiles_read_copier;
 747
 748         INIT_LIST_HEAD(&backpages);
 749         nrbackpages = 0;
 750
 751         ret = space ? -ENODATA : -ENOBUFS;
 752         list_for_each_entry_safe(page, _n, pages, lru) {
 753                 sector_t block0, block;
 754
 755                 /* we assume the absence or presence of the first block is a
 756                  * good enough indication for the page as a whole
 757                  * - TODO: don't use bmap() for this as it is _not_ actually
 758                  *   good enough for this as it doesn't indicate errors, but
 759                  *   it's all we've got for the moment
 760                  */
 761                 block0 = page->index;
 762                 block0 <<= shift;
 763
 764                 block = inode->i_mapping->a_ops->bmap(inode->i_mapping,
 765                                                       block0);
 766                 _debug("%llx -> %llx",
 767                        (unsigned long long) block0,
 768                        (unsigned long long) block);
 769
 770                 if (block) {
 771                         /* we have data - add it to the list to give to the
 772                          * backing fs */
 773                         list_move(&page->lru, &backpages);
 774                         (*nr_pages)--;
 775                         nrbackpages++;
 776                 } else if (space && pagevec_add(&pagevec, page) == 0) {
 777                         fscache_mark_pages_cached(op, &pagevec);
 778                         fscache_retrieval_complete(op, 1);
 779                         ret = -ENODATA;
 780                 } else {
 781                         fscache_retrieval_complete(op, 1);
 782                 }
 783         }
 784
 785         if (pagevec_count(&pagevec) > 0)
 786                 fscache_mark_pages_cached(op, &pagevec);
 787
 788         if (list_empty(pages))
 789                 ret = 0;
 790
 791         /* submit the apparently valid pages to the backing fs to be read from
 792          * disk */
 793         if (nrbackpages > 0) {
 794                 ret2 = cachefiles_read_backing_file(object, op, &backpages);
 795                 if (ret2 == -ENOMEM || ret2 == -EINTR)
 796                         ret = ret2;
 797         }
 798
 799         _leave(" = %d [nr=%u%s]",
 800                ret, *nr_pages, list_empty(pages) ? " empty" : "");
 801         return ret;
 802
 803 all_enobufs:
 804         fscache_retrieval_complete(op, *nr_pages);
 805         return -ENOBUFS;
 806 }
 807
 808 /*
 809  * allocate a block in the cache in which to store a page
 810  * - cache withdrawal is prevented by the caller
 811  * - returns -EINTR if interrupted
 812  * - returns -ENOMEM if ran out of memory
 813  * - returns -ENOBUFS if no buffers can be made available
 814  * - returns -ENOBUFS if page is beyond EOF
 815  * - otherwise:
 816  *   - the metadata will be retained
 817  *   - 0 will be returned
 818  */
 819 int cachefiles_allocate_page(struct fscache_retrieval *op,
 820                              struct page *page,
 821                              gfp_t gfp)
 822 {
 823         struct cachefiles_object *object;
 824         struct cachefiles_cache *cache;
 825         int ret;
 826
 827         object = container_of(op->op.object,
 828                               struct cachefiles_object, fscache);
 829         cache = container_of(object->fscache.cache,
 830                              struct cachefiles_cache, cache);
 831
 832         _enter("%p,{%lx},", object, page->index);
 833
 834         ret = cachefiles_has_space(cache, 0, 1);
 835         if (ret == 0)
 836                 fscache_mark_page_cached(op, page);
 837         else
 838                 ret = -ENOBUFS;
 839
 840         fscache_retrieval_complete(op, 1);
 841         _leave(" = %d", ret);
 842         return ret;
 843 }
 844
 845 /*
 846  * allocate blocks in the cache in which to store a set of pages
 847  * - cache withdrawal is prevented by the caller
 848  * - returns -EINTR if interrupted
 849  * - returns -ENOMEM if ran out of memory
 850  * - returns -ENOBUFS if some buffers couldn't be made available
 851  * - returns -ENOBUFS if some pages are beyond EOF
 852  * - otherwise:
 853  *   - -ENODATA will be returned
 854  * - metadata will be retained for any page marked
 855  */
 856 int cachefiles_allocate_pages(struct fscache_retrieval *op,
 857                               struct list_head *pages,
 858                               unsigned *nr_pages,
 859                               gfp_t gfp)
 860 {
 861         struct cachefiles_object *object;
 862         struct cachefiles_cache *cache;
 863         struct pagevec pagevec;
 864         struct page *page;
 865         int ret;
 866
 867         object = container_of(op->op.object,
 868                               struct cachefiles_object, fscache);
 869         cache = container_of(object->fscache.cache,
 870                              struct cachefiles_cache, cache);
 871
 872         _enter("%p,,,%d,", object, *nr_pages);
 873
 874         ret = cachefiles_has_space(cache, 0, *nr_pages);
 875         if (ret == 0) {
 876                 pagevec_init(&pagevec, 0);
 877
 878                 list_for_each_entry(page, pages, lru) {
 879                         if (pagevec_add(&pagevec, page) == 0)
 880                                 fscache_mark_pages_cached(op, &pagevec);
 881                 }
 882
 883                 if (pagevec_count(&pagevec) > 0)
 884                         fscache_mark_pages_cached(op, &pagevec);
 885                 ret = -ENODATA;
 886         } else {
 887                 ret = -ENOBUFS;
 888         }
 889
 890         fscache_retrieval_complete(op, *nr_pages);
 891         _leave(" = %d", ret);
 892         return ret;
 893 }
 894
 895 /*
 896  * request a page be stored in the cache
 897  * - cache withdrawal is prevented by the caller
 898  * - this request may be ignored if there's no cache block available, in which
 899  *   case -ENOBUFS will be returned
 900  * - if the op is in progress, 0 will be returned
 901  */
 902 int cachefiles_write_page(struct fscache_storage *op, struct page *page)
 903 {
 904         struct cachefiles_object *object;
 905         struct cachefiles_cache *cache;
 906         mm_segment_t old_fs;
 907         struct file *file;
 908         struct path path;
 909         loff_t pos, eof;
 910         size_t len;
 911         void *data;
 912         int ret;
 913
 914         ASSERT(op != NULL);
 915         ASSERT(page != NULL);
 916
 917         object = container_of(op->op.object,
 918                               struct cachefiles_object, fscache);
 919
 920         _enter("%p,%p{%lx},,,", object, page, page->index);
 921
 922         if (!object->backer) {
 923                 _leave(" = -ENOBUFS");
 924                 return -ENOBUFS;
 925         }
 926
 927         ASSERT(S_ISREG(object->backer->d_inode->i_mode));
 928
 929         cache = container_of(object->fscache.cache,
 930                              struct cachefiles_cache, cache);
 931
 932         /* write the page to the backing filesystem and let it store it in its
 933          * own time */
 934         path.mnt = cache->mnt;
 935         path.dentry = object->backer;
 936         file = dentry_open(&path, O_RDWR | O_LARGEFILE, cache->cache_cred);
 937         if (IS_ERR(file)) {
 938                 ret = PTR_ERR(file);
 939         } else {
 940                 ret = -EIO;
 941                 if (file->f_op->write) {
 942                         pos = (loff_t) page->index << PAGE_SHIFT;
 943
 944                         /* we mustn't write more data than we have, so we have
 945                          * to beware of a partial page at EOF */
 946                         eof = object->fscache.store_limit_l;
 947                         len = PAGE_SIZE;
 948                         if (eof & ~PAGE_MASK) {
 949                                 ASSERTCMP(pos, <, eof);
 950                                 if (eof - pos < PAGE_SIZE) {
 951                                         _debug("cut short %llx to %llx",
 952                                                pos, eof);
 953                                         len = eof - pos;
 954                                         ASSERTCMP(pos + len, ==, eof);
 955                                 }
 956                         }
 957
 958                         data = kmap(page);
 959                         old_fs = get_fs();
 960                         set_fs(KERNEL_DS);
 961                         ret = file->f_op->write(
 962                                 file, (const void __user *) data, len, &pos);
 963                         set_fs(old_fs);
 964                         kunmap(page);
 965                         if (ret != len)
 966                                 ret = -EIO;
 967                 }
 968                 fput(file);
 969         }
 970
 971         if (ret < 0) {
 972                 if (ret == -EIO)
 973                         cachefiles_io_error_obj(
 974                                 object, "Write page to backing file failed");
 975                 ret = -ENOBUFS;
 976         }
 977
 978         _leave(" = %d", ret);
 979         return ret;
 980 }
 981
 982 /*
 983  * detach a backing block from a page
 984  * - cache withdrawal is prevented by the caller
 985  */
 986 void cachefiles_uncache_page(struct fscache_object *_object, struct page *page)
 987 {
 988         struct cachefiles_object *object;
 989         struct cachefiles_cache *cache;
 990
 991         object = container_of(_object, struct cachefiles_object, fscache);
 992         cache = container_of(object->fscache.cache,
 993                              struct cachefiles_cache, cache);
 994
 995         _enter("%p,{%lu}", object, page->index);
 996
 997         spin_unlock(&object->fscache.cookie->lock);
 998 }