drivers/xen/grant-table.c

   1 /******************************************************************************
   2  * grant_table.c
   3  *
   4  * Granting foreign access to our memory reservation.
   5  *
   6  * Copyright (c) 2005-2006, Christopher Clark
   7  * Copyright (c) 2004-2005, K A Fraser
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License version 2
  11  * as published by the Free Software Foundation; or, when distributed
  12  * separately from the Linux kernel or incorporated into other
  13  * software packages, subject to the following license:
  14  *
  15  * Permission is hereby granted, free of charge, to any person obtaining a copy
  16  * of this source file (the "Software"), to deal in the Software without
  17  * restriction, including without limitation the rights to use, copy, modify,
  18  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  19  * and to permit persons to whom the Software is furnished to do so, subject to
  20  * the following conditions:
  21  *
  22  * The above copyright notice and this permission notice shall be included in
  23  * all copies or substantial portions of the Software.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  26  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  27  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  28  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  29  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  30  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  31  * IN THE SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/mm.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/uaccess.h>
  40 #include <linux/io.h>
  41 #include <linux/hardirq.h>
  42
  43 #include <xen/xen.h>
  44 #include <xen/interface/xen.h>
  45 #include <xen/page.h>
  46 #include <xen/grant_table.h>
  47 #include <xen/interface/memory.h>
  48 #include <xen/hvc-console.h>
  49 #include <asm/xen/hypercall.h>
  50
  51 #include <asm/pgtable.h>
  52 #include <asm/sync_bitops.h>
  53
  54 /* External tools reserve first few grant table entries. */
  55 #define NR_RESERVED_ENTRIES 8
  56 #define GNTTAB_LIST_END 0xffffffff
  57 #define GREFS_PER_GRANT_FRAME \
  58 (grant_table_version == 1 ?                      \
  59 (PAGE_SIZE / sizeof(struct grant_entry_v1)) :   \
  60 (PAGE_SIZE / sizeof(union grant_entry_v2)))
  61
  62 static grant_ref_t **gnttab_list;
  63 static unsigned int nr_grant_frames;
  64 static unsigned int boot_max_nr_grant_frames;
  65 static int gnttab_free_count;
  66 static grant_ref_t gnttab_free_head;
  67 static DEFINE_SPINLOCK(gnttab_list_lock);
  68 unsigned long xen_hvm_resume_frames;
  69 EXPORT_SYMBOL_GPL(xen_hvm_resume_frames);
  70
  71 static union {
  72         struct grant_entry_v1 *v1;
  73         union grant_entry_v2 *v2;
  74         void *addr;
  75 } gnttab_shared;
  76
  77 /*This is a structure of function pointers for grant table*/
  78 struct gnttab_ops {
  79         /*
  80          * Mapping a list of frames for storing grant entries. Frames parameter
  81          * is used to store grant table address when grant table being setup,
  82          * nr_gframes is the number of frames to map grant table. Returning
  83          * GNTST_okay means success and negative value means failure.
  84          */
  85         int (*map_frames)(unsigned long *frames, unsigned int nr_gframes);
  86         /*
  87          * Release a list of frames which are mapped in map_frames for grant
  88          * entry status.
  89          */
  90         void (*unmap_frames)(void);
  91         /*
  92          * Introducing a valid entry into the grant table, granting the frame of
  93          * this grant entry to domain for accessing or transfering. Ref
  94          * parameter is reference of this introduced grant entry, domid is id of
  95          * granted domain, frame is the page frame to be granted, and flags is
  96          * status of the grant entry to be updated.
  97          */
  98         void (*update_entry)(grant_ref_t ref, domid_t domid,
  99                              unsigned long frame, unsigned flags);
 100         /*
 101          * Stop granting a grant entry to domain for accessing. Ref parameter is
 102          * reference of a grant entry whose grant access will be stopped,
 103          * readonly is not in use in this function. If the grant entry is
 104          * currently mapped for reading or writing, just return failure(==0)
 105          * directly and don't tear down the grant access. Otherwise, stop grant
 106          * access for this entry and return success(==1).
 107          */
 108         int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
 109         /*
 110          * Stop granting a grant entry to domain for transfer. Ref parameter is
 111          * reference of a grant entry whose grant transfer will be stopped. If
 112          * tranfer has not started, just reclaim the grant entry and return
 113          * failure(==0). Otherwise, wait for the transfer to complete and then
 114          * return the frame.
 115          */
 116         unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
 117         /*
 118          * Query the status of a grant entry. Ref parameter is reference of
 119          * queried grant entry, return value is the status of queried entry.
 120          * Detailed status(writing/reading) can be gotten from the return value
 121          * by bit operations.
 122          */
 123         int (*query_foreign_access)(grant_ref_t ref);
 124         /*
 125          * Grant a domain to access a range of bytes within the page referred by
 126          * an available grant entry. Ref parameter is reference of a grant entry
 127          * which will be sub-page accessed, domid is id of grantee domain, frame
 128          * is frame address of subpage grant, flags is grant type and flag
 129          * information, page_off is offset of the range of bytes, and length is
 130          * length of bytes to be accessed.
 131          */
 132         void (*update_subpage_entry)(grant_ref_t ref, domid_t domid,
 133                                      unsigned long frame, int flags,
 134                                      unsigned page_off, unsigned length);
 135         /*
 136          * Redirect an available grant entry on domain A to another grant
 137          * reference of domain B, then allow domain C to use grant reference
 138          * of domain B transitively. Ref parameter is an available grant entry
 139          * reference on domain A, domid is id of domain C which accesses grant
 140          * entry transitively, flags is grant type and flag information,
 141          * trans_domid is id of domain B whose grant entry is finally accessed
 142          * transitively, trans_gref is grant entry transitive reference of
 143          * domain B.
 144          */
 145         void (*update_trans_entry)(grant_ref_t ref, domid_t domid, int flags,
 146                                    domid_t trans_domid, grant_ref_t trans_gref);
 147 };
 148
 149 static struct gnttab_ops *gnttab_interface;
 150
 151 /*This reflects status of grant entries, so act as a global value*/
 152 static grant_status_t *grstatus;
 153
 154 static int grant_table_version;
 155
 156 static struct gnttab_free_callback *gnttab_free_callback_list;
 157
 158 static int gnttab_expand(unsigned int req_entries);
 159
 160 #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
 161 #define SPP (PAGE_SIZE / sizeof(grant_status_t))
 162
 163 static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
 164 {
 165         return &gnttab_list[(entry) / RPP][(entry) % RPP];
 166 }
 167 /* This can be used as an l-value */
 168 #define gnttab_entry(entry) (*__gnttab_entry(entry))
 169
 170 static int get_free_entries(unsigned count)
 171 {
 172         unsigned long flags;
 173         int ref, rc = 0;
 174         grant_ref_t head;
 175
 176         spin_lock_irqsave(&gnttab_list_lock, flags);
 177
 178         if ((gnttab_free_count < count) &&
 179             ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 180                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 181                 return rc;
 182         }
 183
 184         ref = head = gnttab_free_head;
 185         gnttab_free_count -= count;
 186         while (count-- > 1)
 187                 head = gnttab_entry(head);
 188         gnttab_free_head = gnttab_entry(head);
 189         gnttab_entry(head) = GNTTAB_LIST_END;
 190
 191         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 192
 193         return ref;
 194 }
 195
 196 static void do_free_callbacks(void)
 197 {
 198         struct gnttab_free_callback *callback, *next;
 199
 200         callback = gnttab_free_callback_list;
 201         gnttab_free_callback_list = NULL;
 202
 203         while (callback != NULL) {
 204                 next = callback->next;
 205                 if (gnttab_free_count >= callback->count) {
 206                         callback->next = NULL;
 207                         callback->fn(callback->arg);
 208                 } else {
 209                         callback->next = gnttab_free_callback_list;
 210                         gnttab_free_callback_list = callback;
 211                 }
 212                 callback = next;
 213         }
 214 }
 215
 216 static inline void check_free_callbacks(void)
 217 {
 218         if (unlikely(gnttab_free_callback_list))
 219                 do_free_callbacks();
 220 }
 221
 222 static void put_free_entry(grant_ref_t ref)
 223 {
 224         unsigned long flags;
 225         spin_lock_irqsave(&gnttab_list_lock, flags);
 226         gnttab_entry(ref) = gnttab_free_head;
 227         gnttab_free_head = ref;
 228         gnttab_free_count++;
 229         check_free_callbacks();
 230         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 231 }
 232
 233 /*
 234  * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
 235  * Introducing a valid entry into the grant table:
 236  *  1. Write ent->domid.
 237  *  2. Write ent->frame:
 238  *      GTF_permit_access:   Frame to which access is permitted.
 239  *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 240  *                           frame, or zero if none.
 241  *  3. Write memory barrier (WMB).
 242  *  4. Write ent->flags, inc. valid type.
 243  */
 244 static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
 245                                    unsigned long frame, unsigned flags)
 246 {
 247         gnttab_shared.v1[ref].domid = domid;
 248         gnttab_shared.v1[ref].frame = frame;
 249         wmb();
 250         gnttab_shared.v1[ref].flags = flags;
 251 }
 252
 253 static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
 254                                    unsigned long frame, unsigned flags)
 255 {
 256         gnttab_shared.v2[ref].hdr.domid = domid;
 257         gnttab_shared.v2[ref].full_page.frame = frame;
 258         wmb();
 259         gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
 260 }
 261
 262 /*
 263  * Public grant-issuing interface functions
 264  */
 265 void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
 266                                      unsigned long frame, int readonly)
 267 {
 268         gnttab_interface->update_entry(ref, domid, frame,
 269                            GTF_permit_access | (readonly ? GTF_readonly : 0));
 270 }
 271 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
 272
 273 int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
 274                                 int readonly)
 275 {
 276         int ref;
 277
 278         ref = get_free_entries(1);
 279         if (unlikely(ref < 0))
 280                 return -ENOSPC;
 281
 282         gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
 283
 284         return ref;
 285 }
 286 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
 287
 288 void gnttab_update_subpage_entry_v2(grant_ref_t ref, domid_t domid,
 289                                     unsigned long frame, int flags,
 290                                     unsigned page_off,
 291                                     unsigned length)
 292 {
 293         gnttab_shared.v2[ref].sub_page.frame = frame;
 294         gnttab_shared.v2[ref].sub_page.page_off = page_off;
 295         gnttab_shared.v2[ref].sub_page.length = length;
 296         gnttab_shared.v2[ref].hdr.domid = domid;
 297         wmb();
 298         gnttab_shared.v2[ref].hdr.flags =
 299                                 GTF_permit_access | GTF_sub_page | flags;
 300 }
 301
 302 int gnttab_grant_foreign_access_subpage_ref(grant_ref_t ref, domid_t domid,
 303                                             unsigned long frame, int flags,
 304                                             unsigned page_off,
 305                                             unsigned length)
 306 {
 307         if (flags & (GTF_accept_transfer | GTF_reading |
 308                      GTF_writing | GTF_transitive))
 309                 return -EPERM;
 310
 311         if (gnttab_interface->update_subpage_entry == NULL)
 312                 return -ENOSYS;
 313
 314         gnttab_interface->update_subpage_entry(ref, domid, frame, flags,
 315                                                page_off, length);
 316
 317         return 0;
 318 }
 319 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage_ref);
 320
 321 int gnttab_grant_foreign_access_subpage(domid_t domid, unsigned long frame,
 322                                         int flags, unsigned page_off,
 323                                         unsigned length)
 324 {
 325         int ref, rc;
 326
 327         ref = get_free_entries(1);
 328         if (unlikely(ref < 0))
 329                 return -ENOSPC;
 330
 331         rc = gnttab_grant_foreign_access_subpage_ref(ref, domid, frame, flags,
 332                                                      page_off, length);
 333         if (rc < 0) {
 334                 put_free_entry(ref);
 335                 return rc;
 336         }
 337
 338         return ref;
 339 }
 340 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_subpage);
 341
 342 bool gnttab_subpage_grants_available(void)
 343 {
 344         return gnttab_interface->update_subpage_entry != NULL;
 345 }
 346 EXPORT_SYMBOL_GPL(gnttab_subpage_grants_available);
 347
 348 void gnttab_update_trans_entry_v2(grant_ref_t ref, domid_t domid,
 349                                   int flags, domid_t trans_domid,
 350                                   grant_ref_t trans_gref)
 351 {
 352         gnttab_shared.v2[ref].transitive.trans_domid = trans_domid;
 353         gnttab_shared.v2[ref].transitive.gref = trans_gref;
 354         gnttab_shared.v2[ref].hdr.domid = domid;
 355         wmb();
 356         gnttab_shared.v2[ref].hdr.flags =
 357                                 GTF_permit_access | GTF_transitive | flags;
 358 }
 359
 360 int gnttab_grant_foreign_access_trans_ref(grant_ref_t ref, domid_t domid,
 361                                           int flags, domid_t trans_domid,
 362                                           grant_ref_t trans_gref)
 363 {
 364         if (flags & (GTF_accept_transfer | GTF_reading |
 365                      GTF_writing | GTF_sub_page))
 366                 return -EPERM;
 367
 368         if (gnttab_interface->update_trans_entry == NULL)
 369                 return -ENOSYS;
 370
 371         gnttab_interface->update_trans_entry(ref, domid, flags, trans_domid,
 372                                              trans_gref);
 373
 374         return 0;
 375 }
 376 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans_ref);
 377
 378 int gnttab_grant_foreign_access_trans(domid_t domid, int flags,
 379                                       domid_t trans_domid,
 380                                       grant_ref_t trans_gref)
 381 {
 382         int ref, rc;
 383
 384         ref = get_free_entries(1);
 385         if (unlikely(ref < 0))
 386                 return -ENOSPC;
 387
 388         rc = gnttab_grant_foreign_access_trans_ref(ref, domid, flags,
 389                                                    trans_domid, trans_gref);
 390         if (rc < 0) {
 391                 put_free_entry(ref);
 392                 return rc;
 393         }
 394
 395         return ref;
 396 }
 397 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_trans);
 398
 399 bool gnttab_trans_grants_available(void)
 400 {
 401         return gnttab_interface->update_trans_entry != NULL;
 402 }
 403 EXPORT_SYMBOL_GPL(gnttab_trans_grants_available);
 404
 405 static int gnttab_query_foreign_access_v1(grant_ref_t ref)
 406 {
 407         return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
 408 }
 409
 410 static int gnttab_query_foreign_access_v2(grant_ref_t ref)
 411 {
 412         return grstatus[ref] & (GTF_reading|GTF_writing);
 413 }
 414
 415 int gnttab_query_foreign_access(grant_ref_t ref)
 416 {
 417         return gnttab_interface->query_foreign_access(ref);
 418 }
 419 EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
 420
 421 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
 422 {
 423         u16 flags, nflags;
 424         u16 *pflags;
 425
 426         pflags = &gnttab_shared.v1[ref].flags;
 427         nflags = *pflags;
 428         do {
 429                 flags = nflags;
 430                 if (flags & (GTF_reading|GTF_writing))
 431                         return 0;
 432         } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
 433
 434         return 1;
 435 }
 436
 437 static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
 438 {
 439         gnttab_shared.v2[ref].hdr.flags = 0;
 440         mb();
 441         if (grstatus[ref] & (GTF_reading|GTF_writing)) {
 442                 return 0;
 443         } else {
 444                 /* The read of grstatus needs to have acquire
 445                 semantics.  On x86, reads already have
 446                 that, and we just need to protect against
 447                 compiler reorderings.  On other
 448                 architectures we may need a full
 449                 barrier. */
 450 #ifdef CONFIG_X86
 451                 barrier();
 452 #else
 453                 mb();
 454 #endif
 455         }
 456
 457         return 1;
 458 }
 459
 460 static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 461 {
 462         return gnttab_interface->end_foreign_access_ref(ref, readonly);
 463 }
 464
 465 int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
 466 {
 467         if (_gnttab_end_foreign_access_ref(ref, readonly))
 468                 return 1;
 469         pr_warn("WARNING: g.e. %#x still in use!\n", ref);
 470         return 0;
 471 }
 472 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
 473
 474 struct deferred_entry {
 475         struct list_head list;
 476         grant_ref_t ref;
 477         bool ro;
 478         uint16_t warn_delay;
 479         struct page *page;
 480 };
 481 static LIST_HEAD(deferred_list);
 482 static void gnttab_handle_deferred(unsigned long);
 483 static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred, 0, 0);
 484
 485 static void gnttab_handle_deferred(unsigned long unused)
 486 {
 487         unsigned int nr = 10;
 488         struct deferred_entry *first = NULL;
 489         unsigned long flags;
 490
 491         spin_lock_irqsave(&gnttab_list_lock, flags);
 492         while (nr--) {
 493                 struct deferred_entry *entry
 494                         = list_first_entry(&deferred_list,
 495                                            struct deferred_entry, list);
 496
 497                 if (entry == first)
 498                         break;
 499                 list_del(&entry->list);
 500                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 501                 if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
 502                         put_free_entry(entry->ref);
 503                         if (entry->page) {
 504                                 pr_debug("freeing g.e. %#x (pfn %#lx)\n",
 505                                          entry->ref, page_to_pfn(entry->page));
 506                                 __free_page(entry->page);
 507                         } else
 508                                 pr_info("freeing g.e. %#x\n", entry->ref);
 509                         kfree(entry);
 510                         entry = NULL;
 511                 } else {
 512                         if (!--entry->warn_delay)
 513                                 pr_info("g.e. %#x still pending\n",
 514                                         entry->ref);
 515                         if (!first)
 516                                 first = entry;
 517                 }
 518                 spin_lock_irqsave(&gnttab_list_lock, flags);
 519                 if (entry)
 520                         list_add_tail(&entry->list, &deferred_list);
 521                 else if (list_empty(&deferred_list))
 522                         break;
 523         }
 524         if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
 525                 deferred_timer.expires = jiffies + HZ;
 526                 add_timer(&deferred_timer);
 527         }
 528         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 529 }
 530
 531 static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
 532                                 struct page *page)
 533 {
 534         struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 535         const char *what = KERN_WARNING "leaking";
 536
 537         if (entry) {
 538                 unsigned long flags;
 539
 540                 entry->ref = ref;
 541                 entry->ro = readonly;
 542                 entry->page = page;
 543                 entry->warn_delay = 60;
 544                 spin_lock_irqsave(&gnttab_list_lock, flags);
 545                 list_add_tail(&entry->list, &deferred_list);
 546                 if (!timer_pending(&deferred_timer)) {
 547                         deferred_timer.expires = jiffies + HZ;
 548                         add_timer(&deferred_timer);
 549                 }
 550                 spin_unlock_irqrestore(&gnttab_list_lock, flags);
 551                 what = KERN_DEBUG "deferring";
 552         }
 553         printk("%s g.e. %#x (pfn %#lx)\n",
 554                what, ref, page ? page_to_pfn(page) : -1);
 555 }
 556
 557 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
 558                                unsigned long page)
 559 {
 560         if (gnttab_end_foreign_access_ref(ref, readonly)) {
 561                 put_free_entry(ref);
 562                 if (page != 0)
 563                         free_page(page);
 564         } else
 565                 gnttab_add_deferred(ref, readonly,
 566                                     page ? virt_to_page(page) : NULL);
 567 }
 568 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
 569
 570 int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
 571 {
 572         int ref;
 573
 574         ref = get_free_entries(1);
 575         if (unlikely(ref < 0))
 576                 return -ENOSPC;
 577         gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 578
 579         return ref;
 580 }
 581 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
 582
 583 void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
 584                                        unsigned long pfn)
 585 {
 586         gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
 587 }
 588 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
 589
 590 static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
 591 {
 592         unsigned long frame;
 593         u16           flags;
 594         u16          *pflags;
 595
 596         pflags = &gnttab_shared.v1[ref].flags;
 597
 598         /*
 599          * If a transfer is not even yet started, try to reclaim the grant
 600          * reference and return failure (== 0).
 601          */
 602         while (!((flags = *pflags) & GTF_transfer_committed)) {
 603                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 604                         return 0;
 605                 cpu_relax();
 606         }
 607
 608         /* If a transfer is in progress then wait until it is completed. */
 609         while (!(flags & GTF_transfer_completed)) {
 610                 flags = *pflags;
 611                 cpu_relax();
 612         }
 613
 614         rmb();  /* Read the frame number /after/ reading completion status. */
 615         frame = gnttab_shared.v1[ref].frame;
 616         BUG_ON(frame == 0);
 617
 618         return frame;
 619 }
 620
 621 static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
 622 {
 623         unsigned long frame;
 624         u16           flags;
 625         u16          *pflags;
 626
 627         pflags = &gnttab_shared.v2[ref].hdr.flags;
 628
 629         /*
 630          * If a transfer is not even yet started, try to reclaim the grant
 631          * reference and return failure (== 0).
 632          */
 633         while (!((flags = *pflags) & GTF_transfer_committed)) {
 634                 if (sync_cmpxchg(pflags, flags, 0) == flags)
 635                         return 0;
 636                 cpu_relax();
 637         }
 638
 639         /* If a transfer is in progress then wait until it is completed. */
 640         while (!(flags & GTF_transfer_completed)) {
 641                 flags = *pflags;
 642                 cpu_relax();
 643         }
 644
 645         rmb();  /* Read the frame number /after/ reading completion status. */
 646         frame = gnttab_shared.v2[ref].full_page.frame;
 647         BUG_ON(frame == 0);
 648
 649         return frame;
 650 }
 651
 652 unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
 653 {
 654         return gnttab_interface->end_foreign_transfer_ref(ref);
 655 }
 656 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
 657
 658 unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
 659 {
 660         unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
 661         put_free_entry(ref);
 662         return frame;
 663 }
 664 EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
 665
 666 void gnttab_free_grant_reference(grant_ref_t ref)
 667 {
 668         put_free_entry(ref);
 669 }
 670 EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
 671
 672 void gnttab_free_grant_references(grant_ref_t head)
 673 {
 674         grant_ref_t ref;
 675         unsigned long flags;
 676         int count = 1;
 677         if (head == GNTTAB_LIST_END)
 678                 return;
 679         spin_lock_irqsave(&gnttab_list_lock, flags);
 680         ref = head;
 681         while (gnttab_entry(ref) != GNTTAB_LIST_END) {
 682                 ref = gnttab_entry(ref);
 683                 count++;
 684         }
 685         gnttab_entry(ref) = gnttab_free_head;
 686         gnttab_free_head = head;
 687         gnttab_free_count += count;
 688         check_free_callbacks();
 689         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 690 }
 691 EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
 692
 693 int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
 694 {
 695         int h = get_free_entries(count);
 696
 697         if (h < 0)
 698                 return -ENOSPC;
 699
 700         *head = h;
 701
 702         return 0;
 703 }
 704 EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
 705
 706 int gnttab_empty_grant_references(const grant_ref_t *private_head)
 707 {
 708         return (*private_head == GNTTAB_LIST_END);
 709 }
 710 EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
 711
 712 int gnttab_claim_grant_reference(grant_ref_t *private_head)
 713 {
 714         grant_ref_t g = *private_head;
 715         if (unlikely(g == GNTTAB_LIST_END))
 716                 return -ENOSPC;
 717         *private_head = gnttab_entry(g);
 718         return g;
 719 }
 720 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
 721
 722 void gnttab_release_grant_reference(grant_ref_t *private_head,
 723                                     grant_ref_t release)
 724 {
 725         gnttab_entry(release) = *private_head;
 726         *private_head = release;
 727 }
 728 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
 729
 730 void gnttab_request_free_callback(struct gnttab_free_callback *callback,
 731                                   void (*fn)(void *), void *arg, u16 count)
 732 {
 733         unsigned long flags;
 734         spin_lock_irqsave(&gnttab_list_lock, flags);
 735         if (callback->next)
 736                 goto out;
 737         callback->fn = fn;
 738         callback->arg = arg;
 739         callback->count = count;
 740         callback->next = gnttab_free_callback_list;
 741         gnttab_free_callback_list = callback;
 742         check_free_callbacks();
 743 out:
 744         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 745 }
 746 EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
 747
 748 void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
 749 {
 750         struct gnttab_free_callback **pcb;
 751         unsigned long flags;
 752
 753         spin_lock_irqsave(&gnttab_list_lock, flags);
 754         for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
 755                 if (*pcb == callback) {
 756                         *pcb = callback->next;
 757                         break;
 758                 }
 759         }
 760         spin_unlock_irqrestore(&gnttab_list_lock, flags);
 761 }
 762 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 763
 764 static int grow_gnttab_list(unsigned int more_frames)
 765 {
 766         unsigned int new_nr_grant_frames, extra_entries, i;
 767         unsigned int nr_glist_frames, new_nr_glist_frames;
 768
 769         new_nr_grant_frames = nr_grant_frames + more_frames;
 770         extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
 771
 772         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 773         new_nr_glist_frames =
 774                 (new_nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
 775         for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
 776                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
 777                 if (!gnttab_list[i])
 778                         goto grow_nomem;
 779         }
 780
 781
 782         for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 783              i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
 784                 gnttab_entry(i) = i + 1;
 785
 786         gnttab_entry(i) = gnttab_free_head;
 787         gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
 788         gnttab_free_count += extra_entries;
 789
 790         nr_grant_frames = new_nr_grant_frames;
 791
 792         check_free_callbacks();
 793
 794         return 0;
 795
 796 grow_nomem:
 797         for ( ; i >= nr_glist_frames; i--)
 798                 free_page((unsigned long) gnttab_list[i]);
 799         return -ENOMEM;
 800 }
 801
 802 static unsigned int __max_nr_grant_frames(void)
 803 {
 804         struct gnttab_query_size query;
 805         int rc;
 806
 807         query.dom = DOMID_SELF;
 808
 809         rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
 810         if ((rc < 0) || (query.status != GNTST_okay))
 811                 return 4; /* Legacy max supported number of frames */
 812
 813         return query.max_nr_frames;
 814 }
 815
 816 unsigned int gnttab_max_grant_frames(void)
 817 {
 818         unsigned int xen_max = __max_nr_grant_frames();
 819
 820         if (xen_max > boot_max_nr_grant_frames)
 821                 return boot_max_nr_grant_frames;
 822         return xen_max;
 823 }
 824 EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
 825
 826 int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
 827                     struct gnttab_map_grant_ref *kmap_ops,
 828                     struct page **pages, unsigned int count)
 829 {
 830         int i, ret;
 831         bool lazy = false;
 832         pte_t *pte;
 833         unsigned long mfn;
 834
 835         ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
 836         if (ret)
 837                 return ret;
 838
 839         if (xen_feature(XENFEAT_auto_translated_physmap))
 840                 return ret;
 841
 842         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 843                 arch_enter_lazy_mmu_mode();
 844                 lazy = true;
 845         }
 846
 847         for (i = 0; i < count; i++) {
 848                 /* Do not add to override if the map failed. */
 849                 if (map_ops[i].status)
 850                         continue;
 851
 852                 if (map_ops[i].flags & GNTMAP_contains_pte) {
 853                         pte = (pte_t *) (mfn_to_virt(PFN_DOWN(map_ops[i].host_addr)) +
 854                                 (map_ops[i].host_addr & ~PAGE_MASK));
 855                         mfn = pte_mfn(*pte);
 856                 } else {
 857                         mfn = PFN_DOWN(map_ops[i].dev_bus_addr);
 858                 }
 859                 ret = m2p_add_override(mfn, pages[i], kmap_ops ?
 860                                        &kmap_ops[i] : NULL);
 861                 if (ret)
 862                         return ret;
 863         }
 864
 865         if (lazy)
 866                 arch_leave_lazy_mmu_mode();
 867
 868         return ret;
 869 }
 870 EXPORT_SYMBOL_GPL(gnttab_map_refs);
 871
 872 int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
 873                       struct page **pages, unsigned int count, bool clear_pte)
 874 {
 875         int i, ret;
 876         bool lazy = false;
 877
 878         ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
 879         if (ret)
 880                 return ret;
 881
 882         if (xen_feature(XENFEAT_auto_translated_physmap))
 883                 return ret;
 884
 885         if (!in_interrupt() && paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
 886                 arch_enter_lazy_mmu_mode();
 887                 lazy = true;
 888         }
 889
 890         for (i = 0; i < count; i++) {
 891                 ret = m2p_remove_override(pages[i], clear_pte);
 892                 if (ret)
 893                         return ret;
 894         }
 895
 896         if (lazy)
 897                 arch_leave_lazy_mmu_mode();
 898
 899         return ret;
 900 }
 901 EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
 902
 903 static unsigned nr_status_frames(unsigned nr_grant_frames)
 904 {
 905         return (nr_grant_frames * GREFS_PER_GRANT_FRAME + SPP - 1) / SPP;
 906 }
 907
 908 static int gnttab_map_frames_v1(unsigned long *frames, unsigned int nr_gframes)
 909 {
 910         int rc;
 911
 912         rc = arch_gnttab_map_shared(frames, nr_gframes,
 913                                     gnttab_max_grant_frames(),
 914                                     &gnttab_shared.addr);
 915         BUG_ON(rc);
 916
 917         return 0;
 918 }
 919
 920 static void gnttab_unmap_frames_v1(void)
 921 {
 922         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 923 }
 924
 925 static int gnttab_map_frames_v2(unsigned long *frames, unsigned int nr_gframes)
 926 {
 927         uint64_t *sframes;
 928         unsigned int nr_sframes;
 929         struct gnttab_get_status_frames getframes;
 930         int rc;
 931
 932         nr_sframes = nr_status_frames(nr_gframes);
 933
 934         /* No need for kzalloc as it is initialized in following hypercall
 935          * GNTTABOP_get_status_frames.
 936          */
 937         sframes = kmalloc(nr_sframes  * sizeof(uint64_t), GFP_ATOMIC);
 938         if (!sframes)
 939                 return -ENOMEM;
 940
 941         getframes.dom        = DOMID_SELF;
 942         getframes.nr_frames  = nr_sframes;
 943         set_xen_guest_handle(getframes.frame_list, sframes);
 944
 945         rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
 946                                        &getframes, 1);
 947         if (rc == -ENOSYS) {
 948                 kfree(sframes);
 949                 return -ENOSYS;
 950         }
 951
 952         BUG_ON(rc || getframes.status);
 953
 954         rc = arch_gnttab_map_status(sframes, nr_sframes,
 955                                     nr_status_frames(gnttab_max_grant_frames()),
 956                                     &grstatus);
 957         BUG_ON(rc);
 958         kfree(sframes);
 959
 960         rc = arch_gnttab_map_shared(frames, nr_gframes,
 961                                     gnttab_max_grant_frames(),
 962                                     &gnttab_shared.addr);
 963         BUG_ON(rc);
 964
 965         return 0;
 966 }
 967
 968 static void gnttab_unmap_frames_v2(void)
 969 {
 970         arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
 971         arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
 972 }
 973
 974 static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 975 {
 976         struct gnttab_setup_table setup;
 977         unsigned long *frames;
 978         unsigned int nr_gframes = end_idx + 1;
 979         int rc;
 980
 981         if (xen_hvm_domain()) {
 982                 struct xen_add_to_physmap xatp;
 983                 unsigned int i = end_idx;
 984                 rc = 0;
 985                 /*
 986                  * Loop backwards, so that the first hypercall has the largest
 987                  * index, ensuring that the table will grow only once.
 988                  */
 989                 do {
 990                         xatp.domid = DOMID_SELF;
 991                         xatp.idx = i;
 992                         xatp.space = XENMAPSPACE_grant_table;
 993                         xatp.gpfn = (xen_hvm_resume_frames >> PAGE_SHIFT) + i;
 994                         rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
 995                         if (rc != 0) {
 996                                 printk(KERN_WARNING
 997                                                 "grant table add_to_physmap failed, err=%d\n", rc);
 998                                 break;
 999                         }
1000                 } while (i-- > start_idx);
1001
1002                 return rc;
1003         }
1004
1005         /* No need for kzalloc as it is initialized in following hypercall
1006          * GNTTABOP_setup_table.
1007          */
1008         frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
1009         if (!frames)
1010                 return -ENOMEM;
1011
1012         setup.dom        = DOMID_SELF;
1013         setup.nr_frames  = nr_gframes;
1014         set_xen_guest_handle(setup.frame_list, frames);
1015
1016         rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
1017         if (rc == -ENOSYS) {
1018                 kfree(frames);
1019                 return -ENOSYS;
1020         }
1021
1022         BUG_ON(rc || setup.status);
1023
1024         rc = gnttab_interface->map_frames(frames, nr_gframes);
1025
1026         kfree(frames);
1027
1028         return rc;
1029 }
1030
1031 static struct gnttab_ops gnttab_v1_ops = {
1032         .map_frames                     = gnttab_map_frames_v1,
1033         .unmap_frames                   = gnttab_unmap_frames_v1,
1034         .update_entry                   = gnttab_update_entry_v1,
1035         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v1,
1036         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v1,
1037         .query_foreign_access           = gnttab_query_foreign_access_v1,
1038 };
1039
1040 static struct gnttab_ops gnttab_v2_ops = {
1041         .map_frames                     = gnttab_map_frames_v2,
1042         .unmap_frames                   = gnttab_unmap_frames_v2,
1043         .update_entry                   = gnttab_update_entry_v2,
1044         .end_foreign_access_ref         = gnttab_end_foreign_access_ref_v2,
1045         .end_foreign_transfer_ref       = gnttab_end_foreign_transfer_ref_v2,
1046         .query_foreign_access           = gnttab_query_foreign_access_v2,
1047         .update_subpage_entry           = gnttab_update_subpage_entry_v2,
1048         .update_trans_entry             = gnttab_update_trans_entry_v2,
1049 };
1050
1051 static void gnttab_request_version(void)
1052 {
1053         int rc;
1054         struct gnttab_set_version gsv;
1055
1056         if (xen_hvm_domain())
1057                 gsv.version = 1;
1058         else
1059                 gsv.version = 2;
1060         rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
1061         if (rc == 0 && gsv.version == 2) {
1062                 grant_table_version = 2;
1063                 gnttab_interface = &gnttab_v2_ops;
1064         } else if (grant_table_version == 2) {
1065                 /*
1066                  * If we've already used version 2 features,
1067                  * but then suddenly discover that they're not
1068                  * available (e.g. migrating to an older
1069                  * version of Xen), almost unbounded badness
1070                  * can happen.
1071                  */
1072                 panic("we need grant tables version 2, but only version 1 is available");
1073         } else {
1074                 grant_table_version = 1;
1075                 gnttab_interface = &gnttab_v1_ops;
1076         }
1077         printk(KERN_INFO "Grant tables using version %d layout.\n",
1078                 grant_table_version);
1079 }
1080
1081 int gnttab_resume(void)
1082 {
1083         unsigned int max_nr_gframes;
1084
1085         gnttab_request_version();
1086         max_nr_gframes = gnttab_max_grant_frames();
1087         if (max_nr_gframes < nr_grant_frames)
1088                 return -ENOSYS;
1089
1090         if (xen_pv_domain())
1091                 return gnttab_map(0, nr_grant_frames - 1);
1092
1093         if (gnttab_shared.addr == NULL) {
1094                 gnttab_shared.addr = ioremap(xen_hvm_resume_frames,
1095                                                 PAGE_SIZE * max_nr_gframes);
1096                 if (gnttab_shared.addr == NULL) {
1097                         printk(KERN_WARNING
1098                                         "Failed to ioremap gnttab share frames!");
1099                         return -ENOMEM;
1100                 }
1101         }
1102
1103         gnttab_map(0, nr_grant_frames - 1);
1104
1105         return 0;
1106 }
1107
1108 int gnttab_suspend(void)
1109 {
1110         gnttab_interface->unmap_frames();
1111         return 0;
1112 }
1113
1114 static int gnttab_expand(unsigned int req_entries)
1115 {
1116         int rc;
1117         unsigned int cur, extra;
1118
1119         cur = nr_grant_frames;
1120         extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
1121                  GREFS_PER_GRANT_FRAME);
1122         if (cur + extra > gnttab_max_grant_frames())
1123                 return -ENOSPC;
1124
1125         rc = gnttab_map(cur, cur + extra - 1);
1126         if (rc == 0)
1127                 rc = grow_gnttab_list(extra);
1128
1129         return rc;
1130 }
1131
1132 int gnttab_init(void)
1133 {
1134         int i;
1135         unsigned int max_nr_glist_frames, nr_glist_frames;
1136         unsigned int nr_init_grefs;
1137         int ret;
1138
1139         nr_grant_frames = 1;
1140         boot_max_nr_grant_frames = __max_nr_grant_frames();
1141
1142         /* Determine the maximum number of frames required for the
1143          * grant reference free list on the current hypervisor.
1144          */
1145         max_nr_glist_frames = (boot_max_nr_grant_frames *
1146                                GREFS_PER_GRANT_FRAME / RPP);
1147
1148         gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
1149                               GFP_KERNEL);
1150         if (gnttab_list == NULL)
1151                 return -ENOMEM;
1152
1153         nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) / RPP;
1154         for (i = 0; i < nr_glist_frames; i++) {
1155                 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
1156                 if (gnttab_list[i] == NULL) {
1157                         ret = -ENOMEM;
1158                         goto ini_nomem;
1159                 }
1160         }
1161
1162         if (gnttab_resume() < 0) {
1163                 ret = -ENODEV;
1164                 goto ini_nomem;
1165         }
1166
1167         nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
1168
1169         for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
1170                 gnttab_entry(i) = i + 1;
1171
1172         gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
1173         gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
1174         gnttab_free_head  = NR_RESERVED_ENTRIES;
1175
1176         printk("Grant table initialized\n");
1177         return 0;
1178
1179  ini_nomem:
1180         for (i--; i >= 0; i--)
1181                 free_page((unsigned long)gnttab_list[i]);
1182         kfree(gnttab_list);
1183         return ret;
1184 }
1185 EXPORT_SYMBOL_GPL(gnttab_init);
1186
1187 static int __devinit __gnttab_init(void)
1188 {
1189         /* Delay grant-table initialization in the PV on HVM case */
1190         if (xen_hvm_domain())
1191                 return 0;
1192
1193         if (!xen_pv_domain())
1194                 return -ENODEV;
1195
1196         return gnttab_init();
1197 }
1198
1199 core_initcall(__gnttab_init);