1 /* drivers/android/pmem.c
3 * Copyright (C) 2007 Google, Inc.
5 * This software is licensed under the terms of the GNU General Public
6 * License version 2, as published by the Free Software Foundation, and
7 * may be copied, distributed, and modified under those terms.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
16 #include <linux/miscdevice.h>
17 #include <linux/platform_device.h>
19 #include <linux/file.h>
21 #include <linux/list.h>
22 #include <linux/debugfs.h>
23 #include <linux/android_pmem.h>
24 #include <linux/mempolicy.h>
25 #include <linux/sched.h>
26 #include <linux/slab.h>
28 #include <linux/uaccess.h>
29 #include <asm/cacheflush.h>
31 #define PMEM_MAX_DEVICES 10
32 #define PMEM_MAX_ORDER 128
33 #define PMEM_MIN_ALLOC PAGE_SIZE
37 /* indicates that a refernce to this file has been taken via get_pmem_file,
38 * the file should not be released until put_pmem_file is called */
39 #define PMEM_FLAGS_BUSY 0x1
40 /* indicates that this is a suballocation of a larger master range */
41 #define PMEM_FLAGS_CONNECTED (0x1 << 1)
42 /* indicates this is a master and not a sub allocation and that it is mmaped */
43 #define PMEM_FLAGS_MASTERMAP (0x1 << 2)
44 /* submap and unsubmap flags indicate:
45 * 00: subregion has never been mmaped
46 * 10: subregion has been mmaped, reference to the mm was taken
47 * 11: subretion has ben released, refernece to the mm still held
48 * 01: subretion has been released, reference to the mm has been released
50 #define PMEM_FLAGS_SUBMAP (0x1 << 3)
51 #define PMEM_FLAGS_UNSUBMAP (0x1 << 4)
55 /* in alloc mode: an index into the bitmap
56 * in no_alloc mode: the size of the allocation */
58 /* see flags above for descriptions */
60 /* protects this data field, if the mm_mmap sem will be held at the
61 * same time as this sem, the mm sem must be taken first (as this is
62 * the order for vma_open and vma_close ops */
63 struct rw_semaphore sem;
64 /* info about the mmaping process */
65 struct vm_area_struct *vma;
66 /* task struct of the mapping process */
67 struct task_struct *task;
68 /* process id of teh mapping process */
70 /* file descriptor of the master */
72 /* file struct of the master */
73 struct file *master_file;
74 /* a list of currently available regions if this is a suballocation */
75 struct list_head region_list;
76 /* a linked list of data so we can access them for debugging */
77 struct list_head list;
84 unsigned allocated:1; /* 1 if allocated, 0 if free */
85 unsigned order:7; /* size of the region in pmem space */
88 struct pmem_region_node {
89 struct pmem_region region;
90 struct list_head list;
93 #define PMEM_DEBUG_MSGS 0
95 #define DLOG(fmt, args...) \
96 do { printk(KERN_INFO "[%s:%s:%d] "fmt, __FILE__, __func__, __LINE__, \
100 #define DLOG(x...) do {} while (0)
104 struct miscdevice dev;
105 /* physical start address of the remaped pmem space */
107 /* vitual start address of the remaped pmem space */
108 unsigned char __iomem *vbase;
109 /* total size of the pmem space */
111 /* number of entries in the pmem space */
112 unsigned long num_entries;
113 /* pfn of the garbage page in memory */
114 unsigned long garbage_pfn;
115 /* index of the garbage page in the pmem space */
117 /* the bitmap for the region indicating which entries are allocated
118 * and which are free */
119 struct pmem_bits *bitmap;
120 /* indicates the region should not be managed with an allocator */
121 unsigned no_allocator;
122 /* indicates maps of this region should be cached, if a mix of
123 * cached and uncached is desired, set this and open the device with
124 * O_SYNC to get an uncached region */
127 /* in no_allocator mode the first mapper gets the whole space and sets
130 /* for debugging, creates a list of pmem file structs, the
131 * data_list_sem should be taken before pmem_data->sem if both are
133 struct semaphore data_list_sem;
134 struct list_head data_list;
135 /* pmem_sem protects the bitmap array
136 * a write lock should be held when modifying entries in bitmap
137 * a read lock should be held when reading data from bits or
138 * dereferencing a pointer into bitmap
140 * pmem_data->sem protects the pmem data of a particular file
141 * Many of the function that require the pmem_data->sem have a non-
142 * locking version for when the caller is already holding that sem.
144 * IF YOU TAKE BOTH LOCKS TAKE THEM IN THIS ORDER:
145 * down(pmem_data->sem) => down(bitmap_sem)
147 struct rw_semaphore bitmap_sem;
149 long (*ioctl)(struct file *, unsigned int, unsigned long);
150 int (*release)(struct inode *, struct file *);
153 static struct pmem_info pmem[PMEM_MAX_DEVICES];
156 #define PMEM_IS_FREE(id, index) (!(pmem[id].bitmap[index].allocated))
157 #define PMEM_ORDER(id, index) pmem[id].bitmap[index].order
158 #define PMEM_BUDDY_INDEX(id, index) (index ^ (1 << PMEM_ORDER(id, index)))
159 #define PMEM_NEXT_INDEX(id, index) (index + (1 << PMEM_ORDER(id, index)))
160 #define PMEM_OFFSET(index) (index * PMEM_MIN_ALLOC)
161 #define PMEM_START_ADDR(id, index) (PMEM_OFFSET(index) + pmem[id].base)
162 #define PMEM_LEN(id, index) ((1 << PMEM_ORDER(id, index)) * PMEM_MIN_ALLOC)
163 #define PMEM_END_ADDR(id, index) (PMEM_START_ADDR(id, index) + \
165 #define PMEM_START_VADDR(id, index) (PMEM_OFFSET(id, index) + pmem[id].vbase)
166 #define PMEM_END_VADDR(id, index) (PMEM_START_VADDR(id, index) + \
168 #define PMEM_REVOKED(data) (data->flags & PMEM_FLAGS_REVOKED)
169 #define PMEM_IS_PAGE_ALIGNED(addr) (!((addr) & (~PAGE_MASK)))
170 #define PMEM_IS_SUBMAP(data) ((data->flags & PMEM_FLAGS_SUBMAP) && \
171 (!(data->flags & PMEM_FLAGS_UNSUBMAP)))
173 static int pmem_release(struct inode *, struct file *);
174 static int pmem_mmap(struct file *, struct vm_area_struct *);
175 static int pmem_open(struct inode *, struct file *);
176 static long pmem_ioctl(struct file *, unsigned int, unsigned long);
178 const struct file_operations pmem_fops = {
179 .release = pmem_release,
182 .unlocked_ioctl = pmem_ioctl,
183 .llseek = noop_llseek,
186 static int get_id(struct file *file)
188 return MINOR(file->f_dentry->d_inode->i_rdev);
191 static int is_pmem_file(struct file *file)
195 if (unlikely(!file || !file->f_dentry || !file->f_dentry->d_inode))
198 if (unlikely(id >= PMEM_MAX_DEVICES))
200 if (unlikely(file->f_dentry->d_inode->i_rdev !=
201 MKDEV(MISC_MAJOR, pmem[id].dev.minor)))
206 static int has_allocation(struct file *file)
208 struct pmem_data *data;
209 /* check is_pmem_file first if not accessed via pmem_file_ops */
211 if (unlikely(!file->private_data))
213 data = file->private_data;
214 if (unlikely(data->index < 0))
219 static int is_master_owner(struct file *file)
221 struct file *master_file;
222 struct pmem_data *data;
223 int put_needed, ret = 0;
225 if (!is_pmem_file(file) || !has_allocation(file))
227 data = file->private_data;
228 if (PMEM_FLAGS_MASTERMAP & data->flags)
230 master_file = fget_light(data->master_fd, &put_needed);
231 if (master_file && data->master_file == master_file)
233 fput_light(master_file, put_needed);
237 static int pmem_free(int id, int index)
239 /* caller should hold the write lock on pmem_sem! */
240 int buddy, curr = index;
241 DLOG("index %d\n", index);
243 if (pmem[id].no_allocator) {
244 pmem[id].allocated = 0;
247 /* clean up the bitmap, merging any buddies */
248 pmem[id].bitmap[curr].allocated = 0;
249 /* find a slots buddy Buddy# = Slot# ^ (1 << order)
250 * if the buddy is also free merge them
251 * repeat until the buddy is not free or end of the bitmap is reached
254 buddy = PMEM_BUDDY_INDEX(id, curr);
255 if (PMEM_IS_FREE(id, buddy) &&
256 PMEM_ORDER(id, buddy) == PMEM_ORDER(id, curr)) {
257 PMEM_ORDER(id, buddy)++;
258 PMEM_ORDER(id, curr)++;
259 curr = min(buddy, curr);
263 } while (curr < pmem[id].num_entries);
268 static void pmem_revoke(struct file *file, struct pmem_data *data);
270 static int pmem_release(struct inode *inode, struct file *file)
272 struct pmem_data *data = file->private_data;
273 struct pmem_region_node *region_node;
274 struct list_head *elt, *elt2;
275 int id = get_id(file), ret = 0;
278 down(&pmem[id].data_list_sem);
279 /* if this file is a master, revoke all the memory in the connected
281 if (PMEM_FLAGS_MASTERMAP & data->flags) {
282 struct pmem_data *sub_data;
283 list_for_each(elt, &pmem[id].data_list) {
284 sub_data = list_entry(elt, struct pmem_data, list);
285 down_read(&sub_data->sem);
286 if (PMEM_IS_SUBMAP(sub_data) &&
287 file == sub_data->master_file) {
288 up_read(&sub_data->sem);
289 pmem_revoke(file, sub_data);
291 up_read(&sub_data->sem);
294 list_del(&data->list);
295 up(&pmem[id].data_list_sem);
298 down_write(&data->sem);
300 /* if its not a conencted file and it has an allocation, free it */
301 if (!(PMEM_FLAGS_CONNECTED & data->flags) && has_allocation(file)) {
302 down_write(&pmem[id].bitmap_sem);
303 ret = pmem_free(id, data->index);
304 up_write(&pmem[id].bitmap_sem);
307 /* if this file is a submap (mapped, connected file), downref the
309 if (PMEM_FLAGS_SUBMAP & data->flags)
311 put_task_struct(data->task);
315 file->private_data = NULL;
317 list_for_each_safe(elt, elt2, &data->region_list) {
318 region_node = list_entry(elt, struct pmem_region_node, list);
322 BUG_ON(!list_empty(&data->region_list));
324 up_write(&data->sem);
326 if (pmem[id].release)
327 ret = pmem[id].release(inode, file);
332 static int pmem_open(struct inode *inode, struct file *file)
334 struct pmem_data *data;
335 int id = get_id(file);
338 DLOG("current %u file %p(%d)\n", current->pid, file, file_count(file));
339 /* setup file->private_data to indicate its unmapped */
340 /* you can only open a pmem device one time */
341 if (file->private_data != NULL)
343 data = kmalloc(sizeof(struct pmem_data), GFP_KERNEL);
345 printk("pmem: unable to allocate memory for pmem metadata.");
353 data->master_file = NULL;
357 INIT_LIST_HEAD(&data->region_list);
358 init_rwsem(&data->sem);
360 file->private_data = data;
361 INIT_LIST_HEAD(&data->list);
363 down(&pmem[id].data_list_sem);
364 list_add(&data->list, &pmem[id].data_list);
365 up(&pmem[id].data_list_sem);
369 static unsigned long pmem_order(unsigned long len)
373 len = (len + PMEM_MIN_ALLOC - 1)/PMEM_MIN_ALLOC;
375 for (i = 0; i < sizeof(len)*8; i++)
381 static int pmem_allocate(int id, unsigned long len)
383 /* caller should hold the write lock on pmem_sem! */
384 /* return the corresponding pdata[] entry */
386 int end = pmem[id].num_entries;
388 unsigned long order = pmem_order(len);
390 if (pmem[id].no_allocator) {
391 DLOG("no allocator");
392 if ((len > pmem[id].size) || pmem[id].allocated)
394 pmem[id].allocated = 1;
398 if (order > PMEM_MAX_ORDER)
400 DLOG("order %lx\n", order);
402 /* look through the bitmap:
403 * if you find a free slot of the correct order use it
404 * otherwise, use the best fit (smallest with size > order) slot
407 if (PMEM_IS_FREE(id, curr)) {
408 if (PMEM_ORDER(id, curr) == (unsigned char)order) {
409 /* set the not free bit and clear others */
413 if (PMEM_ORDER(id, curr) > (unsigned char)order &&
415 PMEM_ORDER(id, curr) < PMEM_ORDER(id, best_fit)))
418 curr = PMEM_NEXT_INDEX(id, curr);
421 /* if best_fit < 0, there are no suitable slots,
425 printk("pmem: no space left to allocate!\n");
429 /* now partition the best fit:
430 * split the slot into 2 buddies of order - 1
431 * repeat until the slot is of the correct order
433 while (PMEM_ORDER(id, best_fit) > (unsigned char)order) {
435 PMEM_ORDER(id, best_fit) -= 1;
436 buddy = PMEM_BUDDY_INDEX(id, best_fit);
437 PMEM_ORDER(id, buddy) = PMEM_ORDER(id, best_fit);
439 pmem[id].bitmap[best_fit].allocated = 1;
443 static pgprot_t phys_mem_access_prot(struct file *file, pgprot_t vma_prot)
445 int id = get_id(file);
446 #ifdef pgprot_noncached
447 if (pmem[id].cached == 0 || file->f_flags & O_SYNC)
448 return pgprot_noncached(vma_prot);
450 #ifdef pgprot_ext_buffered
451 else if (pmem[id].buffered)
452 return pgprot_ext_buffered(vma_prot);
457 static unsigned long pmem_start_addr(int id, struct pmem_data *data)
459 if (pmem[id].no_allocator)
460 return PMEM_START_ADDR(id, 0);
462 return PMEM_START_ADDR(id, data->index);
466 static void *pmem_start_vaddr(int id, struct pmem_data *data)
468 return pmem_start_addr(id, data) - pmem[id].base + pmem[id].vbase;
471 static unsigned long pmem_len(int id, struct pmem_data *data)
473 if (pmem[id].no_allocator)
476 return PMEM_LEN(id, data->index);
479 static int pmem_map_garbage(int id, struct vm_area_struct *vma,
480 struct pmem_data *data, unsigned long offset,
483 int i, garbage_pages = len >> PAGE_SHIFT;
485 vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP | VM_SHARED | VM_WRITE;
486 for (i = 0; i < garbage_pages; i++) {
487 if (vm_insert_pfn(vma, vma->vm_start + offset + (i * PAGE_SIZE),
488 pmem[id].garbage_pfn))
494 static int pmem_unmap_pfn_range(int id, struct vm_area_struct *vma,
495 struct pmem_data *data, unsigned long offset,
499 DLOG("unmap offset %lx len %lx\n", offset, len);
501 BUG_ON(!PMEM_IS_PAGE_ALIGNED(len));
503 garbage_pages = len >> PAGE_SHIFT;
504 zap_page_range(vma, vma->vm_start + offset, len, NULL);
505 pmem_map_garbage(id, vma, data, offset, len);
509 static int pmem_map_pfn_range(int id, struct vm_area_struct *vma,
510 struct pmem_data *data, unsigned long offset,
513 DLOG("map offset %lx len %lx\n", offset, len);
514 BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma->vm_start));
515 BUG_ON(!PMEM_IS_PAGE_ALIGNED(vma->vm_end));
516 BUG_ON(!PMEM_IS_PAGE_ALIGNED(len));
517 BUG_ON(!PMEM_IS_PAGE_ALIGNED(offset));
519 if (io_remap_pfn_range(vma, vma->vm_start + offset,
520 (pmem_start_addr(id, data) + offset) >> PAGE_SHIFT,
521 len, vma->vm_page_prot)) {
527 static int pmem_remap_pfn_range(int id, struct vm_area_struct *vma,
528 struct pmem_data *data, unsigned long offset,
531 /* hold the mm semp for the vma you are modifying when you call this */
533 zap_page_range(vma, vma->vm_start + offset, len, NULL);
534 return pmem_map_pfn_range(id, vma, data, offset, len);
537 static void pmem_vma_open(struct vm_area_struct *vma)
539 struct file *file = vma->vm_file;
540 struct pmem_data *data = file->private_data;
541 int id = get_id(file);
542 /* this should never be called as we don't support copying pmem
544 BUG_ON(!has_allocation(file));
545 down_write(&data->sem);
546 /* remap the garbage pages, forkers don't get access to the data */
547 pmem_unmap_pfn_range(id, vma, data, 0, vma->vm_start - vma->vm_end);
548 up_write(&data->sem);
551 static void pmem_vma_close(struct vm_area_struct *vma)
553 struct file *file = vma->vm_file;
554 struct pmem_data *data = file->private_data;
556 DLOG("current %u ppid %u file %p count %d\n", current->pid,
557 current->parent->pid, file, file_count(file));
558 if (unlikely(!is_pmem_file(file) || !has_allocation(file))) {
559 printk(KERN_WARNING "pmem: something is very wrong, you are "
560 "closing a vm backing an allocation that doesn't "
564 down_write(&data->sem);
565 if (data->vma == vma) {
567 if ((data->flags & PMEM_FLAGS_CONNECTED) &&
568 (data->flags & PMEM_FLAGS_SUBMAP))
569 data->flags |= PMEM_FLAGS_UNSUBMAP;
571 /* the kernel is going to free this vma now anyway */
572 up_write(&data->sem);
575 static struct vm_operations_struct vm_ops = {
576 .open = pmem_vma_open,
577 .close = pmem_vma_close,
580 static int pmem_mmap(struct file *file, struct vm_area_struct *vma)
582 struct pmem_data *data;
584 unsigned long vma_size = vma->vm_end - vma->vm_start;
585 int ret = 0, id = get_id(file);
587 if (vma->vm_pgoff || !PMEM_IS_PAGE_ALIGNED(vma_size)) {
589 printk(KERN_ERR "pmem: mmaps must be at offset zero, aligned"
590 " and a multiple of pages_size.\n");
595 data = file->private_data;
596 down_write(&data->sem);
597 /* check this file isn't already mmaped, for submaps check this file
598 * has never been mmaped */
599 if ((data->flags & PMEM_FLAGS_MASTERMAP) ||
600 (data->flags & PMEM_FLAGS_SUBMAP) ||
601 (data->flags & PMEM_FLAGS_UNSUBMAP)) {
603 printk(KERN_ERR "pmem: you can only mmap a pmem file once, "
604 "this file is already mmaped. %x\n", data->flags);
609 /* if file->private_data == unalloced, alloc*/
610 if (data && data->index == -1) {
611 down_write(&pmem[id].bitmap_sem);
612 index = pmem_allocate(id, vma->vm_end - vma->vm_start);
613 up_write(&pmem[id].bitmap_sem);
616 /* either no space was available or an error occured */
617 if (!has_allocation(file)) {
619 printk("pmem: could not find allocation for map.\n");
623 if (pmem_len(id, data) < vma_size) {
625 printk(KERN_WARNING "pmem: mmap size [%lu] does not match"
626 "size of backing region [%lu].\n", vma_size,
633 vma->vm_pgoff = pmem_start_addr(id, data) >> PAGE_SHIFT;
634 vma->vm_page_prot = phys_mem_access_prot(file, vma->vm_page_prot);
636 if (data->flags & PMEM_FLAGS_CONNECTED) {
637 struct pmem_region_node *region_node;
638 struct list_head *elt;
639 if (pmem_map_garbage(id, vma, data, 0, vma_size)) {
640 printk("pmem: mmap failed in kernel!\n");
644 list_for_each(elt, &data->region_list) {
645 region_node = list_entry(elt, struct pmem_region_node,
647 DLOG("remapping file: %p %lx %lx\n", file,
648 region_node->region.offset,
649 region_node->region.len);
650 if (pmem_remap_pfn_range(id, vma, data,
651 region_node->region.offset,
652 region_node->region.len)) {
657 data->flags |= PMEM_FLAGS_SUBMAP;
658 get_task_struct(current->group_leader);
659 data->task = current->group_leader;
662 data->pid = current->pid;
664 DLOG("submmapped file %p vma %p pid %u\n", file, vma,
667 if (pmem_map_pfn_range(id, vma, data, 0, vma_size)) {
668 printk(KERN_INFO "pmem: mmap failed in kernel!\n");
672 data->flags |= PMEM_FLAGS_MASTERMAP;
673 data->pid = current->pid;
675 vma->vm_ops = &vm_ops;
677 up_write(&data->sem);
681 /* the following are the api for accessing pmem regions by other drivers
682 * from inside the kernel */
683 int get_pmem_user_addr(struct file *file, unsigned long *start,
686 struct pmem_data *data;
687 if (!is_pmem_file(file) || !has_allocation(file)) {
689 printk(KERN_INFO "pmem: requested pmem data from invalid"
694 data = file->private_data;
695 down_read(&data->sem);
697 *start = data->vma->vm_start;
698 *len = data->vma->vm_end - data->vma->vm_start;
707 int get_pmem_addr(struct file *file, unsigned long *start,
708 unsigned long *vstart, unsigned long *len)
710 struct pmem_data *data;
713 if (!is_pmem_file(file) || !has_allocation(file))
716 data = file->private_data;
717 if (data->index == -1) {
719 printk(KERN_INFO "pmem: requested pmem data from file with no "
726 down_read(&data->sem);
727 *start = pmem_start_addr(id, data);
728 *len = pmem_len(id, data);
729 *vstart = (unsigned long)pmem_start_vaddr(id, data);
732 down_write(&data->sem);
734 up_write(&data->sem);
739 int get_pmem_file(int fd, unsigned long *start, unsigned long *vstart,
740 unsigned long *len, struct file **filp)
745 if (unlikely(file == NULL)) {
746 printk(KERN_INFO "pmem: requested data from file descriptor "
747 "that doesn't exist.");
751 if (get_pmem_addr(file, start, vstart, len))
762 void put_pmem_file(struct file *file)
764 struct pmem_data *data;
767 if (!is_pmem_file(file))
770 data = file->private_data;
772 down_write(&data->sem);
773 if (data->ref == 0) {
774 printk("pmem: pmem_put > pmem_get %s (pid %d)\n",
775 pmem[id].dev.name, data->pid);
779 up_write(&data->sem);
784 void flush_pmem_file(struct file *file, unsigned long offset, unsigned long len)
786 struct pmem_data *data;
789 struct pmem_region_node *region_node;
790 struct list_head *elt;
791 void *flush_start, *flush_end;
793 if (!is_pmem_file(file) || !has_allocation(file))
797 data = file->private_data;
798 if (!pmem[id].cached)
801 down_read(&data->sem);
802 vaddr = pmem_start_vaddr(id, data);
803 /* if this isn't a submmapped file, flush the whole thing */
804 if (unlikely(!(data->flags & PMEM_FLAGS_CONNECTED))) {
805 dmac_flush_range(vaddr, vaddr + pmem_len(id, data));
808 /* otherwise, flush the region of the file we are drawing */
809 list_for_each(elt, &data->region_list) {
810 region_node = list_entry(elt, struct pmem_region_node, list);
811 if ((offset >= region_node->region.offset) &&
812 ((offset + len) <= (region_node->region.offset +
813 region_node->region.len))) {
814 flush_start = vaddr + region_node->region.offset;
815 flush_end = flush_start + region_node->region.len;
816 dmac_flush_range(flush_start, flush_end);
824 static int pmem_connect(unsigned long connect, struct file *file)
826 struct pmem_data *data = file->private_data;
827 struct pmem_data *src_data;
828 struct file *src_file;
829 int ret = 0, put_needed;
831 down_write(&data->sem);
832 /* retrieve the src file and check it is a pmem file with an alloc */
833 src_file = fget_light(connect, &put_needed);
834 DLOG("connect %p to %p\n", file, src_file);
836 printk(KERN_INFO "pmem: src file not found!\n");
840 if (unlikely(!is_pmem_file(src_file) || !has_allocation(src_file))) {
841 printk(KERN_INFO "pmem: src file is not a pmem file or has no "
846 src_data = src_file->private_data;
848 if (has_allocation(file) && (data->index != src_data->index)) {
849 printk(KERN_INFO "pmem: file is already mapped but doesn't "
850 "match this src_file!\n");
854 data->index = src_data->index;
855 data->flags |= PMEM_FLAGS_CONNECTED;
856 data->master_fd = connect;
857 data->master_file = src_file;
860 fput_light(src_file, put_needed);
862 up_write(&data->sem);
866 static void pmem_unlock_data_and_mm(struct pmem_data *data,
867 struct mm_struct *mm)
869 up_write(&data->sem);
871 up_write(&mm->mmap_sem);
876 static int pmem_lock_data_and_mm(struct file *file, struct pmem_data *data,
877 struct mm_struct **locked_mm)
880 struct mm_struct *mm = NULL;
883 down_read(&data->sem);
884 if (PMEM_IS_SUBMAP(data)) {
885 mm = get_task_mm(data->task);
888 printk(KERN_DEBUG "pmem: can't remap task is gone!\n");
897 down_write(&mm->mmap_sem);
899 down_write(&data->sem);
900 /* check that the file didn't get mmaped before we could take the
901 * data sem, this should be safe b/c you can only submap each file
903 if (PMEM_IS_SUBMAP(data) && !mm) {
904 pmem_unlock_data_and_mm(data, mm);
905 up_write(&data->sem);
908 /* now check that vma.mm is still there, it could have been
909 * deleted by vma_close before we could get the data->sem */
910 if ((data->flags & PMEM_FLAGS_UNSUBMAP) && (mm != NULL)) {
911 /* might as well release this */
912 if (data->flags & PMEM_FLAGS_SUBMAP) {
913 put_task_struct(data->task);
915 /* lower the submap flag to show the mm is gone */
916 data->flags &= ~(PMEM_FLAGS_SUBMAP);
918 pmem_unlock_data_and_mm(data, mm);
925 int pmem_remap(struct pmem_region *region, struct file *file,
929 struct pmem_region_node *region_node;
930 struct mm_struct *mm = NULL;
931 struct list_head *elt, *elt2;
932 int id = get_id(file);
933 struct pmem_data *data = file->private_data;
935 /* pmem region must be aligned on a page boundry */
936 if (unlikely(!PMEM_IS_PAGE_ALIGNED(region->offset) ||
937 !PMEM_IS_PAGE_ALIGNED(region->len))) {
939 printk(KERN_DEBUG "pmem: request for unaligned pmem "
940 "suballocation %lx %lx\n", region->offset, region->len);
945 /* if userspace requests a region of len 0, there's nothing to do */
946 if (region->len == 0)
949 /* lock the mm and data */
950 ret = pmem_lock_data_and_mm(file, data, &mm);
954 /* only the owner of the master file can remap the client fds
956 if (!is_master_owner(file)) {
958 printk("pmem: remap requested from non-master process\n");
964 /* check that the requested range is within the src allocation */
965 if (unlikely((region->offset > pmem_len(id, data)) ||
966 (region->len > pmem_len(id, data)) ||
967 (region->offset + region->len > pmem_len(id, data)))) {
969 printk(KERN_INFO "pmem: suballoc doesn't fit in src_file!\n");
975 if (operation == PMEM_MAP) {
976 region_node = kmalloc(sizeof(struct pmem_region_node),
981 printk(KERN_INFO "No space to allocate metadata!");
985 region_node->region = *region;
986 list_add(®ion_node->list, &data->region_list);
987 } else if (operation == PMEM_UNMAP) {
989 list_for_each_safe(elt, elt2, &data->region_list) {
990 region_node = list_entry(elt, struct pmem_region_node,
992 if (region->len == 0 ||
993 (region_node->region.offset == region->offset &&
994 region_node->region.len == region->len)) {
1002 printk("pmem: Unmap region does not map any mapped "
1010 if (data->vma && PMEM_IS_SUBMAP(data)) {
1011 if (operation == PMEM_MAP)
1012 ret = pmem_remap_pfn_range(id, data->vma, data,
1013 region->offset, region->len);
1014 else if (operation == PMEM_UNMAP)
1015 ret = pmem_unmap_pfn_range(id, data->vma, data,
1016 region->offset, region->len);
1020 pmem_unlock_data_and_mm(data, mm);
1024 static void pmem_revoke(struct file *file, struct pmem_data *data)
1026 struct pmem_region_node *region_node;
1027 struct list_head *elt, *elt2;
1028 struct mm_struct *mm = NULL;
1029 int id = get_id(file);
1032 data->master_file = NULL;
1033 ret = pmem_lock_data_and_mm(file, data, &mm);
1034 /* if lock_data_and_mm fails either the task that mapped the fd, or
1035 * the vma that mapped it have already gone away, nothing more
1036 * needs to be done */
1039 /* unmap everything */
1040 /* delete the regions and region list nothing is mapped any more */
1042 list_for_each_safe(elt, elt2, &data->region_list) {
1043 region_node = list_entry(elt, struct pmem_region_node,
1045 pmem_unmap_pfn_range(id, data->vma, data,
1046 region_node->region.offset,
1047 region_node->region.len);
1051 /* delete the master file */
1052 pmem_unlock_data_and_mm(data, mm);
1055 static void pmem_get_size(struct pmem_region *region, struct file *file)
1057 struct pmem_data *data = file->private_data;
1058 int id = get_id(file);
1060 if (!has_allocation(file)) {
1065 region->offset = pmem_start_addr(id, data);
1066 region->len = pmem_len(id, data);
1068 DLOG("offset %lx len %lx\n", region->offset, region->len);
1072 static long pmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1074 struct pmem_data *data;
1075 int id = get_id(file);
1080 struct pmem_region region;
1082 if (!has_allocation(file)) {
1086 data = file->private_data;
1087 region.offset = pmem_start_addr(id, data);
1088 region.len = pmem_len(id, data);
1090 printk(KERN_INFO "pmem: request for physical address "
1091 "of pmem region from process %d.\n", current->pid);
1092 if (copy_to_user((void __user *)arg, ®ion,
1093 sizeof(struct pmem_region)))
1099 struct pmem_region region;
1100 if (copy_from_user(®ion, (void __user *)arg,
1101 sizeof(struct pmem_region)))
1103 data = file->private_data;
1104 return pmem_remap(®ion, file, PMEM_MAP);
1109 struct pmem_region region;
1110 if (copy_from_user(®ion, (void __user *)arg,
1111 sizeof(struct pmem_region)))
1113 data = file->private_data;
1114 return pmem_remap(®ion, file, PMEM_UNMAP);
1119 struct pmem_region region;
1121 pmem_get_size(®ion, file);
1122 if (copy_to_user((void __user *)arg, ®ion,
1123 sizeof(struct pmem_region)))
1127 case PMEM_GET_TOTAL_SIZE:
1129 struct pmem_region region;
1130 DLOG("get total size\n");
1133 region.len = pmem[id].size;
1134 if (copy_to_user((void __user *)arg, ®ion,
1135 sizeof(struct pmem_region)))
1141 if (has_allocation(file))
1143 data = file->private_data;
1144 data->index = pmem_allocate(id, arg);
1149 return pmem_connect(arg, file);
1153 return pmem[id].ioctl(file, cmd, arg);
1160 static ssize_t debug_open(struct inode *inode, struct file *file)
1162 file->private_data = inode->i_private;
1166 static ssize_t debug_read(struct file *file, char __user *buf, size_t count,
1169 struct list_head *elt, *elt2;
1170 struct pmem_data *data;
1171 struct pmem_region_node *region_node;
1172 int id = (int)file->private_data;
1173 const int debug_bufmax = 4096;
1174 static char buffer[4096];
1177 DLOG("debug open\n");
1178 n = scnprintf(buffer, debug_bufmax,
1179 "pid #: mapped regions (offset, len) (offset,len)...\n");
1181 down(&pmem[id].data_list_sem);
1182 list_for_each(elt, &pmem[id].data_list) {
1183 data = list_entry(elt, struct pmem_data, list);
1184 down_read(&data->sem);
1185 n += scnprintf(buffer + n, debug_bufmax - n, "pid %u:",
1187 list_for_each(elt2, &data->region_list) {
1188 region_node = list_entry(elt2, struct pmem_region_node,
1190 n += scnprintf(buffer + n, debug_bufmax - n,
1192 region_node->region.offset,
1193 region_node->region.len);
1195 n += scnprintf(buffer + n, debug_bufmax - n, "\n");
1196 up_read(&data->sem);
1198 up(&pmem[id].data_list_sem);
1202 return simple_read_from_buffer(buf, count, ppos, buffer, n);
1205 static struct file_operations debug_fops = {
1208 .llseek = default_llseek,
1213 static struct miscdevice pmem_dev = {
1219 int pmem_setup(struct android_pmem_platform_data *pdata,
1220 long (*ioctl)(struct file *, unsigned int, unsigned long),
1221 int (*release)(struct inode *, struct file *))
1228 pmem[id].no_allocator = pdata->no_allocator;
1229 pmem[id].cached = pdata->cached;
1230 pmem[id].buffered = pdata->buffered;
1231 pmem[id].base = pdata->start;
1232 pmem[id].size = pdata->size;
1233 pmem[id].ioctl = ioctl;
1234 pmem[id].release = release;
1235 init_rwsem(&pmem[id].bitmap_sem);
1236 sema_init(&pmem[id].data_list_sem, 1);
1237 INIT_LIST_HEAD(&pmem[id].data_list);
1238 pmem[id].dev.name = pdata->name;
1239 pmem[id].dev.minor = id;
1240 pmem[id].dev.fops = &pmem_fops;
1241 printk(KERN_INFO "%s: %d init\n", pdata->name, pdata->cached);
1243 err = misc_register(&pmem[id].dev);
1245 printk(KERN_ALERT "Unable to register pmem driver!\n");
1246 goto err_cant_register_device;
1248 pmem[id].num_entries = pmem[id].size / PMEM_MIN_ALLOC;
1250 pmem[id].bitmap = kcalloc(pmem[id].num_entries,
1251 sizeof(struct pmem_bits), GFP_KERNEL);
1252 if (!pmem[id].bitmap)
1253 goto err_no_mem_for_metadata;
1255 for (i = sizeof(pmem[id].num_entries) * 8 - 1; i >= 0; i--) {
1256 if ((pmem[id].num_entries) & 1<<i) {
1257 PMEM_ORDER(id, index) = i;
1258 index = PMEM_NEXT_INDEX(id, index);
1262 if (pmem[id].cached)
1263 pmem[id].vbase = ioremap_cached(pmem[id].base,
1265 #ifdef ioremap_ext_buffered
1266 else if (pmem[id].buffered)
1267 pmem[id].vbase = ioremap_ext_buffered(pmem[id].base,
1271 pmem[id].vbase = ioremap(pmem[id].base, pmem[id].size);
1273 if (pmem[id].vbase == 0)
1274 goto error_cant_remap;
1276 pmem[id].garbage_pfn = page_to_pfn(alloc_page(GFP_KERNEL));
1277 if (pmem[id].no_allocator)
1278 pmem[id].allocated = 0;
1281 debugfs_create_file(pdata->name, S_IFREG | S_IRUGO, NULL, (void *)id,
1286 kfree(pmem[id].bitmap);
1287 err_no_mem_for_metadata:
1288 misc_deregister(&pmem[id].dev);
1289 err_cant_register_device:
1293 static int pmem_probe(struct platform_device *pdev)
1295 struct android_pmem_platform_data *pdata;
1297 if (!pdev || !pdev->dev.platform_data) {
1298 printk(KERN_ALERT "Unable to probe pmem!\n");
1301 pdata = pdev->dev.platform_data;
1302 return pmem_setup(pdata, NULL, NULL);
1306 static int pmem_remove(struct platform_device *pdev)
1309 __free_page(pfn_to_page(pmem[id].garbage_pfn));
1310 misc_deregister(&pmem[id].dev);
1314 static struct platform_driver pmem_driver = {
1315 .probe = pmem_probe,
1316 .remove = pmem_remove,
1317 .driver = { .name = "android_pmem" }
1321 static int __init pmem_init(void)
1323 return platform_driver_register(&pmem_driver);
1326 static void __exit pmem_exit(void)
1328 platform_driver_unregister(&pmem_driver);
1331 module_init(pmem_init);
1332 module_exit(pmem_exit);