arm: use genalloc for the atomic pool
authorLaura Abbott <lauraa@codeaurora.org>
Thu, 9 Oct 2014 22:26:42 +0000 (15:26 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Fri, 10 Oct 2014 02:25:52 +0000 (22:25 -0400)
ARM currently uses a bitmap for tracking atomic allocations.  genalloc
already handles this type of memory pool allocation so switch to using
that instead.

Signed-off-by: Laura Abbott <lauraa@codeaurora.org>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: David Riley <davidriley@chromium.org>
Cc: Olof Johansson <olof@lixom.net>
Cc: Ritesh Harjain <ritesh.harjani@gmail.com>
Cc: Russell King <linux@arm.linux.org.uk>
Cc: Thierry Reding <thierry.reding@gmail.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
arch/arm/Kconfig
arch/arm/mm/dma-mapping.c

index d9d32de9628c53cbaa9ae6db00fd5c299d0155d7..36d47987a9e0add0a2bd96382ba3d822c843d2a9 100644 (file)
@@ -14,6 +14,7 @@ config ARM
        select CLONE_BACKWARDS
        select CPU_PM if (SUSPEND || CPU_IDLE)
        select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
+       select GENERIC_ALLOCATOR
        select GENERIC_ATOMIC64 if (CPU_V7M || CPU_V6 || !CPU_32v6K || !AEABI)
        select GENERIC_CLOCKEVENTS_BROADCAST if SMP
        select GENERIC_IDLE_POLL_SETUP
index eecc8e60deeada05f299824948209a816c702246..c245d903927fadfdee177b4d94b876ae2473618e 100644 (file)
@@ -12,6 +12,7 @@
 #include <linux/bootmem.h>
 #include <linux/module.h>
 #include <linux/mm.h>
+#include <linux/genalloc.h>
 #include <linux/gfp.h>
 #include <linux/errno.h>
 #include <linux/list.h>
@@ -314,23 +315,13 @@ static void __dma_free_remap(void *cpu_addr, size_t size)
 }
 
 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
+static struct gen_pool *atomic_pool;
 
-struct dma_pool {
-       size_t size;
-       spinlock_t lock;
-       unsigned long *bitmap;
-       unsigned long nr_pages;
-       void *vaddr;
-       struct page **pages;
-};
-
-static struct dma_pool atomic_pool = {
-       .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
-};
+static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE;
 
 static int __init early_coherent_pool(char *p)
 {
-       atomic_pool.size = memparse(p, &p);
+       atomic_pool_size = memparse(p, &p);
        return 0;
 }
 early_param("coherent_pool", early_coherent_pool);
@@ -340,14 +331,14 @@ void __init init_dma_coherent_pool_size(unsigned long size)
        /*
         * Catch any attempt to set the pool size too late.
         */
-       BUG_ON(atomic_pool.vaddr);
+       BUG_ON(atomic_pool);
 
        /*
         * Set architecture specific coherent pool size only if
         * it has not been changed by kernel command line parameter.
         */
-       if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
-               atomic_pool.size = size;
+       if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE)
+               atomic_pool_size = size;
 }
 
 /*
@@ -355,52 +346,44 @@ void __init init_dma_coherent_pool_size(unsigned long size)
  */
 static int __init atomic_pool_init(void)
 {
-       struct dma_pool *pool = &atomic_pool;
        pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL);
        gfp_t gfp = GFP_KERNEL | GFP_DMA;
-       unsigned long nr_pages = pool->size >> PAGE_SHIFT;
-       unsigned long *bitmap;
        struct page *page;
-       struct page **pages;
        void *ptr;
-       int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
-
-       bitmap = kzalloc(bitmap_size, GFP_KERNEL);
-       if (!bitmap)
-               goto no_bitmap;
 
-       pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
-       if (!pages)
-               goto no_pages;
+       atomic_pool = gen_pool_create(PAGE_SHIFT, -1);
+       if (!atomic_pool)
+               goto out;
 
        if (dev_get_cma_area(NULL))
-               ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page,
-                                             atomic_pool_init);
+               ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot,
+                                             &page, atomic_pool_init);
        else
-               ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
-                                          atomic_pool_init);
+               ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot,
+                                          &page, atomic_pool_init);
        if (ptr) {
-               int i;
-
-               for (i = 0; i < nr_pages; i++)
-                       pages[i] = page + i;
-
-               spin_lock_init(&pool->lock);
-               pool->vaddr = ptr;
-               pool->pages = pages;
-               pool->bitmap = bitmap;
-               pool->nr_pages = nr_pages;
-               pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
-                      (unsigned)pool->size / 1024);
+               int ret;
+
+               ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr,
+                                       page_to_phys(page),
+                                       atomic_pool_size, -1);
+               if (ret)
+                       goto destroy_genpool;
+
+               gen_pool_set_algo(atomic_pool,
+                               gen_pool_first_fit_order_align,
+                               (void *)PAGE_SHIFT);
+               pr_info("DMA: preallocated %zd KiB pool for atomic coherent allocations\n",
+                      atomic_pool_size / 1024);
                return 0;
        }
 
-       kfree(pages);
-no_pages:
-       kfree(bitmap);
-no_bitmap:
-       pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
-              (unsigned)pool->size / 1024);
+destroy_genpool:
+       gen_pool_destroy(atomic_pool);
+       atomic_pool = NULL;
+out:
+       pr_err("DMA: failed to allocate %zx KiB pool for atomic coherent allocation\n",
+              atomic_pool_size / 1024);
        return -ENOMEM;
 }
 /*
@@ -504,76 +487,36 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp,
 
 static void *__alloc_from_pool(size_t size, struct page **ret_page)
 {
-       struct dma_pool *pool = &atomic_pool;
-       unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
-       unsigned int pageno;
-       unsigned long flags;
+       unsigned long val;
        void *ptr = NULL;
-       unsigned long align_mask;
 
-       if (!pool->vaddr) {
+       if (!atomic_pool) {
                WARN(1, "coherent pool not initialised!\n");
                return NULL;
        }
 
-       /*
-        * Align the region allocation - allocations from pool are rather
-        * small, so align them to their order in pages, minimum is a page
-        * size. This helps reduce fragmentation of the DMA space.
-        */
-       align_mask = (1 << get_order(size)) - 1;
-
-       spin_lock_irqsave(&pool->lock, flags);
-       pageno = bitmap_find_next_zero_area(pool->bitmap, pool->nr_pages,
-                                           0, count, align_mask);
-       if (pageno < pool->nr_pages) {
-               bitmap_set(pool->bitmap, pageno, count);
-               ptr = pool->vaddr + PAGE_SIZE * pageno;
-               *ret_page = pool->pages[pageno];
-       } else {
-               pr_err_once("ERROR: %u KiB atomic DMA coherent pool is too small!\n"
-                           "Please increase it with coherent_pool= kernel parameter!\n",
-                           (unsigned)pool->size / 1024);
+       val = gen_pool_alloc(atomic_pool, size);
+       if (val) {
+               phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val);
+
+               *ret_page = phys_to_page(phys);
+               ptr = (void *)val;
        }
-       spin_unlock_irqrestore(&pool->lock, flags);
 
        return ptr;
 }
 
 static bool __in_atomic_pool(void *start, size_t size)
 {
-       struct dma_pool *pool = &atomic_pool;
-       void *end = start + size;
-       void *pool_start = pool->vaddr;
-       void *pool_end = pool->vaddr + pool->size;
-
-       if (start < pool_start || start >= pool_end)
-               return false;
-
-       if (end <= pool_end)
-               return true;
-
-       WARN(1, "Wrong coherent size(%p-%p) from atomic pool(%p-%p)\n",
-            start, end - 1, pool_start, pool_end - 1);
-
-       return false;
+       return addr_in_gen_pool(atomic_pool, (unsigned long)start, size);
 }
 
 static int __free_from_pool(void *start, size_t size)
 {
-       struct dma_pool *pool = &atomic_pool;
-       unsigned long pageno, count;
-       unsigned long flags;
-
        if (!__in_atomic_pool(start, size))
                return 0;
 
-       pageno = (start - pool->vaddr) >> PAGE_SHIFT;
-       count = size >> PAGE_SHIFT;
-
-       spin_lock_irqsave(&pool->lock, flags);
-       bitmap_clear(pool->bitmap, pageno, count);
-       spin_unlock_irqrestore(&pool->lock, flags);
+       gen_pool_free(atomic_pool, (unsigned long)start, size);
 
        return 1;
 }
@@ -1316,11 +1259,13 @@ static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t si
 
 static struct page **__atomic_get_pages(void *addr)
 {
-       struct dma_pool *pool = &atomic_pool;
-       struct page **pages = pool->pages;
-       int offs = (addr - pool->vaddr) >> PAGE_SHIFT;
+       struct page *page;
+       phys_addr_t phys;
+
+       phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr);
+       page = phys_to_page(phys);
 
-       return pages + offs;
+       return (struct page **)page;
 }
 
 static struct page **__iommu_get_pages(void *cpu_addr, struct dma_attrs *attrs)