Merge branch 'slub/cleanups' into slab/next

* Fix a merge conflict in mm/slub.c::acquire_slab() due to commit 02d7633
  ("slub: fix a memory leak in get_partial_node()").

Conflicts:
	mm/slub.c

Signed-off-by: Pekka Enberg <penberg@kernel.org>

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index c2f8c8bc56edd08183dd0dd22b4f7d3b39f1c63d..ebdcf4ba42ee942aac6cf7269468575e7af89a29 100644 (file)
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -48,7 +48,6 @@ struct kmem_cache_cpu {
        unsigned long tid;      /* Globally unique transaction id */
        struct page *page;      /* The slab from which we are allocating */
        struct page *partial;   /* Partially allocated frozen slabs */
-       int node;               /* The node of the page (or -1 for debug) */
 #ifdef CONFIG_SLUB_STATS
        unsigned stat[NR_SLUB_STAT_ITEMS];
 #endif

diff --git a/mm/slub.c b/mm/slub.c
index 8c691fa1cf3c78a91fa301bcd0cd323df1b28a1e..2de3c996f327079bdae3de9262bf69ba2bab496e 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1490,12 +1490,12 @@ static inline void remove_partial(struct kmem_cache_node *n,
 }
 
 /*
- * Lock slab, remove from the partial list and put the object into the
- * per cpu freelist.
+ * Remove slab from the partial list, freeze it and
+ * return the pointer to the freelist.
  *
  * Returns a list of objects or NULL if it fails.
  *
- * Must hold list_lock.
+ * Must hold list_lock since we modify the partial list.
  */
 static inline void *acquire_slab(struct kmem_cache *s,
                struct kmem_cache_node *n, struct page *page,
@@ -1510,26 +1510,27 @@ static inline void *acquire_slab(struct kmem_cache *s,
         * The old freelist is the list of objects for the
         * per cpu allocation list.
         */
-       do {
-               freelist = page->freelist;
-               counters = page->counters;
-               new.counters = counters;
-               if (mode) {
-                       new.inuse = page->objects;
-                       new.freelist = NULL;
-               } else {
-                       new.freelist = freelist;
-               }
+       freelist = page->freelist;
+       counters = page->counters;
+       new.counters = counters;
+       if (mode) {
+               new.inuse = page->objects;
+               new.freelist = NULL;
+       } else {
+               new.freelist = freelist;
+       }
 
-               VM_BUG_ON(new.frozen);
-               new.frozen = 1;
+       VM_BUG_ON(new.frozen);
+       new.frozen = 1;
 
-       } while (!__cmpxchg_double_slab(s, page,
+       if (!__cmpxchg_double_slab(s, page,
                        freelist, counters,
                        new.freelist, new.counters,
-                       "lock and freeze"));
+                       "acquire_slab"))
+               return NULL;
 
        remove_partial(n, page);
+       WARN_ON(!freelist);
        return freelist;
 }
 
@@ -1563,7 +1564,6 @@ static void *get_partial_node(struct kmem_cache *s,
 
                if (!object) {
                        c->page = page;
-                       c->node = page_to_nid(page);
                        stat(s, ALLOC_FROM_PARTIAL);
                        object = t;
                        available =  page->objects - page->inuse;
@@ -1731,14 +1731,12 @@ void init_kmem_cache_cpus(struct kmem_cache *s)
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
+static void deactivate_slab(struct kmem_cache *s, struct page *page, void *freelist)
 {
        enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
-       struct page *page = c->page;
        struct kmem_cache_node *n = get_node(s, page_to_nid(page));
        int lock = 0;
        enum slab_modes l = M_NONE, m = M_NONE;
-       void *freelist;
        void *nextfree;
        int tail = DEACTIVATE_TO_HEAD;
        struct page new;
@@ -1749,11 +1747,6 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
                tail = DEACTIVATE_TO_TAIL;
        }
 
-       c->tid = next_tid(c->tid);
-       c->page = NULL;
-       freelist = c->freelist;
-       c->freelist = NULL;
-
        /*
         * Stage one: Free all available per cpu objects back
         * to the page freelist while it is still frozen. Leave the
@@ -2011,7 +2004,11 @@ int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
        stat(s, CPUSLAB_FLUSH);
-       deactivate_slab(s, c);
+       deactivate_slab(s, c->page, c->freelist);
+
+       c->tid = next_tid(c->tid);
+       c->page = NULL;
+       c->freelist = NULL;
 }
 
 /*
@@ -2055,10 +2052,10 @@ static void flush_all(struct kmem_cache *s)
  * Check if the objects in a per cpu structure fit numa
  * locality expectations.
  */
-static inline int node_match(struct kmem_cache_cpu *c, int node)
+static inline int node_match(struct page *page, int node)
 {
 #ifdef CONFIG_NUMA
-       if (node != NUMA_NO_NODE && c->node != node)
+       if (node != NUMA_NO_NODE && page_to_nid(page) != node)
                return 0;
 #endif
        return 1;
@@ -2130,10 +2127,16 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
 static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
                        int node, struct kmem_cache_cpu **pc)
 {
-       void *object;
-       struct kmem_cache_cpu *c;
-       struct page *page = new_slab(s, flags, node);
+       void *freelist;
+       struct kmem_cache_cpu *c = *pc;
+       struct page *page;
+
+       freelist = get_partial(s, flags, node, c);
 
+       if (freelist)
+               return freelist;
+
+       page = new_slab(s, flags, node);
        if (page) {
                c = __this_cpu_ptr(s->cpu_slab);
                if (c->page)
@@ -2143,17 +2146,16 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
                 * No other reference to the page yet so we can
                 * muck around with it freely without cmpxchg
                 */
-               object = page->freelist;
+               freelist = page->freelist;
                page->freelist = NULL;
 
                stat(s, ALLOC_SLAB);
-               c->node = page_to_nid(page);
                c->page = page;
                *pc = c;
        } else
-               object = NULL;
+               freelist = NULL;
 
-       return object;
+       return freelist;
 }
 
 /*
@@ -2173,6 +2175,7 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
        do {
                freelist = page->freelist;
                counters = page->counters;
+
                new.counters = counters;
                VM_BUG_ON(!new.frozen);
 
@@ -2206,7 +2209,8 @@ static inline void *get_freelist(struct kmem_cache *s, struct page *page)
 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
                          unsigned long addr, struct kmem_cache_cpu *c)
 {
-       void **object;
+       void *freelist;
+       struct page *page;
        unsigned long flags;
 
        local_irq_save(flags);
@@ -2219,25 +2223,29 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
        c = this_cpu_ptr(s->cpu_slab);
 #endif
 
-       if (!c->page)
+       page = c->page;
+       if (!page)
                goto new_slab;
 redo:
-       if (unlikely(!node_match(c, node))) {
+
+       if (unlikely(!node_match(page, node))) {
                stat(s, ALLOC_NODE_MISMATCH);
-               deactivate_slab(s, c);
+               deactivate_slab(s, page, c->freelist);
+               c->page = NULL;
+               c->freelist = NULL;
                goto new_slab;
        }
 
        /* must check again c->freelist in case of cpu migration or IRQ */
-       object = c->freelist;
-       if (object)
+       freelist = c->freelist;
+       if (freelist)
                goto load_freelist;
 
        stat(s, ALLOC_SLOWPATH);
 
-       object = get_freelist(s, c->page);
+       freelist = get_freelist(s, page);
 
-       if (!object) {
+       if (!freelist) {
                c->page = NULL;
                stat(s, DEACTIVATE_BYPASS);
                goto new_slab;
@@ -2246,50 +2254,50 @@ redo:
        stat(s, ALLOC_REFILL);
 
 load_freelist:
-       c->freelist = get_freepointer(s, object);
+       /*
+        * freelist is pointing to the list of objects to be used.
+        * page is pointing to the page from which the objects are obtained.
+        * That page must be frozen for per cpu allocations to work.
+        */
+       VM_BUG_ON(!c->page->frozen);
+       c->freelist = get_freepointer(s, freelist);
        c->tid = next_tid(c->tid);
        local_irq_restore(flags);
-       return object;
+       return freelist;
 
 new_slab:
 
        if (c->partial) {
-               c->page = c->partial;
-               c->partial = c->page->next;
-               c->node = page_to_nid(c->page);
+               page = c->page = c->partial;
+               c->partial = page->next;
                stat(s, CPU_PARTIAL_ALLOC);
                c->freelist = NULL;
                goto redo;
        }
 
-       /* Then do expensive stuff like retrieving pages from the partial lists */
-       object = get_partial(s, gfpflags, node, c);
-
-       if (unlikely(!object)) {
+       freelist = new_slab_objects(s, gfpflags, node, &c);
 
-               object = new_slab_objects(s, gfpflags, node, &c);
+       if (unlikely(!freelist)) {
+               if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
+                       slab_out_of_memory(s, gfpflags, node);
 
-               if (unlikely(!object)) {
-                       if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit())
-                               slab_out_of_memory(s, gfpflags, node);
-
-                       local_irq_restore(flags);
-                       return NULL;
-               }
+               local_irq_restore(flags);
+               return NULL;
        }
 
+       page = c->page;
        if (likely(!kmem_cache_debug(s)))
                goto load_freelist;
 
        /* Only entered in the debug case */
-       if (!alloc_debug_processing(s, c->page, object, addr))
+       if (!alloc_debug_processing(s, page, freelist, addr))
                goto new_slab;  /* Slab failed checks. Next slab needed */
 
-       c->freelist = get_freepointer(s, object);
-       deactivate_slab(s, c);
-       c->node = NUMA_NO_NODE;
+       deactivate_slab(s, page, get_freepointer(s, freelist));
+       c->page = NULL;
+       c->freelist = NULL;
        local_irq_restore(flags);
-       return object;
+       return freelist;
 }
 
 /*
@@ -2307,6 +2315,7 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
 {
        void **object;
        struct kmem_cache_cpu *c;
+       struct page *page;
        unsigned long tid;
 
        if (slab_pre_alloc_hook(s, gfpflags))
@@ -2332,7 +2341,8 @@ redo:
        barrier();
 
        object = c->freelist;
-       if (unlikely(!object || !node_match(c, node)))
+       page = c->page;
+       if (unlikely(!object || !node_match(page, node)))
 
                object = __slab_alloc(s, gfpflags, node, addr, c);
 
@@ -4500,30 +4510,31 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
 
                for_each_possible_cpu(cpu) {
                        struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
-                       int node = ACCESS_ONCE(c->node);
+                       int node;
                        struct page *page;
 
-                       if (node < 0)
-                               continue;
                        page = ACCESS_ONCE(c->page);
-                       if (page) {
-                               if (flags & SO_TOTAL)
-                                       x = page->objects;
-                               else if (flags & SO_OBJECTS)
-                                       x = page->inuse;
-                               else
-                                       x = 1;
+                       if (!page)
+                               continue;
 
-                               total += x;
-                               nodes[node] += x;
-                       }
-                       page = c->partial;
+                       node = page_to_nid(page);
+                       if (flags & SO_TOTAL)
+                               x = page->objects;
+                       else if (flags & SO_OBJECTS)
+                               x = page->inuse;
+                       else
+                               x = 1;
 
+                       total += x;
+                       nodes[node] += x;
+
+                       page = ACCESS_ONCE(c->partial);
                        if (page) {
                                x = page->pobjects;
                                total += x;
                                nodes[node] += x;
                        }
+
                        per_cpu[node]++;
                }
        }