mm, page_alloc: distinguish between being unable to sleep, unwilling to sleep and...

[firefly-linux-kernel-4.4.55.git] / include / linux / gfp.h

diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index f92cbd2f44507adda1333d60adc5fb4686a3f029..b56e811b6f7c77f91a3535da405c919574c2f4f7 100644 (file)
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -14,7 +14,7 @@ struct vm_area_struct;
 #define ___GFP_HIGHMEM         0x02u
 #define ___GFP_DMA32           0x04u
 #define ___GFP_MOVABLE         0x08u
-#define ___GFP_WAIT            0x10u
+#define ___GFP_RECLAIMABLE     0x10u
 #define ___GFP_HIGH            0x20u
 #define ___GFP_IO              0x40u
 #define ___GFP_FS              0x80u
@@ -29,12 +29,13 @@ struct vm_area_struct;
 #define ___GFP_NOMEMALLOC      0x10000u
 #define ___GFP_HARDWALL                0x20000u
 #define ___GFP_THISNODE                0x40000u
-#define ___GFP_RECLAIMABLE     0x80000u
+#define ___GFP_ATOMIC          0x80000u
 #define ___GFP_NOACCOUNT       0x100000u
 #define ___GFP_NOTRACK         0x200000u
-#define ___GFP_NO_KSWAPD       0x400000u
+#define ___GFP_DIRECT_RECLAIM  0x400000u
 #define ___GFP_OTHER_NODE      0x800000u
 #define ___GFP_WRITE           0x1000000u
+#define ___GFP_KSWAPD_RECLAIM  0x2000000u
 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
 
 /*
@@ -71,7 +72,7 @@ struct vm_area_struct;
  * __GFP_MOVABLE: Flag that this page will be movable by the page migration
  * mechanism or reclaimed
  */
-#define __GFP_WAIT     ((__force gfp_t)___GFP_WAIT)    /* Can wait and reschedule? */
+#define __GFP_ATOMIC   ((__force gfp_t)___GFP_ATOMIC)  /* Caller cannot wait or reschedule */
 #define __GFP_HIGH     ((__force gfp_t)___GFP_HIGH)    /* Should access emergency pools? */
 #define __GFP_IO       ((__force gfp_t)___GFP_IO)      /* Can start physical IO? */
 #define __GFP_FS       ((__force gfp_t)___GFP_FS)      /* Can call down to low-level FS? */
@@ -94,23 +95,37 @@ struct vm_area_struct;
 #define __GFP_NOACCOUNT        ((__force gfp_t)___GFP_NOACCOUNT) /* Don't account to kmemcg */
 #define __GFP_NOTRACK  ((__force gfp_t)___GFP_NOTRACK)  /* Don't track with kmemcheck */
 
-#define __GFP_NO_KSWAPD        ((__force gfp_t)___GFP_NO_KSWAPD)
 #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */
 #define __GFP_WRITE    ((__force gfp_t)___GFP_WRITE)   /* Allocator intends to dirty page */
 
+/*
+ * A caller that is willing to wait may enter direct reclaim and will
+ * wake kswapd to reclaim pages in the background until the high
+ * watermark is met. A caller may wish to clear __GFP_DIRECT_RECLAIM to
+ * avoid unnecessary delays when a fallback option is available but
+ * still allow kswapd to reclaim in the background. The kswapd flag
+ * can be cleared when the reclaiming of pages would cause unnecessary
+ * disruption.
+ */
+#define __GFP_WAIT ((__force gfp_t)(___GFP_DIRECT_RECLAIM|___GFP_KSWAPD_RECLAIM))
+#define __GFP_DIRECT_RECLAIM   ((__force gfp_t)___GFP_DIRECT_RECLAIM) /* Caller can reclaim */
+#define __GFP_KSWAPD_RECLAIM   ((__force gfp_t)___GFP_KSWAPD_RECLAIM) /* kswapd can wake */
+
 /*
  * This may seem redundant, but it's a way of annotating false positives vs.
  * allocations that simply cannot be supported (e.g. page tables).
  */
 #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK)
 
-#define __GFP_BITS_SHIFT 25    /* Room for N __GFP_FOO bits */
+#define __GFP_BITS_SHIFT 26    /* Room for N __GFP_FOO bits */
 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
 
-/* This equals 0, but use constants in case they ever change */
-#define GFP_NOWAIT     (GFP_ATOMIC & ~__GFP_HIGH)
-/* GFP_ATOMIC means both !wait (__GFP_WAIT not set) and use emergency pool */
-#define GFP_ATOMIC     (__GFP_HIGH)
+/*
+ * GFP_ATOMIC callers can not sleep, need the allocation to succeed.
+ * A lower watermark is applied to allow access to "atomic reserves"
+ */
+#define GFP_ATOMIC     (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM)
+#define GFP_NOWAIT     (__GFP_KSWAPD_RECLAIM)
 #define GFP_NOIO       (__GFP_WAIT)
 #define GFP_NOFS       (__GFP_WAIT | __GFP_IO)
 #define GFP_KERNEL     (__GFP_WAIT | __GFP_IO | __GFP_FS)
@@ -119,13 +134,14 @@ struct vm_area_struct;
 #define GFP_USER       (__GFP_WAIT | __GFP_IO | __GFP_FS | __GFP_HARDWALL)
 #define GFP_HIGHUSER   (GFP_USER | __GFP_HIGHMEM)
 #define GFP_HIGHUSER_MOVABLE   (GFP_HIGHUSER | __GFP_MOVABLE)
-#define GFP_IOFS       (__GFP_IO | __GFP_FS)
-#define GFP_TRANSHUGE  (GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
-                        __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | \
-                        __GFP_NO_KSWAPD)
+#define GFP_IOFS       (__GFP_IO | __GFP_FS | __GFP_KSWAPD_RECLAIM)
+#define GFP_TRANSHUGE  ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
+                        __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN) & \
+                        ~__GFP_KSWAPD_RECLAIM)
 
 /* This mask makes up all the page movable related flags */
 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
+#define GFP_MOVABLE_SHIFT 3
 
 /* Control page allocator reclaim behavior */
 #define GFP_RECLAIM_MASK (__GFP_WAIT|__GFP_HIGH|__GFP_IO|__GFP_FS|\
@@ -152,14 +168,20 @@ struct vm_area_struct;
 /* Convert GFP flags to their corresponding migrate type */
 static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
 {
-       WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
+       VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
+       BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
+       BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE);
 
        if (unlikely(page_group_by_mobility_disabled))
                return MIGRATE_UNMOVABLE;
 
        /* Group based on mobility */
-       return (((gfp_flags & __GFP_MOVABLE) != 0) << 1) |
-               ((gfp_flags & __GFP_RECLAIMABLE) != 0);
+       return (gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT;
+}
+
+static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
+{
+       return gfp_flags & __GFP_DIRECT_RECLAIM;
 }
 
 #ifdef CONFIG_HIGHMEM