UBUNTU: Ubuntu-4.15.0-96.97

[mirror_ubuntu-bionic-kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index cfd56e5a35fb9790ac6426dd62abb9964f1ce72b..e4af90b21467145c9404a62b382dee4ffdb47e24 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -710,7 +710,7 @@ void object_err(struct kmem_cache *s, struct page *page,
        print_trailer(s, page, object);
 }
 
-static void slab_err(struct kmem_cache *s, struct page *page,
+static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
                        const char *fmt, ...)
 {
        va_list args;
@@ -1360,10 +1360,8 @@ static inline void kfree_hook(const void *x)
        kasan_kfree_large(x);
 }
 
-static inline void *slab_free_hook(struct kmem_cache *s, void *x)
+static inline bool slab_free_hook(struct kmem_cache *s, void *x)
 {
-       void *freeptr;
-
        kmemleak_free_recursive(x, s->flags);
 
        /*
@@ -1383,17 +1381,12 @@ static inline void *slab_free_hook(struct kmem_cache *s, void *x)
        if (!(s->flags & SLAB_DEBUG_OBJECTS))
                debug_check_no_obj_freed(x, s->object_size);
 
-       freeptr = get_freepointer(s, x);
-       /*
-        * kasan_slab_free() may put x into memory quarantine, delaying its
-        * reuse. In this case the object's freelist pointer is changed.
-        */
-       kasan_slab_free(s, x);
-       return freeptr;
+       /* KASAN might put x into memory quarantine, delaying its reuse */
+       return kasan_slab_free(s, x);
 }
 
-static inline void slab_free_freelist_hook(struct kmem_cache *s,
-                                          void *head, void *tail)
+static inline bool slab_free_freelist_hook(struct kmem_cache *s,
+                                          void **head, void **tail)
 {
 /*
  * Compiler cannot detect this function can be removed if slab_free_hook()
@@ -1404,13 +1397,33 @@ static inline void slab_free_freelist_hook(struct kmem_cache *s,
        defined(CONFIG_DEBUG_OBJECTS_FREE) ||   \
        defined(CONFIG_KASAN)
 
-       void *object = head;
-       void *tail_obj = tail ? : head;
-       void *freeptr;
+       void *object;
+       void *next = *head;
+       void *old_tail = *tail ? *tail : *head;
+
+       /* Head and tail of the reconstructed freelist */
+       *head = NULL;
+       *tail = NULL;
 
        do {
-               freeptr = slab_free_hook(s, object);
-       } while ((object != tail_obj) && (object = freeptr));
+               object = next;
+               next = get_freepointer(s, object);
+               /* If object's reuse doesn't have to be delayed */
+               if (!slab_free_hook(s, object)) {
+                       /* Move object to the new freelist */
+                       set_freepointer(s, object, *head);
+                       *head = object;
+                       if (!*tail)
+                               *tail = object;
+               }
+       } while (object != old_tail);
+
+       if (*head == *tail)
+               *tail = NULL;
+
+       return *head != NULL;
+#else
+       return true;
 #endif
 }
 
@@ -1809,7 +1822,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 {
        struct page *page, *page2;
        void *object = NULL;
-       int available = 0;
+       unsigned int available = 0;
        int objects;
 
        /*
@@ -2965,14 +2978,12 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
                                      void *head, void *tail, int cnt,
                                      unsigned long addr)
 {
-       slab_free_freelist_hook(s, head, tail);
        /*
-        * slab_free_freelist_hook() could have put the items into quarantine.
-        * If so, no need to free them.
+        * With KASAN enabled slab_free_freelist_hook modifies the freelist
+        * to remove objects, whose reuse must be delayed.
         */
-       if (s->flags & SLAB_KASAN && !(s->flags & SLAB_TYPESAFE_BY_RCU))
-               return;
-       do_slab_free(s, page, head, tail, cnt, addr);
+       if (slab_free_freelist_hook(s, &head, &tail))
+               do_slab_free(s, page, head, tail, cnt, addr);
 }
 
 #ifdef CONFIG_KASAN
@@ -3558,6 +3569,9 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
        if (s->flags & SLAB_CACHE_DMA)
                s->allocflags |= GFP_DMA;
 
+       if (s->flags & SLAB_CACHE_DMA32)
+               s->allocflags |= GFP_DMA32;
+
        if (s->flags & SLAB_RECLAIM_ACCOUNT)
                s->allocflags |= __GFP_RECLAIMABLE;
 
@@ -4792,7 +4806,17 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
                }
        }
 
-       get_online_mems();
+       /*
+        * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
+        * already held which will conflict with an existing lock order:
+        *
+        * mem_hotplug_lock->slab_mutex->kernfs_mutex
+        *
+        * We don't really need mem_hotplug_lock (to hold off
+        * slab_mem_going_offline_callback) here because slab's memory hot
+        * unplug code doesn't destroy the kmem_cache->node[] data.
+        */
+
 #ifdef CONFIG_SLUB_DEBUG
        if (flags & SO_ALL) {
                struct kmem_cache_node *n;
@@ -4833,7 +4857,6 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
                        x += sprintf(buf + x, " N%d=%lu",
                                        node, nodes[node]);
 #endif
-       put_online_mems();
        kfree(nodes);
        return x + sprintf(buf + x, "\n");
 }
@@ -4944,10 +4967,10 @@ static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
                                 size_t length)
 {
-       unsigned long objects;
+       unsigned int objects;
        int err;
 
-       err = kstrtoul(buf, 10, &objects);
+       err = kstrtouint(buf, 10, &objects);
        if (err)
                return err;
        if (objects && !kmem_cache_has_cpu_partial(s))
@@ -5630,6 +5653,8 @@ static char *create_unique_id(struct kmem_cache *s)
         */
        if (s->flags & SLAB_CACHE_DMA)
                *p++ = 'd';
+       if (s->flags & SLAB_CACHE_DMA32)
+               *p++ = 'D';
        if (s->flags & SLAB_RECLAIM_ACCOUNT)
                *p++ = 'a';
        if (s->flags & SLAB_CONSISTENCY_CHECKS)
@@ -5662,7 +5687,6 @@ static void sysfs_slab_remove_workfn(struct work_struct *work)
        kset_unregister(s->memcg_kset);
 #endif
        kobject_uevent(&s->kobj, KOBJ_REMOVE);
-       kobject_del(&s->kobj);
 out:
        kobject_put(&s->kobj);
 }
@@ -5747,6 +5771,12 @@ static void sysfs_slab_remove(struct kmem_cache *s)
        schedule_work(&s->kobj_remove_work);
 }
 
+void sysfs_slab_unlink(struct kmem_cache *s)
+{
+       if (slab_state >= FULL)
+               kobject_del(&s->kobj);
+}
+
 void sysfs_slab_release(struct kmem_cache *s)
 {
        if (slab_state >= FULL)