x86/mm: Drop usage of __flush_tlb_all() in kernel_physical_mapping_init()

[mirror_ubuntu-bionic-kernel.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index cfd56e5a35fb9790ac6426dd62abb9964f1ce72b..a996d84b190a00881ec6aadba9db3824aee4045a 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -710,7 +710,7 @@ void object_err(struct kmem_cache *s, struct page *page,
        print_trailer(s, page, object);
 }
 
-static void slab_err(struct kmem_cache *s, struct page *page,
+static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
                        const char *fmt, ...)
 {
        va_list args;
@@ -1360,10 +1360,8 @@ static inline void kfree_hook(const void *x)
        kasan_kfree_large(x);
 }
 
-static inline void *slab_free_hook(struct kmem_cache *s, void *x)
+static inline bool slab_free_hook(struct kmem_cache *s, void *x)
 {
-       void *freeptr;
-
        kmemleak_free_recursive(x, s->flags);
 
        /*
@@ -1383,17 +1381,12 @@ static inline void *slab_free_hook(struct kmem_cache *s, void *x)
        if (!(s->flags & SLAB_DEBUG_OBJECTS))
                debug_check_no_obj_freed(x, s->object_size);
 
-       freeptr = get_freepointer(s, x);
-       /*
-        * kasan_slab_free() may put x into memory quarantine, delaying its
-        * reuse. In this case the object's freelist pointer is changed.
-        */
-       kasan_slab_free(s, x);
-       return freeptr;
+       /* KASAN might put x into memory quarantine, delaying its reuse */
+       return kasan_slab_free(s, x);
 }
 
-static inline void slab_free_freelist_hook(struct kmem_cache *s,
-                                          void *head, void *tail)
+static inline bool slab_free_freelist_hook(struct kmem_cache *s,
+                                          void **head, void **tail)
 {
 /*
  * Compiler cannot detect this function can be removed if slab_free_hook()
@@ -1404,13 +1397,33 @@ static inline void slab_free_freelist_hook(struct kmem_cache *s,
        defined(CONFIG_DEBUG_OBJECTS_FREE) ||   \
        defined(CONFIG_KASAN)
 
-       void *object = head;
-       void *tail_obj = tail ? : head;
-       void *freeptr;
+       void *object;
+       void *next = *head;
+       void *old_tail = *tail ? *tail : *head;
+
+       /* Head and tail of the reconstructed freelist */
+       *head = NULL;
+       *tail = NULL;
 
        do {
-               freeptr = slab_free_hook(s, object);
-       } while ((object != tail_obj) && (object = freeptr));
+               object = next;
+               next = get_freepointer(s, object);
+               /* If object's reuse doesn't have to be delayed */
+               if (!slab_free_hook(s, object)) {
+                       /* Move object to the new freelist */
+                       set_freepointer(s, object, *head);
+                       *head = object;
+                       if (!*tail)
+                               *tail = object;
+               }
+       } while (object != old_tail);
+
+       if (*head == *tail)
+               *tail = NULL;
+
+       return *head != NULL;
+#else
+       return true;
 #endif
 }
 
@@ -2965,14 +2978,12 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
                                      void *head, void *tail, int cnt,
                                      unsigned long addr)
 {
-       slab_free_freelist_hook(s, head, tail);
        /*
-        * slab_free_freelist_hook() could have put the items into quarantine.
-        * If so, no need to free them.
+        * With KASAN enabled slab_free_freelist_hook modifies the freelist
+        * to remove objects, whose reuse must be delayed.
         */
-       if (s->flags & SLAB_KASAN && !(s->flags & SLAB_TYPESAFE_BY_RCU))
-               return;
-       do_slab_free(s, page, head, tail, cnt, addr);
+       if (slab_free_freelist_hook(s, &head, &tail))
+               do_slab_free(s, page, head, tail, cnt, addr);
 }
 
 #ifdef CONFIG_KASAN
@@ -5662,7 +5673,6 @@ static void sysfs_slab_remove_workfn(struct work_struct *work)
        kset_unregister(s->memcg_kset);
 #endif
        kobject_uevent(&s->kobj, KOBJ_REMOVE);
-       kobject_del(&s->kobj);
 out:
        kobject_put(&s->kobj);
 }
@@ -5747,6 +5757,12 @@ static void sysfs_slab_remove(struct kmem_cache *s)
        schedule_work(&s->kobj_remove_work);
 }
 
+void sysfs_slab_unlink(struct kmem_cache *s)
+{
+       if (slab_state >= FULL)
+               kobject_del(&s->kobj);
+}
+
 void sysfs_slab_release(struct kmem_cache *s)
 {
        if (slab_state >= FULL)