mm: memcg/slab: generalize postponed non-root kmem_cache deactivation

[mirror_ubuntu-jammy-kernel.git] / mm / slab.h

diff --git a/mm/slab.h b/mm/slab.h
index 43ac818b8592bc472b4b67e19831b404cc798aca..dc83583ee9dd4654fb879d52c2434818fea53f5f 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -172,6 +172,7 @@ int __kmem_cache_shutdown(struct kmem_cache *);
 void __kmem_cache_release(struct kmem_cache *);
 int __kmem_cache_shrink(struct kmem_cache *);
 void __kmemcg_cache_deactivate(struct kmem_cache *s);
+void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s);
 void slab_kmem_cache_release(struct kmem_cache *);
 
 struct seq_file;
@@ -289,9 +290,7 @@ static __always_inline void memcg_uncharge_slab(struct page *page, int order,
 }
 
 extern void slab_init_memcg_params(struct kmem_cache *);
-extern void memcg_link_cache(struct kmem_cache *s);
-extern void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,
-                               void (*deact_fn)(struct kmem_cache *));
+extern void memcg_link_cache(struct kmem_cache *s, struct mem_cgroup *memcg);
 
 #else /* CONFIG_MEMCG_KMEM */
 
@@ -310,7 +309,7 @@ static inline bool is_root_cache(struct kmem_cache *s)
 static inline bool slab_equal_or_root(struct kmem_cache *s,
                                      struct kmem_cache *p)
 {
-       return true;
+       return s == p;
 }
 
 static inline const char *cache_name(struct kmem_cache *s)
@@ -344,16 +343,27 @@ static inline void slab_init_memcg_params(struct kmem_cache *s)
 {
 }
 
-static inline void memcg_link_cache(struct kmem_cache *s)
+static inline void memcg_link_cache(struct kmem_cache *s,
+                                   struct mem_cgroup *memcg)
 {
 }
 
 #endif /* CONFIG_MEMCG_KMEM */
 
+static inline struct kmem_cache *virt_to_cache(const void *obj)
+{
+       struct page *page;
+
+       page = virt_to_head_page(obj);
+       if (WARN_ONCE(!PageSlab(page), "%s: Object is not a Slab page!\n",
+                                       __func__))
+               return NULL;
+       return page->slab_cache;
+}
+
 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
 {
        struct kmem_cache *cachep;
-       struct page *page;
 
        /*
         * When kmemcg is not being used, both assignments should return the
@@ -363,18 +373,15 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
         * will also be a constant.
         */
        if (!memcg_kmem_enabled() &&
+           !IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
            !unlikely(s->flags & SLAB_CONSISTENCY_CHECKS))
                return s;
 
-       page = virt_to_head_page(x);
-       cachep = page->slab_cache;
-       if (slab_equal_or_root(cachep, s))
-               return cachep;
-
-       pr_err("%s: Wrong slab cache. %s but object is from %s\n",
-              __func__, s->name, cachep->name);
-       WARN_ON_ONCE(1);
-       return s;
+       cachep = virt_to_cache(x);
+       WARN_ONCE(cachep && !slab_equal_or_root(cachep, s),
+                 "%s: Wrong slab cache. %s but object is from %s\n",
+                 __func__, s->name, cachep->name);
+       return cachep;
 }
 
 static inline size_t slab_ksize(const struct kmem_cache *s)