mm: switch gup_hugepte to use try_get_compound_head

[mirror_ubuntu-jammy-kernel.git] / kernel / jump_label.c

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 24f0d3b1526b115dbec8a383916ff9e3b68b130e..df3008419a1d0a34229580d86a2aec2f0e305d2a 100644 (file)
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -37,12 +37,26 @@ static int jump_label_cmp(const void *a, const void *b)
        const struct jump_entry *jea = a;
        const struct jump_entry *jeb = b;
 
+       /*
+        * Entrires are sorted by key.
+        */
        if (jump_entry_key(jea) < jump_entry_key(jeb))
                return -1;
 
        if (jump_entry_key(jea) > jump_entry_key(jeb))
                return 1;
 
+       /*
+        * In the batching mode, entries should also be sorted by the code
+        * inside the already sorted list of entries, enabling a bsearch in
+        * the vector.
+        */
+       if (jump_entry_code(jea) < jump_entry_code(jeb))
+               return -1;
+
+       if (jump_entry_code(jea) > jump_entry_code(jeb))
+               return 1;
+
        return 0;
 }
 
@@ -400,6 +414,7 @@ static bool jump_label_can_update(struct jump_entry *entry, bool init)
        return true;
 }
 
+#ifndef HAVE_JUMP_LABEL_BATCH
 static void __jump_label_update(struct static_key *key,
                                struct jump_entry *entry,
                                struct jump_entry *stop,
@@ -410,6 +425,28 @@ static void __jump_label_update(struct static_key *key,
                        arch_jump_label_transform(entry, jump_label_type(entry));
        }
 }
+#else
+static void __jump_label_update(struct static_key *key,
+                               struct jump_entry *entry,
+                               struct jump_entry *stop,
+                               bool init)
+{
+       for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
+
+               if (!jump_label_can_update(entry, init))
+                       continue;
+
+               if (!arch_jump_label_transform_queue(entry, jump_label_type(entry))) {
+                       /*
+                        * Queue is full: Apply the current queue and try again.
+                        */
+                       arch_jump_label_transform_apply();
+                       BUG_ON(!arch_jump_label_transform_queue(entry, jump_label_type(entry)));
+               }
+       }
+       arch_jump_label_transform_apply();
+}
+#endif
 
 void __init jump_label_init(void)
 {