mm: switch gup_hugepte to use try_get_compound_head

[mirror_ubuntu-jammy-kernel.git] / kernel / jump_label.c

diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 0bfa10f4410c5d80049b94126ed97dab1b19010a..df3008419a1d0a34229580d86a2aec2f0e305d2a 100644 (file)
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -37,12 +37,26 @@ static int jump_label_cmp(const void *a, const void *b)
        const struct jump_entry *jea = a;
        const struct jump_entry *jeb = b;
 
+       /*
+        * Entrires are sorted by key.
+        */
        if (jump_entry_key(jea) < jump_entry_key(jeb))
                return -1;
 
        if (jump_entry_key(jea) > jump_entry_key(jeb))
                return 1;
 
+       /*
+        * In the batching mode, entries should also be sorted by the code
+        * inside the already sorted list of entries, enabling a bsearch in
+        * the vector.
+        */
+       if (jump_entry_code(jea) < jump_entry_code(jeb))
+               return -1;
+
+       if (jump_entry_code(jea) > jump_entry_code(jeb))
+               return 1;
+
        return 0;
 }
 
@@ -384,25 +398,55 @@ static enum jump_label_type jump_label_type(struct jump_entry *entry)
        return enabled ^ branch;
 }
 
+static bool jump_label_can_update(struct jump_entry *entry, bool init)
+{
+       /*
+        * Cannot update code that was in an init text area.
+        */
+       if (!init && jump_entry_is_init(entry))
+               return false;
+
+       if (!kernel_text_address(jump_entry_code(entry))) {
+               WARN_ONCE(1, "can't patch jump_label at %pS", (void *)jump_entry_code(entry));
+               return false;
+       }
+
+       return true;
+}
+
+#ifndef HAVE_JUMP_LABEL_BATCH
 static void __jump_label_update(struct static_key *key,
                                struct jump_entry *entry,
                                struct jump_entry *stop,
                                bool init)
 {
        for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
-               /*
-                * An entry->code of 0 indicates an entry which has been
-                * disabled because it was in an init text area.
-                */
-               if (init || !jump_entry_is_init(entry)) {
-                       if (kernel_text_address(jump_entry_code(entry)))
-                               arch_jump_label_transform(entry, jump_label_type(entry));
-                       else
-                               WARN_ONCE(1, "can't patch jump_label at %pS",
-                                         (void *)jump_entry_code(entry));
+               if (jump_label_can_update(entry, init))
+                       arch_jump_label_transform(entry, jump_label_type(entry));
+       }
+}
+#else
+static void __jump_label_update(struct static_key *key,
+                               struct jump_entry *entry,
+                               struct jump_entry *stop,
+                               bool init)
+{
+       for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
+
+               if (!jump_label_can_update(entry, init))
+                       continue;
+
+               if (!arch_jump_label_transform_queue(entry, jump_label_type(entry))) {
+                       /*
+                        * Queue is full: Apply the current queue and try again.
+                        */
+                       arch_jump_label_transform_apply();
+                       BUG_ON(!arch_jump_label_transform_queue(entry, jump_label_type(entry)));
                }
        }
+       arch_jump_label_transform_apply();
 }
+#endif
 
 void __init jump_label_init(void)
 {