Documentation: Add documentation for Processor MMIO Stale Data

[mirror_ubuntu-jammy-kernel.git] / mm / memfd.c

diff --git a/mm/memfd.c b/mm/memfd.c
index 081dd33e6a61b4bf573e059c16b605696e32fa4a..475d095dd7f5326141c605ab4da781f7db6ca7b5 100644 (file)
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -31,20 +31,28 @@
 static void memfd_tag_pins(struct xa_state *xas)
 {
        struct page *page;
-       unsigned int tagged = 0;
+       int latency = 0;
+       int cache_count;
 
        lru_add_drain();
 
        xas_lock_irq(xas);
        xas_for_each(xas, page, ULONG_MAX) {
-               if (xa_is_value(page))
-                       continue;
-               page = find_subpage(page, xas->xa_index);
-               if (page_count(page) - page_mapcount(page) > 1)
+               cache_count = 1;
+               if (!xa_is_value(page) &&
+                   PageTransHuge(page) && !PageHuge(page))
+                       cache_count = HPAGE_PMD_NR;
+
+               if (!xa_is_value(page) &&
+                   page_count(page) - total_mapcount(page) != cache_count)
                        xas_set_mark(xas, MEMFD_TAG_PINNED);
+               if (cache_count != 1)
+                       xas_set(xas, page->index + cache_count);
 
-               if (++tagged % XA_CHECK_SCHED)
+               latency += cache_count;
+               if (latency < XA_CHECK_SCHED)
                        continue;
+               latency = 0;
 
                xas_pause(xas);
                xas_unlock_irq(xas);
@@ -73,7 +81,8 @@ static int memfd_wait_for_pins(struct address_space *mapping)
 
        error = 0;
        for (scan = 0; scan <= LAST_SCAN; scan++) {
-               unsigned int tagged = 0;
+               int latency = 0;
+               int cache_count;
 
                if (!xas_marked(&xas, MEMFD_TAG_PINNED))
                        break;
@@ -87,10 +96,14 @@ static int memfd_wait_for_pins(struct address_space *mapping)
                xas_lock_irq(&xas);
                xas_for_each_marked(&xas, page, ULONG_MAX, MEMFD_TAG_PINNED) {
                        bool clear = true;
-                       if (xa_is_value(page))
-                               continue;
-                       page = find_subpage(page, xas.xa_index);
-                       if (page_count(page) - page_mapcount(page) != 1) {
+
+                       cache_count = 1;
+                       if (!xa_is_value(page) &&
+                           PageTransHuge(page) && !PageHuge(page))
+                               cache_count = HPAGE_PMD_NR;
+
+                       if (!xa_is_value(page) && cache_count !=
+                           page_count(page) - total_mapcount(page)) {
                                /*
                                 * On the last scan, we clean up all those tags
                                 * we inserted; but make a note that we still
@@ -103,8 +116,11 @@ static int memfd_wait_for_pins(struct address_space *mapping)
                        }
                        if (clear)
                                xas_clear_mark(&xas, MEMFD_TAG_PINNED);
-                       if (++tagged % XA_CHECK_SCHED)
+
+                       latency += cache_count;
+                       if (latency < XA_CHECK_SCHED)
                                continue;
+                       latency = 0;
 
                        xas_pause(&xas);
                        xas_unlock_irq(&xas);