mm/gup_test.c

   1 #include <linux/kernel.h>
   2 #include <linux/mm.h>
   3 #include <linux/slab.h>
   4 #include <linux/uaccess.h>
   5 #include <linux/ktime.h>
   6 #include <linux/debugfs.h>
   7 #include "gup_test.h"
   8
   9 static void put_back_pages(unsigned int cmd, struct page **pages,
  10                            unsigned long nr_pages, unsigned int gup_test_flags)
  11 {
  12         unsigned long i;
  13
  14         switch (cmd) {
  15         case GUP_FAST_BENCHMARK:
  16         case GUP_BASIC_TEST:
  17                 for (i = 0; i < nr_pages; i++)
  18                         put_page(pages[i]);
  19                 break;
  20
  21         case PIN_FAST_BENCHMARK:
  22         case PIN_BASIC_TEST:
  23         case PIN_LONGTERM_BENCHMARK:
  24                 unpin_user_pages(pages, nr_pages);
  25                 break;
  26         case DUMP_USER_PAGES_TEST:
  27                 if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
  28                         unpin_user_pages(pages, nr_pages);
  29                 } else {
  30                         for (i = 0; i < nr_pages; i++)
  31                                 put_page(pages[i]);
  32
  33                 }
  34                 break;
  35         }
  36 }
  37
  38 static void verify_dma_pinned(unsigned int cmd, struct page **pages,
  39                               unsigned long nr_pages)
  40 {
  41         unsigned long i;
  42         struct page *page;
  43
  44         switch (cmd) {
  45         case PIN_FAST_BENCHMARK:
  46         case PIN_BASIC_TEST:
  47         case PIN_LONGTERM_BENCHMARK:
  48                 for (i = 0; i < nr_pages; i++) {
  49                         page = pages[i];
  50                         if (WARN(!page_maybe_dma_pinned(page),
  51                                  "pages[%lu] is NOT dma-pinned\n", i)) {
  52
  53                                 dump_page(page, "gup_test failure");
  54                                 break;
  55                         } else if (cmd == PIN_LONGTERM_BENCHMARK &&
  56                                 WARN(!is_pinnable_page(page),
  57                                      "pages[%lu] is NOT pinnable but pinned\n",
  58                                      i)) {
  59                                 dump_page(page, "gup_test failure");
  60                                 break;
  61                         }
  62                 }
  63                 break;
  64         }
  65 }
  66
  67 static void dump_pages_test(struct gup_test *gup, struct page **pages,
  68                             unsigned long nr_pages)
  69 {
  70         unsigned int index_to_dump;
  71         unsigned int i;
  72
  73         /*
  74          * Zero out any user-supplied page index that is out of range. Remember:
  75          * .which_pages[] contains a 1-based set of page indices.
  76          */
  77         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  78                 if (gup->which_pages[i] > nr_pages) {
  79                         pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
  80                                 i, gup->which_pages[i]);
  81                         gup->which_pages[i] = 0;
  82                 }
  83         }
  84
  85         for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
  86                 index_to_dump = gup->which_pages[i];
  87
  88                 if (index_to_dump) {
  89                         index_to_dump--; // Decode from 1-based, to 0-based
  90                         pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
  91                                 index_to_dump, gup->addr);
  92                         dump_page(pages[index_to_dump],
  93                                   "gup_test: dump_pages() test");
  94                 }
  95         }
  96 }
  97
  98 static int __gup_test_ioctl(unsigned int cmd,
  99                 struct gup_test *gup)
 100 {
 101         ktime_t start_time, end_time;
 102         unsigned long i, nr_pages, addr, next;
 103         long nr;
 104         struct page **pages;
 105         int ret = 0;
 106         bool needs_mmap_lock =
 107                 cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
 108
 109         if (gup->size > ULONG_MAX)
 110                 return -EINVAL;
 111
 112         nr_pages = gup->size / PAGE_SIZE;
 113         pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
 114         if (!pages)
 115                 return -ENOMEM;
 116
 117         if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
 118                 ret = -EINTR;
 119                 goto free_pages;
 120         }
 121
 122         i = 0;
 123         nr = gup->nr_pages_per_call;
 124         start_time = ktime_get();
 125         for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
 126                 if (nr != gup->nr_pages_per_call)
 127                         break;
 128
 129                 next = addr + nr * PAGE_SIZE;
 130                 if (next > gup->addr + gup->size) {
 131                         next = gup->addr + gup->size;
 132                         nr = (next - addr) / PAGE_SIZE;
 133                 }
 134
 135                 switch (cmd) {
 136                 case GUP_FAST_BENCHMARK:
 137                         nr = get_user_pages_fast(addr, nr, gup->gup_flags,
 138                                                  pages + i);
 139                         break;
 140                 case GUP_BASIC_TEST:
 141                         nr = get_user_pages(addr, nr, gup->gup_flags, pages + i,
 142                                             NULL);
 143                         break;
 144                 case PIN_FAST_BENCHMARK:
 145                         nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
 146                                                  pages + i);
 147                         break;
 148                 case PIN_BASIC_TEST:
 149                         nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i,
 150                                             NULL);
 151                         break;
 152                 case PIN_LONGTERM_BENCHMARK:
 153                         nr = pin_user_pages(addr, nr,
 154                                             gup->gup_flags | FOLL_LONGTERM,
 155                                             pages + i, NULL);
 156                         break;
 157                 case DUMP_USER_PAGES_TEST:
 158                         if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
 159                                 nr = pin_user_pages(addr, nr, gup->gup_flags,
 160                                                     pages + i, NULL);
 161                         else
 162                                 nr = get_user_pages(addr, nr, gup->gup_flags,
 163                                                     pages + i, NULL);
 164                         break;
 165                 default:
 166                         ret = -EINVAL;
 167                         goto unlock;
 168                 }
 169
 170                 if (nr <= 0)
 171                         break;
 172                 i += nr;
 173         }
 174         end_time = ktime_get();
 175
 176         /* Shifting the meaning of nr_pages: now it is actual number pinned: */
 177         nr_pages = i;
 178
 179         gup->get_delta_usec = ktime_us_delta(end_time, start_time);
 180         gup->size = addr - gup->addr;
 181
 182         /*
 183          * Take an un-benchmark-timed moment to verify DMA pinned
 184          * state: print a warning if any non-dma-pinned pages are found:
 185          */
 186         verify_dma_pinned(cmd, pages, nr_pages);
 187
 188         if (cmd == DUMP_USER_PAGES_TEST)
 189                 dump_pages_test(gup, pages, nr_pages);
 190
 191         start_time = ktime_get();
 192
 193         put_back_pages(cmd, pages, nr_pages, gup->test_flags);
 194
 195         end_time = ktime_get();
 196         gup->put_delta_usec = ktime_us_delta(end_time, start_time);
 197
 198 unlock:
 199         if (needs_mmap_lock)
 200                 mmap_read_unlock(current->mm);
 201 free_pages:
 202         kvfree(pages);
 203         return ret;
 204 }
 205
 206 static long gup_test_ioctl(struct file *filep, unsigned int cmd,
 207                 unsigned long arg)
 208 {
 209         struct gup_test gup;
 210         int ret;
 211
 212         switch (cmd) {
 213         case GUP_FAST_BENCHMARK:
 214         case PIN_FAST_BENCHMARK:
 215         case PIN_LONGTERM_BENCHMARK:
 216         case GUP_BASIC_TEST:
 217         case PIN_BASIC_TEST:
 218         case DUMP_USER_PAGES_TEST:
 219                 break;
 220         default:
 221                 return -EINVAL;
 222         }
 223
 224         if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
 225                 return -EFAULT;
 226
 227         ret = __gup_test_ioctl(cmd, &gup);
 228         if (ret)
 229                 return ret;
 230
 231         if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
 232                 return -EFAULT;
 233
 234         return 0;
 235 }
 236
 237 static const struct file_operations gup_test_fops = {
 238         .open = nonseekable_open,
 239         .unlocked_ioctl = gup_test_ioctl,
 240 };
 241
 242 static int __init gup_test_init(void)
 243 {
 244         debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
 245                                    &gup_test_fops);
 246
 247         return 0;
 248 }
 249
 250 late_initcall(gup_test_init);