1 #include <linux/kernel.h>
3 #include <linux/slab.h>
4 #include <linux/uaccess.h>
5 #include <linux/ktime.h>
6 #include <linux/debugfs.h>
9 static void put_back_pages(unsigned int cmd
, struct page
**pages
,
10 unsigned long nr_pages
, unsigned int gup_test_flags
)
15 case GUP_FAST_BENCHMARK
:
17 for (i
= 0; i
< nr_pages
; i
++)
21 case PIN_FAST_BENCHMARK
:
23 case PIN_LONGTERM_BENCHMARK
:
24 unpin_user_pages(pages
, nr_pages
);
26 case DUMP_USER_PAGES_TEST
:
27 if (gup_test_flags
& GUP_TEST_FLAG_DUMP_PAGES_USE_PIN
) {
28 unpin_user_pages(pages
, nr_pages
);
30 for (i
= 0; i
< nr_pages
; i
++)
38 static void verify_dma_pinned(unsigned int cmd
, struct page
**pages
,
39 unsigned long nr_pages
)
45 case PIN_FAST_BENCHMARK
:
47 case PIN_LONGTERM_BENCHMARK
:
48 for (i
= 0; i
< nr_pages
; i
++) {
50 if (WARN(!page_maybe_dma_pinned(page
),
51 "pages[%lu] is NOT dma-pinned\n", i
)) {
53 dump_page(page
, "gup_test failure");
55 } else if (cmd
== PIN_LONGTERM_BENCHMARK
&&
56 WARN(!is_pinnable_page(page
),
57 "pages[%lu] is NOT pinnable but pinned\n",
59 dump_page(page
, "gup_test failure");
67 static void dump_pages_test(struct gup_test
*gup
, struct page
**pages
,
68 unsigned long nr_pages
)
70 unsigned int index_to_dump
;
74 * Zero out any user-supplied page index that is out of range. Remember:
75 * .which_pages[] contains a 1-based set of page indices.
77 for (i
= 0; i
< GUP_TEST_MAX_PAGES_TO_DUMP
; i
++) {
78 if (gup
->which_pages
[i
] > nr_pages
) {
79 pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
80 i
, gup
->which_pages
[i
]);
81 gup
->which_pages
[i
] = 0;
85 for (i
= 0; i
< GUP_TEST_MAX_PAGES_TO_DUMP
; i
++) {
86 index_to_dump
= gup
->which_pages
[i
];
89 index_to_dump
--; // Decode from 1-based, to 0-based
90 pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
91 index_to_dump
, gup
->addr
);
92 dump_page(pages
[index_to_dump
],
93 "gup_test: dump_pages() test");
98 static int __gup_test_ioctl(unsigned int cmd
,
101 ktime_t start_time
, end_time
;
102 unsigned long i
, nr_pages
, addr
, next
;
106 bool needs_mmap_lock
=
107 cmd
!= GUP_FAST_BENCHMARK
&& cmd
!= PIN_FAST_BENCHMARK
;
109 if (gup
->size
> ULONG_MAX
)
112 nr_pages
= gup
->size
/ PAGE_SIZE
;
113 pages
= kvcalloc(nr_pages
, sizeof(void *), GFP_KERNEL
);
117 if (needs_mmap_lock
&& mmap_read_lock_killable(current
->mm
)) {
123 nr
= gup
->nr_pages_per_call
;
124 start_time
= ktime_get();
125 for (addr
= gup
->addr
; addr
< gup
->addr
+ gup
->size
; addr
= next
) {
126 if (nr
!= gup
->nr_pages_per_call
)
129 next
= addr
+ nr
* PAGE_SIZE
;
130 if (next
> gup
->addr
+ gup
->size
) {
131 next
= gup
->addr
+ gup
->size
;
132 nr
= (next
- addr
) / PAGE_SIZE
;
136 case GUP_FAST_BENCHMARK
:
137 nr
= get_user_pages_fast(addr
, nr
, gup
->gup_flags
,
141 nr
= get_user_pages(addr
, nr
, gup
->gup_flags
, pages
+ i
,
144 case PIN_FAST_BENCHMARK
:
145 nr
= pin_user_pages_fast(addr
, nr
, gup
->gup_flags
,
149 nr
= pin_user_pages(addr
, nr
, gup
->gup_flags
, pages
+ i
,
152 case PIN_LONGTERM_BENCHMARK
:
153 nr
= pin_user_pages(addr
, nr
,
154 gup
->gup_flags
| FOLL_LONGTERM
,
157 case DUMP_USER_PAGES_TEST
:
158 if (gup
->test_flags
& GUP_TEST_FLAG_DUMP_PAGES_USE_PIN
)
159 nr
= pin_user_pages(addr
, nr
, gup
->gup_flags
,
162 nr
= get_user_pages(addr
, nr
, gup
->gup_flags
,
174 end_time
= ktime_get();
176 /* Shifting the meaning of nr_pages: now it is actual number pinned: */
179 gup
->get_delta_usec
= ktime_us_delta(end_time
, start_time
);
180 gup
->size
= addr
- gup
->addr
;
183 * Take an un-benchmark-timed moment to verify DMA pinned
184 * state: print a warning if any non-dma-pinned pages are found:
186 verify_dma_pinned(cmd
, pages
, nr_pages
);
188 if (cmd
== DUMP_USER_PAGES_TEST
)
189 dump_pages_test(gup
, pages
, nr_pages
);
191 start_time
= ktime_get();
193 put_back_pages(cmd
, pages
, nr_pages
, gup
->test_flags
);
195 end_time
= ktime_get();
196 gup
->put_delta_usec
= ktime_us_delta(end_time
, start_time
);
200 mmap_read_unlock(current
->mm
);
206 static long gup_test_ioctl(struct file
*filep
, unsigned int cmd
,
213 case GUP_FAST_BENCHMARK
:
214 case PIN_FAST_BENCHMARK
:
215 case PIN_LONGTERM_BENCHMARK
:
218 case DUMP_USER_PAGES_TEST
:
224 if (copy_from_user(&gup
, (void __user
*)arg
, sizeof(gup
)))
227 ret
= __gup_test_ioctl(cmd
, &gup
);
231 if (copy_to_user((void __user
*)arg
, &gup
, sizeof(gup
)))
237 static const struct file_operations gup_test_fops
= {
238 .open
= nonseekable_open
,
239 .unlocked_ioctl
= gup_test_ioctl
,
242 static int __init
gup_test_init(void)
244 debugfs_create_file_unsafe("gup_test", 0600, NULL
, NULL
,
250 late_initcall(gup_test_init
);