]>
Commit | Line | Data |
---|---|---|
64c349f4 KS |
1 | #include <linux/kernel.h> |
2 | #include <linux/mm.h> | |
3 | #include <linux/slab.h> | |
4 | #include <linux/uaccess.h> | |
5 | #include <linux/ktime.h> | |
6 | #include <linux/debugfs.h> | |
7 | ||
8 | #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark) | |
657d4f79 BS |
9 | #define GUP_BENCHMARK _IOWR('g', 2, struct gup_benchmark) |
10 | #define PIN_FAST_BENCHMARK _IOWR('g', 3, struct gup_benchmark) | |
11 | #define PIN_BENCHMARK _IOWR('g', 4, struct gup_benchmark) | |
12 | #define PIN_LONGTERM_BENCHMARK _IOWR('g', 5, struct gup_benchmark) | |
64c349f4 KS |
13 | |
14 | struct gup_benchmark { | |
26db3d09 KB |
15 | __u64 get_delta_usec; |
16 | __u64 put_delta_usec; | |
64c349f4 KS |
17 | __u64 addr; |
18 | __u64 size; | |
19 | __u32 nr_pages_per_call; | |
20 | __u32 flags; | |
26db3d09 | 21 | __u64 expansion[10]; /* For future use */ |
64c349f4 KS |
22 | }; |
23 | ||
41c45d37 JH |
24 | static void put_back_pages(unsigned int cmd, struct page **pages, |
25 | unsigned long nr_pages) | |
26 | { | |
27 | unsigned long i; | |
28 | ||
29 | switch (cmd) { | |
30 | case GUP_FAST_BENCHMARK: | |
41c45d37 JH |
31 | case GUP_BENCHMARK: |
32 | for (i = 0; i < nr_pages; i++) | |
33 | put_page(pages[i]); | |
34 | break; | |
35 | ||
36 | case PIN_FAST_BENCHMARK: | |
37 | case PIN_BENCHMARK: | |
657d4f79 | 38 | case PIN_LONGTERM_BENCHMARK: |
41c45d37 JH |
39 | unpin_user_pages(pages, nr_pages); |
40 | break; | |
41 | } | |
42 | } | |
43 | ||
44 | static void verify_dma_pinned(unsigned int cmd, struct page **pages, | |
45 | unsigned long nr_pages) | |
46 | { | |
47 | unsigned long i; | |
48 | struct page *page; | |
49 | ||
50 | switch (cmd) { | |
51 | case PIN_FAST_BENCHMARK: | |
52 | case PIN_BENCHMARK: | |
657d4f79 | 53 | case PIN_LONGTERM_BENCHMARK: |
41c45d37 JH |
54 | for (i = 0; i < nr_pages; i++) { |
55 | page = pages[i]; | |
56 | if (WARN(!page_maybe_dma_pinned(page), | |
57 | "pages[%lu] is NOT dma-pinned\n", i)) { | |
58 | ||
59 | dump_page(page, "gup_benchmark failure"); | |
60 | break; | |
61 | } | |
62 | } | |
63 | break; | |
64 | } | |
65 | } | |
66 | ||
64c349f4 KS |
67 | static int __gup_benchmark_ioctl(unsigned int cmd, |
68 | struct gup_benchmark *gup) | |
69 | { | |
70 | ktime_t start_time, end_time; | |
51896864 Y |
71 | unsigned long i, nr_pages, addr, next; |
72 | int nr; | |
64c349f4 | 73 | struct page **pages; |
a7c46c0c | 74 | int ret = 0; |
f3964599 JH |
75 | bool needs_mmap_lock = |
76 | cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; | |
64c349f4 | 77 | |
4b408c74 DC |
78 | if (gup->size > ULONG_MAX) |
79 | return -EINVAL; | |
80 | ||
64c349f4 | 81 | nr_pages = gup->size / PAGE_SIZE; |
778e1cdd | 82 | pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL); |
64c349f4 KS |
83 | if (!pages) |
84 | return -ENOMEM; | |
85 | ||
f3964599 JH |
86 | if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) { |
87 | ret = -EINTR; | |
88 | goto free_pages; | |
89 | } | |
90 | ||
64c349f4 KS |
91 | i = 0; |
92 | nr = gup->nr_pages_per_call; | |
93 | start_time = ktime_get(); | |
94 | for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { | |
95 | if (nr != gup->nr_pages_per_call) | |
96 | break; | |
97 | ||
98 | next = addr + nr * PAGE_SIZE; | |
99 | if (next > gup->addr + gup->size) { | |
100 | next = gup->addr + gup->size; | |
101 | nr = (next - addr) / PAGE_SIZE; | |
102 | } | |
103 | ||
bdffe23e JH |
104 | /* Filter out most gup flags: only allow a tiny subset here: */ |
105 | gup->flags &= FOLL_WRITE; | |
106 | ||
714a3a1e KB |
107 | switch (cmd) { |
108 | case GUP_FAST_BENCHMARK: | |
bdffe23e | 109 | nr = get_user_pages_fast(addr, nr, gup->flags, |
714a3a1e KB |
110 | pages + i); |
111 | break; | |
714a3a1e | 112 | case GUP_BENCHMARK: |
bdffe23e | 113 | nr = get_user_pages(addr, nr, gup->flags, pages + i, |
714a3a1e KB |
114 | NULL); |
115 | break; | |
41c45d37 JH |
116 | case PIN_FAST_BENCHMARK: |
117 | nr = pin_user_pages_fast(addr, nr, gup->flags, | |
118 | pages + i); | |
119 | break; | |
120 | case PIN_BENCHMARK: | |
121 | nr = pin_user_pages(addr, nr, gup->flags, pages + i, | |
122 | NULL); | |
123 | break; | |
657d4f79 BS |
124 | case PIN_LONGTERM_BENCHMARK: |
125 | nr = pin_user_pages(addr, nr, | |
126 | gup->flags | FOLL_LONGTERM, | |
127 | pages + i, NULL); | |
128 | break; | |
714a3a1e | 129 | default: |
a7c46c0c | 130 | ret = -EINVAL; |
f3964599 | 131 | goto unlock; |
714a3a1e KB |
132 | } |
133 | ||
09e35a4a MT |
134 | if (nr <= 0) |
135 | break; | |
64c349f4 KS |
136 | i += nr; |
137 | } | |
138 | end_time = ktime_get(); | |
139 | ||
41c45d37 JH |
140 | /* Shifting the meaning of nr_pages: now it is actual number pinned: */ |
141 | nr_pages = i; | |
142 | ||
26db3d09 | 143 | gup->get_delta_usec = ktime_us_delta(end_time, start_time); |
64c349f4 KS |
144 | gup->size = addr - gup->addr; |
145 | ||
41c45d37 JH |
146 | /* |
147 | * Take an un-benchmark-timed moment to verify DMA pinned | |
148 | * state: print a warning if any non-dma-pinned pages are found: | |
149 | */ | |
150 | verify_dma_pinned(cmd, pages, nr_pages); | |
151 | ||
26db3d09 | 152 | start_time = ktime_get(); |
41c45d37 JH |
153 | |
154 | put_back_pages(cmd, pages, nr_pages); | |
155 | ||
26db3d09 KB |
156 | end_time = ktime_get(); |
157 | gup->put_delta_usec = ktime_us_delta(end_time, start_time); | |
64c349f4 | 158 | |
f3964599 JH |
159 | unlock: |
160 | if (needs_mmap_lock) | |
161 | mmap_read_unlock(current->mm); | |
162 | free_pages: | |
64c349f4 | 163 | kvfree(pages); |
a7c46c0c | 164 | return ret; |
64c349f4 KS |
165 | } |
166 | ||
167 | static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd, | |
168 | unsigned long arg) | |
169 | { | |
170 | struct gup_benchmark gup; | |
171 | int ret; | |
172 | ||
714a3a1e KB |
173 | switch (cmd) { |
174 | case GUP_FAST_BENCHMARK: | |
714a3a1e | 175 | case GUP_BENCHMARK: |
41c45d37 JH |
176 | case PIN_FAST_BENCHMARK: |
177 | case PIN_BENCHMARK: | |
657d4f79 | 178 | case PIN_LONGTERM_BENCHMARK: |
714a3a1e KB |
179 | break; |
180 | default: | |
64c349f4 | 181 | return -EINVAL; |
714a3a1e | 182 | } |
64c349f4 KS |
183 | |
184 | if (copy_from_user(&gup, (void __user *)arg, sizeof(gup))) | |
185 | return -EFAULT; | |
186 | ||
187 | ret = __gup_benchmark_ioctl(cmd, &gup); | |
188 | if (ret) | |
189 | return ret; | |
190 | ||
191 | if (copy_to_user((void __user *)arg, &gup, sizeof(gup))) | |
192 | return -EFAULT; | |
193 | ||
194 | return 0; | |
195 | } | |
196 | ||
197 | static const struct file_operations gup_benchmark_fops = { | |
198 | .open = nonseekable_open, | |
199 | .unlocked_ioctl = gup_benchmark_ioctl, | |
200 | }; | |
201 | ||
202 | static int gup_benchmark_init(void) | |
203 | { | |
d9f7979c GKH |
204 | debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL, |
205 | &gup_benchmark_fops); | |
64c349f4 KS |
206 | |
207 | return 0; | |
208 | } | |
209 | ||
210 | late_initcall(gup_benchmark_init); |