]>
Commit | Line | Data |
---|---|---|
de5506e1 CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * Portions based on net/core/datagram.c and copyrighted by their authors. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License as published by the Free | |
7 | * Software Foundation; either version 2 of the License, or (at your option) | |
8 | * any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License along with | |
16 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
17 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
18 | * | |
19 | * The full GNU General Public License is included in this distribution in the | |
20 | * file called COPYING. | |
21 | */ | |
22 | ||
23 | /* | |
24 | * This code allows the net stack to make use of a DMA engine for | |
25 | * skb to iovec copies. | |
26 | */ | |
27 | ||
28 | #include <linux/dmaengine.h> | |
29 | #include <linux/pagemap.h> | |
30 | #include <net/tcp.h> /* for memcpy_toiovec */ | |
31 | #include <asm/io.h> | |
32 | #include <asm/uaccess.h> | |
33 | ||
56e0873b | 34 | static int num_pages_spanned(struct iovec *iov) |
de5506e1 CL |
35 | { |
36 | return | |
37 | ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - | |
38 | ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); | |
39 | } | |
40 | ||
41 | /* | |
42 | * Pin down all the iovec pages needed for len bytes. | |
43 | * Return a struct dma_pinned_list to keep track of pages pinned down. | |
44 | * | |
45 | * We are allocating a single chunk of memory, and then carving it up into | |
46 | * 3 sections, the latter 2 whose size depends on the number of iovecs and the | |
47 | * total number of pages, respectively. | |
48 | */ | |
49 | struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) | |
50 | { | |
51 | struct dma_pinned_list *local_list; | |
52 | struct page **pages; | |
53 | int i; | |
54 | int ret; | |
55 | int nr_iovecs = 0; | |
56 | int iovec_len_used = 0; | |
57 | int iovec_pages_used = 0; | |
58 | long err; | |
59 | ||
60 | /* don't pin down non-user-based iovecs */ | |
61 | if (segment_eq(get_fs(), KERNEL_DS)) | |
62 | return NULL; | |
63 | ||
64 | /* determine how many iovecs/pages there are, up front */ | |
65 | do { | |
66 | iovec_len_used += iov[nr_iovecs].iov_len; | |
67 | iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); | |
68 | nr_iovecs++; | |
69 | } while (iovec_len_used < len); | |
70 | ||
71 | /* single kmalloc for pinned list, page_list[], and the page arrays */ | |
72 | local_list = kmalloc(sizeof(*local_list) | |
73 | + (nr_iovecs * sizeof (struct dma_page_list)) | |
74 | + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); | |
75 | if (!local_list) { | |
76 | err = -ENOMEM; | |
77 | goto out; | |
78 | } | |
79 | ||
80 | /* list of pages starts right after the page list array */ | |
81 | pages = (struct page **) &local_list->page_list[nr_iovecs]; | |
82 | ||
83 | for (i = 0; i < nr_iovecs; i++) { | |
84 | struct dma_page_list *page_list = &local_list->page_list[i]; | |
85 | ||
86 | len -= iov[i].iov_len; | |
87 | ||
88 | if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) { | |
89 | err = -EFAULT; | |
90 | goto unpin; | |
91 | } | |
92 | ||
93 | page_list->nr_pages = num_pages_spanned(&iov[i]); | |
94 | page_list->base_address = iov[i].iov_base; | |
95 | ||
96 | page_list->pages = pages; | |
97 | pages += page_list->nr_pages; | |
98 | ||
99 | /* pin pages down */ | |
100 | down_read(¤t->mm->mmap_sem); | |
101 | ret = get_user_pages( | |
102 | current, | |
103 | current->mm, | |
104 | (unsigned long) iov[i].iov_base, | |
105 | page_list->nr_pages, | |
106 | 1, /* write */ | |
107 | 0, /* force */ | |
108 | page_list->pages, | |
109 | NULL); | |
110 | up_read(¤t->mm->mmap_sem); | |
111 | ||
112 | if (ret != page_list->nr_pages) { | |
113 | err = -ENOMEM; | |
114 | goto unpin; | |
115 | } | |
116 | ||
117 | local_list->nr_iovecs = i + 1; | |
118 | } | |
119 | ||
120 | return local_list; | |
121 | ||
122 | unpin: | |
123 | dma_unpin_iovec_pages(local_list); | |
124 | out: | |
125 | return ERR_PTR(err); | |
126 | } | |
127 | ||
128 | void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) | |
129 | { | |
130 | int i, j; | |
131 | ||
132 | if (!pinned_list) | |
133 | return; | |
134 | ||
135 | for (i = 0; i < pinned_list->nr_iovecs; i++) { | |
136 | struct dma_page_list *page_list = &pinned_list->page_list[i]; | |
137 | for (j = 0; j < page_list->nr_pages; j++) { | |
138 | set_page_dirty_lock(page_list->pages[j]); | |
139 | page_cache_release(page_list->pages[j]); | |
140 | } | |
141 | } | |
142 | ||
143 | kfree(pinned_list); | |
144 | } | |
145 | ||
de5506e1 CL |
146 | |
147 | /* | |
148 | * We have already pinned down the pages we will be using in the iovecs. | |
149 | * Each entry in iov array has corresponding entry in pinned_list->page_list. | |
150 | * Using array indexing to keep iov[] and page_list[] in sync. | |
151 | * Initial elements in iov array's iov->iov_len will be 0 if already copied into | |
152 | * by another call. | |
153 | * iov array length remaining guaranteed to be bigger than len. | |
154 | */ | |
155 | dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
156 | struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) | |
157 | { | |
158 | int iov_byte_offset; | |
159 | int copy; | |
160 | dma_cookie_t dma_cookie = 0; | |
161 | int iovec_idx; | |
162 | int page_idx; | |
163 | ||
164 | if (!chan) | |
165 | return memcpy_toiovec(iov, kdata, len); | |
166 | ||
de5506e1 CL |
167 | iovec_idx = 0; |
168 | while (iovec_idx < pinned_list->nr_iovecs) { | |
169 | struct dma_page_list *page_list; | |
170 | ||
171 | /* skip already used-up iovecs */ | |
172 | while (!iov[iovec_idx].iov_len) | |
173 | iovec_idx++; | |
174 | ||
175 | page_list = &pinned_list->page_list[iovec_idx]; | |
176 | ||
177 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
178 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
179 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
180 | ||
181 | /* break up copies to not cross page boundary */ | |
182 | while (iov[iovec_idx].iov_len) { | |
183 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
184 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
185 | ||
186 | dma_cookie = dma_async_memcpy_buf_to_pg(chan, | |
187 | page_list->pages[page_idx], | |
188 | iov_byte_offset, | |
189 | kdata, | |
190 | copy); | |
191 | ||
192 | len -= copy; | |
193 | iov[iovec_idx].iov_len -= copy; | |
194 | iov[iovec_idx].iov_base += copy; | |
195 | ||
196 | if (!len) | |
197 | return dma_cookie; | |
198 | ||
199 | kdata += copy; | |
200 | iov_byte_offset = 0; | |
201 | page_idx++; | |
202 | } | |
203 | iovec_idx++; | |
204 | } | |
205 | ||
206 | /* really bad if we ever run out of iovecs */ | |
207 | BUG(); | |
208 | return -EFAULT; | |
209 | } | |
210 | ||
211 | dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
212 | struct dma_pinned_list *pinned_list, struct page *page, | |
213 | unsigned int offset, size_t len) | |
214 | { | |
215 | int iov_byte_offset; | |
216 | int copy; | |
217 | dma_cookie_t dma_cookie = 0; | |
218 | int iovec_idx; | |
219 | int page_idx; | |
220 | int err; | |
221 | ||
222 | /* this needs as-yet-unimplemented buf-to-buff, so punt. */ | |
223 | /* TODO: use dma for this */ | |
224 | if (!chan || !pinned_list) { | |
225 | u8 *vaddr = kmap(page); | |
226 | err = memcpy_toiovec(iov, vaddr + offset, len); | |
227 | kunmap(page); | |
228 | return err; | |
229 | } | |
230 | ||
231 | iovec_idx = 0; | |
232 | while (iovec_idx < pinned_list->nr_iovecs) { | |
233 | struct dma_page_list *page_list; | |
234 | ||
235 | /* skip already used-up iovecs */ | |
236 | while (!iov[iovec_idx].iov_len) | |
237 | iovec_idx++; | |
238 | ||
239 | page_list = &pinned_list->page_list[iovec_idx]; | |
240 | ||
241 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
242 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
243 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
244 | ||
245 | /* break up copies to not cross page boundary */ | |
246 | while (iov[iovec_idx].iov_len) { | |
247 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
248 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
249 | ||
250 | dma_cookie = dma_async_memcpy_pg_to_pg(chan, | |
251 | page_list->pages[page_idx], | |
252 | iov_byte_offset, | |
253 | page, | |
254 | offset, | |
255 | copy); | |
256 | ||
257 | len -= copy; | |
258 | iov[iovec_idx].iov_len -= copy; | |
259 | iov[iovec_idx].iov_base += copy; | |
260 | ||
261 | if (!len) | |
262 | return dma_cookie; | |
263 | ||
264 | offset += copy; | |
265 | iov_byte_offset = 0; | |
266 | page_idx++; | |
267 | } | |
268 | iovec_idx++; | |
269 | } | |
270 | ||
271 | /* really bad if we ever run out of iovecs */ | |
272 | BUG(); | |
273 | return -EFAULT; | |
274 | } |