]>
Commit | Line | Data |
---|---|---|
de5506e1 CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * Portions based on net/core/datagram.c and copyrighted by their authors. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License as published by the Free | |
7 | * Software Foundation; either version 2 of the License, or (at your option) | |
8 | * any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License along with | |
16 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
17 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
18 | * | |
19 | * The full GNU General Public License is included in this distribution in the | |
20 | * file called COPYING. | |
21 | */ | |
22 | ||
23 | /* | |
24 | * This code allows the net stack to make use of a DMA engine for | |
25 | * skb to iovec copies. | |
26 | */ | |
27 | ||
28 | #include <linux/dmaengine.h> | |
29 | #include <linux/pagemap.h> | |
5a0e3ad6 | 30 | #include <linux/slab.h> |
de5506e1 CL |
31 | #include <net/tcp.h> /* for memcpy_toiovec */ |
32 | #include <asm/io.h> | |
33 | #include <asm/uaccess.h> | |
34 | ||
56e0873b | 35 | static int num_pages_spanned(struct iovec *iov) |
de5506e1 CL |
36 | { |
37 | return | |
38 | ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) - | |
39 | ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT); | |
40 | } | |
41 | ||
42 | /* | |
43 | * Pin down all the iovec pages needed for len bytes. | |
44 | * Return a struct dma_pinned_list to keep track of pages pinned down. | |
45 | * | |
46 | * We are allocating a single chunk of memory, and then carving it up into | |
47 | * 3 sections, the latter 2 whose size depends on the number of iovecs and the | |
48 | * total number of pages, respectively. | |
49 | */ | |
50 | struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len) | |
51 | { | |
52 | struct dma_pinned_list *local_list; | |
53 | struct page **pages; | |
54 | int i; | |
55 | int ret; | |
56 | int nr_iovecs = 0; | |
57 | int iovec_len_used = 0; | |
58 | int iovec_pages_used = 0; | |
de5506e1 CL |
59 | |
60 | /* don't pin down non-user-based iovecs */ | |
61 | if (segment_eq(get_fs(), KERNEL_DS)) | |
62 | return NULL; | |
63 | ||
64 | /* determine how many iovecs/pages there are, up front */ | |
65 | do { | |
66 | iovec_len_used += iov[nr_iovecs].iov_len; | |
67 | iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]); | |
68 | nr_iovecs++; | |
69 | } while (iovec_len_used < len); | |
70 | ||
71 | /* single kmalloc for pinned list, page_list[], and the page arrays */ | |
72 | local_list = kmalloc(sizeof(*local_list) | |
73 | + (nr_iovecs * sizeof (struct dma_page_list)) | |
74 | + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL); | |
c2c0b4c5 | 75 | if (!local_list) |
de5506e1 | 76 | goto out; |
de5506e1 CL |
77 | |
78 | /* list of pages starts right after the page list array */ | |
79 | pages = (struct page **) &local_list->page_list[nr_iovecs]; | |
80 | ||
c2c0b4c5 MS |
81 | local_list->nr_iovecs = 0; |
82 | ||
de5506e1 CL |
83 | for (i = 0; i < nr_iovecs; i++) { |
84 | struct dma_page_list *page_list = &local_list->page_list[i]; | |
85 | ||
86 | len -= iov[i].iov_len; | |
87 | ||
c2c0b4c5 | 88 | if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) |
de5506e1 | 89 | goto unpin; |
de5506e1 CL |
90 | |
91 | page_list->nr_pages = num_pages_spanned(&iov[i]); | |
92 | page_list->base_address = iov[i].iov_base; | |
93 | ||
94 | page_list->pages = pages; | |
95 | pages += page_list->nr_pages; | |
96 | ||
97 | /* pin pages down */ | |
98 | down_read(¤t->mm->mmap_sem); | |
99 | ret = get_user_pages( | |
100 | current, | |
101 | current->mm, | |
102 | (unsigned long) iov[i].iov_base, | |
103 | page_list->nr_pages, | |
104 | 1, /* write */ | |
105 | 0, /* force */ | |
106 | page_list->pages, | |
107 | NULL); | |
108 | up_read(¤t->mm->mmap_sem); | |
109 | ||
c2c0b4c5 | 110 | if (ret != page_list->nr_pages) |
de5506e1 | 111 | goto unpin; |
de5506e1 CL |
112 | |
113 | local_list->nr_iovecs = i + 1; | |
114 | } | |
115 | ||
116 | return local_list; | |
117 | ||
118 | unpin: | |
119 | dma_unpin_iovec_pages(local_list); | |
120 | out: | |
c2c0b4c5 | 121 | return NULL; |
de5506e1 CL |
122 | } |
123 | ||
124 | void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list) | |
125 | { | |
126 | int i, j; | |
127 | ||
128 | if (!pinned_list) | |
129 | return; | |
130 | ||
131 | for (i = 0; i < pinned_list->nr_iovecs; i++) { | |
132 | struct dma_page_list *page_list = &pinned_list->page_list[i]; | |
133 | for (j = 0; j < page_list->nr_pages; j++) { | |
134 | set_page_dirty_lock(page_list->pages[j]); | |
135 | page_cache_release(page_list->pages[j]); | |
136 | } | |
137 | } | |
138 | ||
139 | kfree(pinned_list); | |
140 | } | |
141 | ||
de5506e1 CL |
142 | |
143 | /* | |
144 | * We have already pinned down the pages we will be using in the iovecs. | |
145 | * Each entry in iov array has corresponding entry in pinned_list->page_list. | |
146 | * Using array indexing to keep iov[] and page_list[] in sync. | |
147 | * Initial elements in iov array's iov->iov_len will be 0 if already copied into | |
148 | * by another call. | |
149 | * iov array length remaining guaranteed to be bigger than len. | |
150 | */ | |
151 | dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
152 | struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len) | |
153 | { | |
154 | int iov_byte_offset; | |
155 | int copy; | |
156 | dma_cookie_t dma_cookie = 0; | |
157 | int iovec_idx; | |
158 | int page_idx; | |
159 | ||
160 | if (!chan) | |
161 | return memcpy_toiovec(iov, kdata, len); | |
162 | ||
de5506e1 CL |
163 | iovec_idx = 0; |
164 | while (iovec_idx < pinned_list->nr_iovecs) { | |
165 | struct dma_page_list *page_list; | |
166 | ||
167 | /* skip already used-up iovecs */ | |
168 | while (!iov[iovec_idx].iov_len) | |
169 | iovec_idx++; | |
170 | ||
171 | page_list = &pinned_list->page_list[iovec_idx]; | |
172 | ||
173 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
174 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
175 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
176 | ||
177 | /* break up copies to not cross page boundary */ | |
178 | while (iov[iovec_idx].iov_len) { | |
179 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
180 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
181 | ||
182 | dma_cookie = dma_async_memcpy_buf_to_pg(chan, | |
183 | page_list->pages[page_idx], | |
184 | iov_byte_offset, | |
185 | kdata, | |
186 | copy); | |
4b652f0d DW |
187 | /* poll for a descriptor slot */ |
188 | if (unlikely(dma_cookie < 0)) { | |
189 | dma_async_issue_pending(chan); | |
190 | continue; | |
191 | } | |
de5506e1 CL |
192 | |
193 | len -= copy; | |
194 | iov[iovec_idx].iov_len -= copy; | |
195 | iov[iovec_idx].iov_base += copy; | |
196 | ||
197 | if (!len) | |
198 | return dma_cookie; | |
199 | ||
200 | kdata += copy; | |
201 | iov_byte_offset = 0; | |
202 | page_idx++; | |
203 | } | |
204 | iovec_idx++; | |
205 | } | |
206 | ||
207 | /* really bad if we ever run out of iovecs */ | |
208 | BUG(); | |
209 | return -EFAULT; | |
210 | } | |
211 | ||
212 | dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov, | |
213 | struct dma_pinned_list *pinned_list, struct page *page, | |
214 | unsigned int offset, size_t len) | |
215 | { | |
216 | int iov_byte_offset; | |
217 | int copy; | |
218 | dma_cookie_t dma_cookie = 0; | |
219 | int iovec_idx; | |
220 | int page_idx; | |
221 | int err; | |
222 | ||
223 | /* this needs as-yet-unimplemented buf-to-buff, so punt. */ | |
224 | /* TODO: use dma for this */ | |
225 | if (!chan || !pinned_list) { | |
226 | u8 *vaddr = kmap(page); | |
227 | err = memcpy_toiovec(iov, vaddr + offset, len); | |
228 | kunmap(page); | |
229 | return err; | |
230 | } | |
231 | ||
232 | iovec_idx = 0; | |
233 | while (iovec_idx < pinned_list->nr_iovecs) { | |
234 | struct dma_page_list *page_list; | |
235 | ||
236 | /* skip already used-up iovecs */ | |
237 | while (!iov[iovec_idx].iov_len) | |
238 | iovec_idx++; | |
239 | ||
240 | page_list = &pinned_list->page_list[iovec_idx]; | |
241 | ||
242 | iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK); | |
243 | page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK) | |
244 | - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT; | |
245 | ||
246 | /* break up copies to not cross page boundary */ | |
247 | while (iov[iovec_idx].iov_len) { | |
248 | copy = min_t(int, PAGE_SIZE - iov_byte_offset, len); | |
249 | copy = min_t(int, copy, iov[iovec_idx].iov_len); | |
250 | ||
251 | dma_cookie = dma_async_memcpy_pg_to_pg(chan, | |
252 | page_list->pages[page_idx], | |
253 | iov_byte_offset, | |
254 | page, | |
255 | offset, | |
256 | copy); | |
4b652f0d DW |
257 | /* poll for a descriptor slot */ |
258 | if (unlikely(dma_cookie < 0)) { | |
259 | dma_async_issue_pending(chan); | |
260 | continue; | |
261 | } | |
de5506e1 CL |
262 | |
263 | len -= copy; | |
264 | iov[iovec_idx].iov_len -= copy; | |
265 | iov[iovec_idx].iov_base += copy; | |
266 | ||
267 | if (!len) | |
268 | return dma_cookie; | |
269 | ||
270 | offset += copy; | |
271 | iov_byte_offset = 0; | |
272 | page_idx++; | |
273 | } | |
274 | iovec_idx++; | |
275 | } | |
276 | ||
277 | /* really bad if we ever run out of iovecs */ | |
278 | BUG(); | |
279 | return -EFAULT; | |
280 | } |