]>
Commit | Line | Data |
---|---|---|
7875e18e AG |
1 | /* |
2 | * Copyright (c) 2006 Oracle. All rights reserved. | |
3 | * | |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | * | |
32 | */ | |
33 | #include <linux/kernel.h> | |
5a0e3ad6 | 34 | #include <linux/slab.h> |
bc3b2d7f | 35 | #include <linux/export.h> |
01883eda SV |
36 | #include <linux/skbuff.h> |
37 | #include <linux/list.h> | |
38 | #include <linux/errqueue.h> | |
7875e18e AG |
39 | |
40 | #include "rds.h" | |
7875e18e | 41 | |
7875e18e AG |
42 | static unsigned int rds_exthdr_size[__RDS_EXTHDR_MAX] = { |
43 | [RDS_EXTHDR_NONE] = 0, | |
44 | [RDS_EXTHDR_VERSION] = sizeof(struct rds_ext_header_version), | |
45 | [RDS_EXTHDR_RDMA] = sizeof(struct rds_ext_header_rdma), | |
46 | [RDS_EXTHDR_RDMA_DEST] = sizeof(struct rds_ext_header_rdma_dest), | |
5916e2c1 | 47 | [RDS_EXTHDR_NPATHS] = sizeof(u16), |
905dd418 | 48 | [RDS_EXTHDR_GEN_NUM] = sizeof(u32), |
7875e18e AG |
49 | }; |
50 | ||
51 | ||
52 | void rds_message_addref(struct rds_message *rm) | |
53 | { | |
6c5a1c4a RE |
54 | rdsdebug("addref rm %p ref %d\n", rm, refcount_read(&rm->m_refcount)); |
55 | refcount_inc(&rm->m_refcount); | |
7875e18e | 56 | } |
616b757a | 57 | EXPORT_SYMBOL_GPL(rds_message_addref); |
7875e18e | 58 | |
01883eda SV |
59 | static inline bool skb_zcookie_add(struct sk_buff *skb, u32 cookie) |
60 | { | |
401910db SV |
61 | struct rds_zcopy_cookies *ck = (struct rds_zcopy_cookies *)skb->cb; |
62 | int ncookies = ck->num; | |
01883eda | 63 | |
401910db | 64 | if (ncookies == RDS_MAX_ZCOOKIES) |
01883eda | 65 | return false; |
401910db SV |
66 | ck->cookies[ncookies] = cookie; |
67 | ck->num = ++ncookies; | |
01883eda SV |
68 | return true; |
69 | } | |
70 | ||
71 | static void rds_rm_zerocopy_callback(struct rds_sock *rs, | |
72 | struct rds_znotifier *znotif) | |
73 | { | |
01883eda | 74 | struct sk_buff *skb, *tail; |
01883eda SV |
75 | unsigned long flags; |
76 | struct sk_buff_head *q; | |
77 | u32 cookie = znotif->z_cookie; | |
401910db | 78 | struct rds_zcopy_cookies *ck; |
01883eda | 79 | |
401910db | 80 | q = &rs->rs_zcookie_queue; |
01883eda SV |
81 | spin_lock_irqsave(&q->lock, flags); |
82 | tail = skb_peek_tail(q); | |
83 | ||
84 | if (tail && skb_zcookie_add(tail, cookie)) { | |
85 | spin_unlock_irqrestore(&q->lock, flags); | |
86 | mm_unaccount_pinned_pages(&znotif->z_mmp); | |
87 | consume_skb(rds_skb_from_znotifier(znotif)); | |
401910db | 88 | /* caller invokes rds_wake_sk_sleep() */ |
01883eda SV |
89 | return; |
90 | } | |
91 | ||
92 | skb = rds_skb_from_znotifier(znotif); | |
401910db SV |
93 | ck = (struct rds_zcopy_cookies *)skb->cb; |
94 | memset(ck, 0, sizeof(*ck)); | |
01883eda SV |
95 | WARN_ON(!skb_zcookie_add(skb, cookie)); |
96 | ||
97 | __skb_queue_tail(q, skb); | |
98 | ||
99 | spin_unlock_irqrestore(&q->lock, flags); | |
401910db | 100 | /* caller invokes rds_wake_sk_sleep() */ |
01883eda SV |
101 | |
102 | mm_unaccount_pinned_pages(&znotif->z_mmp); | |
103 | } | |
104 | ||
7875e18e AG |
105 | /* |
106 | * This relies on dma_map_sg() not touching sg[].page during merging. | |
107 | */ | |
108 | static void rds_message_purge(struct rds_message *rm) | |
109 | { | |
ea8994cb | 110 | unsigned long i, flags; |
01883eda | 111 | bool zcopy = false; |
7875e18e AG |
112 | |
113 | if (unlikely(test_bit(RDS_MSG_PAGEVEC, &rm->m_flags))) | |
114 | return; | |
115 | ||
ea8994cb SV |
116 | spin_lock_irqsave(&rm->m_rs_lock, flags); |
117 | if (rm->m_rs) { | |
01883eda SV |
118 | struct rds_sock *rs = rm->m_rs; |
119 | ||
120 | if (rm->data.op_mmp_znotifier) { | |
121 | zcopy = true; | |
122 | rds_rm_zerocopy_callback(rs, rm->data.op_mmp_znotifier); | |
401910db | 123 | rds_wake_sk_sleep(rs); |
01883eda SV |
124 | rm->data.op_mmp_znotifier = NULL; |
125 | } | |
126 | sock_put(rds_rs_to_sk(rs)); | |
ea8994cb SV |
127 | rm->m_rs = NULL; |
128 | } | |
129 | spin_unlock_irqrestore(&rm->m_rs_lock, flags); | |
7875e18e | 130 | |
01883eda SV |
131 | for (i = 0; i < rm->data.op_nents; i++) { |
132 | /* XXX will have to put_page for page refs */ | |
133 | if (!zcopy) | |
134 | __free_page(sg_page(&rm->data.op_sg[i])); | |
135 | else | |
136 | put_page(sg_page(&rm->data.op_sg[i])); | |
137 | } | |
138 | rm->data.op_nents = 0; | |
139 | ||
f8b3aaf2 AG |
140 | if (rm->rdma.op_active) |
141 | rds_rdma_free_op(&rm->rdma); | |
142 | if (rm->rdma.op_rdma_mr) | |
143 | rds_mr_put(rm->rdma.op_rdma_mr); | |
d0ab25a8 AG |
144 | |
145 | if (rm->atomic.op_active) | |
146 | rds_atomic_free_op(&rm->atomic); | |
147 | if (rm->atomic.op_rdma_mr) | |
148 | rds_mr_put(rm->atomic.op_rdma_mr); | |
7875e18e | 149 | } |
7875e18e AG |
150 | |
151 | void rds_message_put(struct rds_message *rm) | |
152 | { | |
6c5a1c4a RE |
153 | rdsdebug("put rm %p ref %d\n", rm, refcount_read(&rm->m_refcount)); |
154 | WARN(!refcount_read(&rm->m_refcount), "danger refcount zero on %p\n", rm); | |
155 | if (refcount_dec_and_test(&rm->m_refcount)) { | |
7875e18e AG |
156 | BUG_ON(!list_empty(&rm->m_sock_item)); |
157 | BUG_ON(!list_empty(&rm->m_conn_item)); | |
158 | rds_message_purge(rm); | |
159 | ||
160 | kfree(rm); | |
161 | } | |
162 | } | |
616b757a | 163 | EXPORT_SYMBOL_GPL(rds_message_put); |
7875e18e | 164 | |
7875e18e AG |
165 | void rds_message_populate_header(struct rds_header *hdr, __be16 sport, |
166 | __be16 dport, u64 seq) | |
167 | { | |
168 | hdr->h_flags = 0; | |
169 | hdr->h_sport = sport; | |
170 | hdr->h_dport = dport; | |
171 | hdr->h_sequence = cpu_to_be64(seq); | |
172 | hdr->h_exthdr[0] = RDS_EXTHDR_NONE; | |
173 | } | |
616b757a | 174 | EXPORT_SYMBOL_GPL(rds_message_populate_header); |
7875e18e | 175 | |
ff51bf84 | 176 | int rds_message_add_extension(struct rds_header *hdr, unsigned int type, |
177 | const void *data, unsigned int len) | |
7875e18e AG |
178 | { |
179 | unsigned int ext_len = sizeof(u8) + len; | |
180 | unsigned char *dst; | |
181 | ||
182 | /* For now, refuse to add more than one extension header */ | |
183 | if (hdr->h_exthdr[0] != RDS_EXTHDR_NONE) | |
184 | return 0; | |
185 | ||
f64f9e71 | 186 | if (type >= __RDS_EXTHDR_MAX || len != rds_exthdr_size[type]) |
7875e18e AG |
187 | return 0; |
188 | ||
189 | if (ext_len >= RDS_HEADER_EXT_SPACE) | |
190 | return 0; | |
191 | dst = hdr->h_exthdr; | |
192 | ||
193 | *dst++ = type; | |
194 | memcpy(dst, data, len); | |
195 | ||
196 | dst[len] = RDS_EXTHDR_NONE; | |
197 | return 1; | |
198 | } | |
616b757a | 199 | EXPORT_SYMBOL_GPL(rds_message_add_extension); |
7875e18e AG |
200 | |
201 | /* | |
202 | * If a message has extension headers, retrieve them here. | |
203 | * Call like this: | |
204 | * | |
205 | * unsigned int pos = 0; | |
206 | * | |
207 | * while (1) { | |
208 | * buflen = sizeof(buffer); | |
209 | * type = rds_message_next_extension(hdr, &pos, buffer, &buflen); | |
210 | * if (type == RDS_EXTHDR_NONE) | |
211 | * break; | |
212 | * ... | |
213 | * } | |
214 | */ | |
215 | int rds_message_next_extension(struct rds_header *hdr, | |
216 | unsigned int *pos, void *buf, unsigned int *buflen) | |
217 | { | |
218 | unsigned int offset, ext_type, ext_len; | |
219 | u8 *src = hdr->h_exthdr; | |
220 | ||
221 | offset = *pos; | |
222 | if (offset >= RDS_HEADER_EXT_SPACE) | |
223 | goto none; | |
224 | ||
225 | /* Get the extension type and length. For now, the | |
226 | * length is implied by the extension type. */ | |
227 | ext_type = src[offset++]; | |
228 | ||
229 | if (ext_type == RDS_EXTHDR_NONE || ext_type >= __RDS_EXTHDR_MAX) | |
230 | goto none; | |
231 | ext_len = rds_exthdr_size[ext_type]; | |
232 | if (offset + ext_len > RDS_HEADER_EXT_SPACE) | |
233 | goto none; | |
234 | ||
235 | *pos = offset + ext_len; | |
236 | if (ext_len < *buflen) | |
237 | *buflen = ext_len; | |
238 | memcpy(buf, src + offset, *buflen); | |
239 | return ext_type; | |
240 | ||
241 | none: | |
242 | *pos = RDS_HEADER_EXT_SPACE; | |
243 | *buflen = 0; | |
244 | return RDS_EXTHDR_NONE; | |
245 | } | |
246 | ||
7875e18e AG |
247 | int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset) |
248 | { | |
249 | struct rds_ext_header_rdma_dest ext_hdr; | |
250 | ||
251 | ext_hdr.h_rdma_rkey = cpu_to_be32(r_key); | |
252 | ext_hdr.h_rdma_offset = cpu_to_be32(offset); | |
253 | return rds_message_add_extension(hdr, RDS_EXTHDR_RDMA_DEST, &ext_hdr, sizeof(ext_hdr)); | |
254 | } | |
616b757a | 255 | EXPORT_SYMBOL_GPL(rds_message_add_rdma_dest_extension); |
7875e18e | 256 | |
fc445084 AG |
257 | /* |
258 | * Each rds_message is allocated with extra space for the scatterlist entries | |
259 | * rds ops will need. This is to minimize memory allocation count. Then, each rds op | |
260 | * can grab SGs when initializing its part of the rds_message. | |
261 | */ | |
262 | struct rds_message *rds_message_alloc(unsigned int extra_len, gfp_t gfp) | |
7875e18e AG |
263 | { |
264 | struct rds_message *rm; | |
265 | ||
ece6b0a2 CW |
266 | if (extra_len > KMALLOC_MAX_SIZE - sizeof(struct rds_message)) |
267 | return NULL; | |
268 | ||
fc445084 | 269 | rm = kzalloc(sizeof(struct rds_message) + extra_len, gfp); |
7875e18e AG |
270 | if (!rm) |
271 | goto out; | |
272 | ||
fc445084 AG |
273 | rm->m_used_sgs = 0; |
274 | rm->m_total_sgs = extra_len / sizeof(struct scatterlist); | |
275 | ||
6c5a1c4a | 276 | refcount_set(&rm->m_refcount, 1); |
7875e18e AG |
277 | INIT_LIST_HEAD(&rm->m_sock_item); |
278 | INIT_LIST_HEAD(&rm->m_conn_item); | |
279 | spin_lock_init(&rm->m_rs_lock); | |
c83188dc | 280 | init_waitqueue_head(&rm->m_flush_wait); |
7875e18e AG |
281 | |
282 | out: | |
283 | return rm; | |
284 | } | |
285 | ||
fc445084 AG |
286 | /* |
287 | * RDS ops use this to grab SG entries from the rm's sg pool. | |
288 | */ | |
289 | struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents) | |
290 | { | |
291 | struct scatterlist *sg_first = (struct scatterlist *) &rm[1]; | |
292 | struct scatterlist *sg_ret; | |
293 | ||
294 | WARN_ON(rm->m_used_sgs + nents > rm->m_total_sgs); | |
ee4c7b47 | 295 | WARN_ON(!nents); |
fc445084 | 296 | |
d139ff09 AG |
297 | if (rm->m_used_sgs + nents > rm->m_total_sgs) |
298 | return NULL; | |
299 | ||
fc445084 | 300 | sg_ret = &sg_first[rm->m_used_sgs]; |
f4dd96f7 | 301 | sg_init_table(sg_ret, nents); |
fc445084 AG |
302 | rm->m_used_sgs += nents; |
303 | ||
304 | return sg_ret; | |
305 | } | |
306 | ||
7875e18e AG |
307 | struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len) |
308 | { | |
309 | struct rds_message *rm; | |
310 | unsigned int i; | |
ff87e97a AG |
311 | int num_sgs = ceil(total_len, PAGE_SIZE); |
312 | int extra_bytes = num_sgs * sizeof(struct scatterlist); | |
7875e18e | 313 | |
f2ec76f2 | 314 | rm = rds_message_alloc(extra_bytes, GFP_NOWAIT); |
8690bfa1 | 315 | if (!rm) |
7875e18e AG |
316 | return ERR_PTR(-ENOMEM); |
317 | ||
318 | set_bit(RDS_MSG_PAGEVEC, &rm->m_flags); | |
319 | rm->m_inc.i_hdr.h_len = cpu_to_be32(total_len); | |
6c7cc6e4 AG |
320 | rm->data.op_nents = ceil(total_len, PAGE_SIZE); |
321 | rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs); | |
aa58163a PE |
322 | if (!rm->data.op_sg) { |
323 | rds_message_put(rm); | |
d139ff09 | 324 | return ERR_PTR(-ENOMEM); |
aa58163a | 325 | } |
7875e18e | 326 | |
6c7cc6e4 AG |
327 | for (i = 0; i < rm->data.op_nents; ++i) { |
328 | sg_set_page(&rm->data.op_sg[i], | |
7875e18e AG |
329 | virt_to_page(page_addrs[i]), |
330 | PAGE_SIZE, 0); | |
331 | } | |
332 | ||
333 | return rm; | |
334 | } | |
335 | ||
d40a126b | 336 | int rds_message_zcopy_from_user(struct rds_message *rm, struct iov_iter *from) |
7875e18e | 337 | { |
7875e18e | 338 | unsigned long sg_off; |
7875e18e | 339 | struct scatterlist *sg; |
fc445084 | 340 | int ret = 0; |
0cebacce | 341 | int length = iov_iter_count(from); |
d40a126b SV |
342 | int total_copied = 0; |
343 | struct sk_buff *skb; | |
7875e18e | 344 | |
083735f4 | 345 | rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from)); |
7875e18e AG |
346 | |
347 | /* | |
348 | * now allocate and copy in the data payload. | |
349 | */ | |
6c7cc6e4 | 350 | sg = rm->data.op_sg; |
7875e18e AG |
351 | sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */ |
352 | ||
d40a126b SV |
353 | skb = alloc_skb(0, GFP_KERNEL); |
354 | if (!skb) | |
355 | return -ENOMEM; | |
356 | BUILD_BUG_ON(sizeof(skb->cb) < max_t(int, sizeof(struct rds_znotifier), | |
357 | sizeof(struct rds_zcopy_cookies))); | |
358 | rm->data.op_mmp_znotifier = RDS_ZCOPY_SKB(skb); | |
359 | if (mm_account_pinned_pages(&rm->data.op_mmp_znotifier->z_mmp, | |
360 | length)) { | |
361 | ret = -ENOMEM; | |
362 | goto err; | |
363 | } | |
364 | while (iov_iter_count(from)) { | |
365 | struct page *pages; | |
366 | size_t start; | |
367 | ssize_t copied; | |
368 | ||
369 | copied = iov_iter_get_pages(from, &pages, PAGE_SIZE, | |
370 | 1, &start); | |
371 | if (copied < 0) { | |
372 | struct mmpin *mmp; | |
373 | int i; | |
374 | ||
375 | for (i = 0; i < rm->data.op_nents; i++) | |
376 | put_page(sg_page(&rm->data.op_sg[i])); | |
377 | mmp = &rm->data.op_mmp_znotifier->z_mmp; | |
378 | mm_unaccount_pinned_pages(mmp); | |
379 | ret = -EFAULT; | |
0cebacce SV |
380 | goto err; |
381 | } | |
d40a126b SV |
382 | total_copied += copied; |
383 | iov_iter_advance(from, copied); | |
384 | length -= copied; | |
385 | sg_set_page(sg, pages, copied, start); | |
386 | rm->data.op_nents++; | |
387 | sg++; | |
388 | } | |
389 | WARN_ON_ONCE(length != 0); | |
390 | return ret; | |
0cebacce | 391 | err: |
d40a126b SV |
392 | consume_skb(skb); |
393 | rm->data.op_mmp_znotifier = NULL; | |
394 | return ret; | |
395 | } | |
396 | ||
397 | int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from, | |
398 | bool zcopy) | |
399 | { | |
400 | unsigned long to_copy, nbytes; | |
401 | unsigned long sg_off; | |
402 | struct scatterlist *sg; | |
403 | int ret = 0; | |
404 | ||
405 | rm->m_inc.i_hdr.h_len = cpu_to_be32(iov_iter_count(from)); | |
406 | ||
407 | /* now allocate and copy in the data payload. */ | |
408 | sg = rm->data.op_sg; | |
409 | sg_off = 0; /* Dear gcc, sg->page will be null from kzalloc. */ | |
410 | ||
411 | if (zcopy) | |
412 | return rds_message_zcopy_from_user(rm, from); | |
0cebacce | 413 | |
083735f4 | 414 | while (iov_iter_count(from)) { |
8690bfa1 | 415 | if (!sg_page(sg)) { |
083735f4 | 416 | ret = rds_page_remainder_alloc(sg, iov_iter_count(from), |
7875e18e AG |
417 | GFP_HIGHUSER); |
418 | if (ret) | |
083735f4 | 419 | return ret; |
6c7cc6e4 | 420 | rm->data.op_nents++; |
7875e18e AG |
421 | sg_off = 0; |
422 | } | |
423 | ||
083735f4 AV |
424 | to_copy = min_t(unsigned long, iov_iter_count(from), |
425 | sg->length - sg_off); | |
7875e18e | 426 | |
083735f4 | 427 | rds_stats_add(s_copy_from_user, to_copy); |
d0a47d32 SV |
428 | nbytes = copy_page_from_iter(sg_page(sg), sg->offset + sg_off, |
429 | to_copy, from); | |
430 | if (nbytes != to_copy) | |
083735f4 | 431 | return -EFAULT; |
7875e18e | 432 | |
7875e18e AG |
433 | sg_off += to_copy; |
434 | ||
435 | if (sg_off == sg->length) | |
436 | sg++; | |
437 | } | |
438 | ||
fc445084 | 439 | return ret; |
7875e18e AG |
440 | } |
441 | ||
c310e72c | 442 | int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to) |
7875e18e AG |
443 | { |
444 | struct rds_message *rm; | |
7875e18e AG |
445 | struct scatterlist *sg; |
446 | unsigned long to_copy; | |
7875e18e AG |
447 | unsigned long vec_off; |
448 | int copied; | |
449 | int ret; | |
450 | u32 len; | |
451 | ||
452 | rm = container_of(inc, struct rds_message, m_inc); | |
453 | len = be32_to_cpu(rm->m_inc.i_hdr.h_len); | |
454 | ||
6c7cc6e4 | 455 | sg = rm->data.op_sg; |
7875e18e AG |
456 | vec_off = 0; |
457 | copied = 0; | |
458 | ||
c310e72c | 459 | while (iov_iter_count(to) && copied < len) { |
6ff4a8ad GU |
460 | to_copy = min_t(unsigned long, iov_iter_count(to), |
461 | sg->length - vec_off); | |
7875e18e AG |
462 | to_copy = min_t(unsigned long, to_copy, len - copied); |
463 | ||
c310e72c AV |
464 | rds_stats_add(s_copy_to_user, to_copy); |
465 | ret = copy_page_to_iter(sg_page(sg), sg->offset + vec_off, | |
466 | to_copy, to); | |
467 | if (ret != to_copy) | |
468 | return -EFAULT; | |
7875e18e | 469 | |
7875e18e AG |
470 | vec_off += to_copy; |
471 | copied += to_copy; | |
472 | ||
473 | if (vec_off == sg->length) { | |
474 | vec_off = 0; | |
475 | sg++; | |
476 | } | |
477 | } | |
478 | ||
479 | return copied; | |
480 | } | |
481 | ||
482 | /* | |
483 | * If the message is still on the send queue, wait until the transport | |
484 | * is done with it. This is particularly important for RDMA operations. | |
485 | */ | |
486 | void rds_message_wait(struct rds_message *rm) | |
487 | { | |
c83188dc | 488 | wait_event_interruptible(rm->m_flush_wait, |
7875e18e AG |
489 | !test_bit(RDS_MSG_MAPPED, &rm->m_flags)); |
490 | } | |
491 | ||
492 | void rds_message_unmapped(struct rds_message *rm) | |
493 | { | |
494 | clear_bit(RDS_MSG_MAPPED, &rm->m_flags); | |
c83188dc | 495 | wake_up_interruptible(&rm->m_flush_wait); |
7875e18e | 496 | } |
616b757a | 497 | EXPORT_SYMBOL_GPL(rds_message_unmapped); |
7875e18e | 498 |