]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright (c) Intel Corporation. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
11fdf7f2 | 34 | #include "spdk/stdinc.h" |
7c673cae | 35 | |
11fdf7f2 | 36 | #include "spdk_internal/copy_engine.h" |
7c673cae | 37 | |
11fdf7f2 TL |
38 | #include "spdk/env.h" |
39 | #include "spdk/event.h" | |
7c673cae | 40 | #include "spdk/log.h" |
11fdf7f2 | 41 | #include "spdk/thread.h" |
7c673cae FG |
42 | |
43 | static size_t g_max_copy_module_size = 0; | |
44 | ||
45 | static struct spdk_copy_engine *hw_copy_engine = NULL; | |
46 | /* Memcpy engine always exist */ | |
47 | static struct spdk_copy_engine *mem_copy_engine = NULL; | |
48 | ||
49 | TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list = | |
50 | TAILQ_HEAD_INITIALIZER(spdk_copy_module_list); | |
51 | ||
52 | struct copy_io_channel { | |
53 | struct spdk_copy_engine *engine; | |
54 | struct spdk_io_channel *ch; | |
55 | }; | |
56 | ||
11fdf7f2 TL |
57 | struct spdk_copy_module_if *g_copy_engine_module = NULL; |
58 | spdk_copy_fini_cb g_fini_cb_fn = NULL; | |
59 | void *g_fini_cb_arg = NULL; | |
60 | ||
7c673cae FG |
61 | void |
62 | spdk_copy_engine_register(struct spdk_copy_engine *copy_engine) | |
63 | { | |
64 | assert(hw_copy_engine == NULL); | |
65 | hw_copy_engine = copy_engine; | |
66 | } | |
67 | ||
68 | static void | |
69 | spdk_memcpy_register(struct spdk_copy_engine *copy_engine) | |
70 | { | |
71 | assert(mem_copy_engine == NULL); | |
72 | mem_copy_engine = copy_engine; | |
73 | } | |
74 | ||
11fdf7f2 TL |
75 | static void |
76 | spdk_memcpy_unregister(void) | |
77 | { | |
78 | mem_copy_engine = NULL; | |
79 | } | |
80 | ||
7c673cae FG |
81 | static void |
82 | copy_engine_done(void *ref, int status) | |
83 | { | |
84 | struct spdk_copy_task *req = (struct spdk_copy_task *)ref; | |
85 | ||
86 | req->cb(req, status); | |
87 | } | |
88 | ||
11fdf7f2 | 89 | int |
7c673cae FG |
90 | spdk_copy_submit(struct spdk_copy_task *copy_req, struct spdk_io_channel *ch, |
91 | void *dst, void *src, uint64_t nbytes, spdk_copy_completion_cb cb) | |
92 | { | |
93 | struct spdk_copy_task *req = copy_req; | |
94 | struct copy_io_channel *copy_ch = spdk_io_channel_get_ctx(ch); | |
95 | ||
96 | req->cb = cb; | |
97 | return copy_ch->engine->copy(req->offload_ctx, copy_ch->ch, dst, src, nbytes, | |
98 | copy_engine_done); | |
99 | } | |
100 | ||
11fdf7f2 | 101 | int |
7c673cae FG |
102 | spdk_copy_submit_fill(struct spdk_copy_task *copy_req, struct spdk_io_channel *ch, |
103 | void *dst, uint8_t fill, uint64_t nbytes, spdk_copy_completion_cb cb) | |
104 | { | |
105 | struct spdk_copy_task *req = copy_req; | |
106 | struct copy_io_channel *copy_ch = spdk_io_channel_get_ctx(ch); | |
107 | ||
108 | req->cb = cb; | |
109 | return copy_ch->engine->fill(req->offload_ctx, copy_ch->ch, dst, fill, nbytes, | |
110 | copy_engine_done); | |
111 | } | |
112 | ||
113 | /* memcpy default copy engine */ | |
11fdf7f2 | 114 | static int |
7c673cae FG |
115 | mem_copy_submit(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes, |
116 | spdk_copy_completion_cb cb) | |
117 | { | |
118 | struct spdk_copy_task *copy_req; | |
119 | ||
11fdf7f2 TL |
120 | memcpy(dst, src, (size_t)nbytes); |
121 | ||
7c673cae FG |
122 | copy_req = (struct spdk_copy_task *)((uintptr_t)cb_arg - |
123 | offsetof(struct spdk_copy_task, offload_ctx)); | |
124 | cb(copy_req, 0); | |
11fdf7f2 | 125 | return 0; |
7c673cae FG |
126 | } |
127 | ||
11fdf7f2 | 128 | static int |
7c673cae FG |
129 | mem_copy_fill(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes, |
130 | spdk_copy_completion_cb cb) | |
131 | { | |
132 | struct spdk_copy_task *copy_req; | |
133 | ||
134 | memset(dst, fill, nbytes); | |
135 | copy_req = (struct spdk_copy_task *)((uintptr_t)cb_arg - | |
136 | offsetof(struct spdk_copy_task, offload_ctx)); | |
137 | cb(copy_req, 0); | |
138 | ||
11fdf7f2 | 139 | return 0; |
7c673cae FG |
140 | } |
141 | ||
11fdf7f2 | 142 | static struct spdk_io_channel *mem_get_io_channel(void); |
7c673cae FG |
143 | |
144 | static struct spdk_copy_engine memcpy_copy_engine = { | |
145 | .copy = mem_copy_submit, | |
146 | .fill = mem_copy_fill, | |
147 | .get_io_channel = mem_get_io_channel, | |
148 | }; | |
149 | ||
150 | static int | |
11fdf7f2 | 151 | memcpy_create_cb(void *io_device, void *ctx_buf) |
7c673cae FG |
152 | { |
153 | return 0; | |
154 | } | |
155 | ||
156 | static void | |
157 | memcpy_destroy_cb(void *io_device, void *ctx_buf) | |
158 | { | |
159 | } | |
160 | ||
11fdf7f2 | 161 | static struct spdk_io_channel *mem_get_io_channel(void) |
7c673cae | 162 | { |
11fdf7f2 | 163 | return spdk_get_io_channel(&memcpy_copy_engine); |
7c673cae FG |
164 | } |
165 | ||
166 | static size_t | |
167 | copy_engine_mem_get_ctx_size(void) | |
168 | { | |
169 | return sizeof(struct spdk_copy_task); | |
170 | } | |
171 | ||
172 | size_t | |
173 | spdk_copy_task_size(void) | |
174 | { | |
175 | return g_max_copy_module_size; | |
176 | } | |
177 | ||
178 | void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module) | |
179 | { | |
180 | TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq); | |
181 | if (copy_module->get_ctx_size && copy_module->get_ctx_size() > g_max_copy_module_size) { | |
182 | g_max_copy_module_size = copy_module->get_ctx_size(); | |
183 | } | |
184 | } | |
185 | ||
186 | static int | |
11fdf7f2 | 187 | copy_create_cb(void *io_device, void *ctx_buf) |
7c673cae FG |
188 | { |
189 | struct copy_io_channel *copy_ch = ctx_buf; | |
190 | ||
191 | if (hw_copy_engine != NULL) { | |
11fdf7f2 | 192 | copy_ch->ch = hw_copy_engine->get_io_channel(); |
7c673cae FG |
193 | if (copy_ch->ch != NULL) { |
194 | copy_ch->engine = hw_copy_engine; | |
195 | return 0; | |
196 | } | |
197 | } | |
198 | ||
11fdf7f2 | 199 | copy_ch->ch = mem_copy_engine->get_io_channel(); |
7c673cae FG |
200 | assert(copy_ch->ch != NULL); |
201 | copy_ch->engine = mem_copy_engine; | |
202 | return 0; | |
203 | } | |
204 | ||
205 | static void | |
206 | copy_destroy_cb(void *io_device, void *ctx_buf) | |
207 | { | |
208 | struct copy_io_channel *copy_ch = ctx_buf; | |
209 | ||
210 | spdk_put_io_channel(copy_ch->ch); | |
211 | } | |
212 | ||
213 | struct spdk_io_channel * | |
11fdf7f2 | 214 | spdk_copy_engine_get_io_channel(void) |
7c673cae | 215 | { |
11fdf7f2 | 216 | return spdk_get_io_channel(&spdk_copy_module_list); |
7c673cae FG |
217 | } |
218 | ||
219 | static int | |
220 | copy_engine_mem_init(void) | |
221 | { | |
222 | spdk_memcpy_register(&memcpy_copy_engine); | |
11fdf7f2 TL |
223 | spdk_io_device_register(&memcpy_copy_engine, memcpy_create_cb, memcpy_destroy_cb, 0, |
224 | "memcpy_engine"); | |
7c673cae FG |
225 | |
226 | return 0; | |
227 | } | |
228 | ||
229 | static void | |
11fdf7f2 | 230 | copy_engine_mem_fini(void *ctxt) |
7c673cae | 231 | { |
11fdf7f2 TL |
232 | spdk_io_device_unregister(&memcpy_copy_engine, NULL); |
233 | spdk_memcpy_unregister(); | |
7c673cae | 234 | |
11fdf7f2 | 235 | spdk_copy_engine_module_finish(); |
7c673cae FG |
236 | } |
237 | ||
238 | static void | |
11fdf7f2 | 239 | spdk_copy_engine_module_initialize(void) |
7c673cae FG |
240 | { |
241 | struct spdk_copy_module_if *copy_engine_module; | |
242 | ||
243 | TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { | |
11fdf7f2 | 244 | copy_engine_module->module_init(); |
7c673cae FG |
245 | } |
246 | } | |
247 | ||
11fdf7f2 | 248 | int |
7c673cae FG |
249 | spdk_copy_engine_initialize(void) |
250 | { | |
251 | spdk_copy_engine_module_initialize(); | |
252 | /* | |
253 | * We need a unique identifier for the copy engine framework, so use the | |
254 | * spdk_copy_module_list address for this purpose. | |
255 | */ | |
256 | spdk_io_device_register(&spdk_copy_module_list, copy_create_cb, copy_destroy_cb, | |
11fdf7f2 TL |
257 | sizeof(struct copy_io_channel), "copy_module"); |
258 | ||
7c673cae FG |
259 | return 0; |
260 | } | |
261 | ||
11fdf7f2 TL |
262 | static void |
263 | spdk_copy_engine_module_finish_cb(void) | |
7c673cae | 264 | { |
11fdf7f2 TL |
265 | spdk_copy_fini_cb cb_fn = g_fini_cb_fn; |
266 | ||
267 | cb_fn(g_fini_cb_arg); | |
268 | g_fini_cb_fn = NULL; | |
269 | g_fini_cb_arg = NULL; | |
270 | } | |
271 | ||
272 | void | |
273 | spdk_copy_engine_module_finish(void) | |
274 | { | |
275 | if (!g_copy_engine_module) { | |
276 | g_copy_engine_module = TAILQ_FIRST(&spdk_copy_module_list); | |
277 | } else { | |
278 | g_copy_engine_module = TAILQ_NEXT(g_copy_engine_module, tailq); | |
279 | } | |
280 | ||
281 | if (!g_copy_engine_module) { | |
282 | spdk_copy_engine_module_finish_cb(); | |
283 | return; | |
284 | } | |
285 | ||
286 | if (g_copy_engine_module->module_fini) { | |
287 | spdk_thread_send_msg(spdk_get_thread(), g_copy_engine_module->module_fini, NULL); | |
288 | } else { | |
289 | spdk_copy_engine_module_finish(); | |
290 | } | |
291 | } | |
292 | ||
293 | void | |
294 | spdk_copy_engine_finish(spdk_copy_fini_cb cb_fn, void *cb_arg) | |
295 | { | |
296 | assert(cb_fn != NULL); | |
297 | ||
298 | g_fini_cb_fn = cb_fn; | |
299 | g_fini_cb_arg = cb_arg; | |
300 | ||
301 | spdk_io_device_unregister(&spdk_copy_module_list, NULL); | |
7c673cae | 302 | spdk_copy_engine_module_finish(); |
7c673cae FG |
303 | } |
304 | ||
11fdf7f2 TL |
305 | void |
306 | spdk_copy_engine_config_text(FILE *fp) | |
307 | { | |
308 | struct spdk_copy_module_if *copy_engine_module; | |
309 | ||
310 | TAILQ_FOREACH(copy_engine_module, &spdk_copy_module_list, tailq) { | |
311 | if (copy_engine_module->config_text) { | |
312 | copy_engine_module->config_text(fp); | |
313 | } | |
314 | } | |
315 | } | |
316 | ||
317 | SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init, copy_engine_mem_fini, | |
318 | NULL, copy_engine_mem_get_ctx_size) |