4 * Copyright (c) Intel Corporation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include "spdk/stdinc.h"
36 #include "spdk_internal/copy_engine.h"
39 #include "spdk/event.h"
41 #include "spdk/thread.h"
43 static size_t g_max_copy_module_size
= 0;
45 static struct spdk_copy_engine
*hw_copy_engine
= NULL
;
46 /* Memcpy engine always exist */
47 static struct spdk_copy_engine
*mem_copy_engine
= NULL
;
49 TAILQ_HEAD(, spdk_copy_module_if
) spdk_copy_module_list
=
50 TAILQ_HEAD_INITIALIZER(spdk_copy_module_list
);
52 struct copy_io_channel
{
53 struct spdk_copy_engine
*engine
;
54 struct spdk_io_channel
*ch
;
57 struct spdk_copy_module_if
*g_copy_engine_module
= NULL
;
58 spdk_copy_fini_cb g_fini_cb_fn
= NULL
;
59 void *g_fini_cb_arg
= NULL
;
62 spdk_copy_engine_register(struct spdk_copy_engine
*copy_engine
)
64 assert(hw_copy_engine
== NULL
);
65 hw_copy_engine
= copy_engine
;
69 spdk_memcpy_register(struct spdk_copy_engine
*copy_engine
)
71 assert(mem_copy_engine
== NULL
);
72 mem_copy_engine
= copy_engine
;
76 spdk_memcpy_unregister(void)
78 mem_copy_engine
= NULL
;
82 copy_engine_done(void *ref
, int status
)
84 struct spdk_copy_task
*req
= (struct spdk_copy_task
*)ref
;
90 spdk_copy_submit(struct spdk_copy_task
*copy_req
, struct spdk_io_channel
*ch
,
91 void *dst
, void *src
, uint64_t nbytes
, spdk_copy_completion_cb cb
)
93 struct spdk_copy_task
*req
= copy_req
;
94 struct copy_io_channel
*copy_ch
= spdk_io_channel_get_ctx(ch
);
97 return copy_ch
->engine
->copy(req
->offload_ctx
, copy_ch
->ch
, dst
, src
, nbytes
,
102 spdk_copy_submit_fill(struct spdk_copy_task
*copy_req
, struct spdk_io_channel
*ch
,
103 void *dst
, uint8_t fill
, uint64_t nbytes
, spdk_copy_completion_cb cb
)
105 struct spdk_copy_task
*req
= copy_req
;
106 struct copy_io_channel
*copy_ch
= spdk_io_channel_get_ctx(ch
);
109 return copy_ch
->engine
->fill(req
->offload_ctx
, copy_ch
->ch
, dst
, fill
, nbytes
,
113 /* memcpy default copy engine */
115 mem_copy_submit(void *cb_arg
, struct spdk_io_channel
*ch
, void *dst
, void *src
, uint64_t nbytes
,
116 spdk_copy_completion_cb cb
)
118 struct spdk_copy_task
*copy_req
;
120 memcpy(dst
, src
, (size_t)nbytes
);
122 copy_req
= (struct spdk_copy_task
*)((uintptr_t)cb_arg
-
123 offsetof(struct spdk_copy_task
, offload_ctx
));
129 mem_copy_fill(void *cb_arg
, struct spdk_io_channel
*ch
, void *dst
, uint8_t fill
, uint64_t nbytes
,
130 spdk_copy_completion_cb cb
)
132 struct spdk_copy_task
*copy_req
;
134 memset(dst
, fill
, nbytes
);
135 copy_req
= (struct spdk_copy_task
*)((uintptr_t)cb_arg
-
136 offsetof(struct spdk_copy_task
, offload_ctx
));
142 static struct spdk_io_channel
*mem_get_io_channel(void);
144 static struct spdk_copy_engine memcpy_copy_engine
= {
145 .copy
= mem_copy_submit
,
146 .fill
= mem_copy_fill
,
147 .get_io_channel
= mem_get_io_channel
,
151 memcpy_create_cb(void *io_device
, void *ctx_buf
)
157 memcpy_destroy_cb(void *io_device
, void *ctx_buf
)
161 static struct spdk_io_channel
*mem_get_io_channel(void)
163 return spdk_get_io_channel(&memcpy_copy_engine
);
167 copy_engine_mem_get_ctx_size(void)
169 return sizeof(struct spdk_copy_task
);
173 spdk_copy_task_size(void)
175 return g_max_copy_module_size
;
178 void spdk_copy_module_list_add(struct spdk_copy_module_if
*copy_module
)
180 TAILQ_INSERT_TAIL(&spdk_copy_module_list
, copy_module
, tailq
);
181 if (copy_module
->get_ctx_size
&& copy_module
->get_ctx_size() > g_max_copy_module_size
) {
182 g_max_copy_module_size
= copy_module
->get_ctx_size();
187 copy_create_cb(void *io_device
, void *ctx_buf
)
189 struct copy_io_channel
*copy_ch
= ctx_buf
;
191 if (hw_copy_engine
!= NULL
) {
192 copy_ch
->ch
= hw_copy_engine
->get_io_channel();
193 if (copy_ch
->ch
!= NULL
) {
194 copy_ch
->engine
= hw_copy_engine
;
199 copy_ch
->ch
= mem_copy_engine
->get_io_channel();
200 assert(copy_ch
->ch
!= NULL
);
201 copy_ch
->engine
= mem_copy_engine
;
206 copy_destroy_cb(void *io_device
, void *ctx_buf
)
208 struct copy_io_channel
*copy_ch
= ctx_buf
;
210 spdk_put_io_channel(copy_ch
->ch
);
213 struct spdk_io_channel
*
214 spdk_copy_engine_get_io_channel(void)
216 return spdk_get_io_channel(&spdk_copy_module_list
);
220 copy_engine_mem_init(void)
222 spdk_memcpy_register(&memcpy_copy_engine
);
223 spdk_io_device_register(&memcpy_copy_engine
, memcpy_create_cb
, memcpy_destroy_cb
, 0,
230 copy_engine_mem_fini(void *ctxt
)
232 spdk_io_device_unregister(&memcpy_copy_engine
, NULL
);
233 spdk_memcpy_unregister();
235 spdk_copy_engine_module_finish();
239 spdk_copy_engine_module_initialize(void)
241 struct spdk_copy_module_if
*copy_engine_module
;
243 TAILQ_FOREACH(copy_engine_module
, &spdk_copy_module_list
, tailq
) {
244 copy_engine_module
->module_init();
249 spdk_copy_engine_initialize(void)
251 spdk_copy_engine_module_initialize();
253 * We need a unique identifier for the copy engine framework, so use the
254 * spdk_copy_module_list address for this purpose.
256 spdk_io_device_register(&spdk_copy_module_list
, copy_create_cb
, copy_destroy_cb
,
257 sizeof(struct copy_io_channel
), "copy_module");
263 spdk_copy_engine_module_finish_cb(void)
265 spdk_copy_fini_cb cb_fn
= g_fini_cb_fn
;
267 cb_fn(g_fini_cb_arg
);
269 g_fini_cb_arg
= NULL
;
273 spdk_copy_engine_module_finish(void)
275 if (!g_copy_engine_module
) {
276 g_copy_engine_module
= TAILQ_FIRST(&spdk_copy_module_list
);
278 g_copy_engine_module
= TAILQ_NEXT(g_copy_engine_module
, tailq
);
281 if (!g_copy_engine_module
) {
282 spdk_copy_engine_module_finish_cb();
286 if (g_copy_engine_module
->module_fini
) {
287 spdk_thread_send_msg(spdk_get_thread(), g_copy_engine_module
->module_fini
, NULL
);
289 spdk_copy_engine_module_finish();
294 spdk_copy_engine_finish(spdk_copy_fini_cb cb_fn
, void *cb_arg
)
296 assert(cb_fn
!= NULL
);
298 g_fini_cb_fn
= cb_fn
;
299 g_fini_cb_arg
= cb_arg
;
301 spdk_io_device_unregister(&spdk_copy_module_list
, NULL
);
302 spdk_copy_engine_module_finish();
306 spdk_copy_engine_config_text(FILE *fp
)
308 struct spdk_copy_module_if
*copy_engine_module
;
310 TAILQ_FOREACH(copy_engine_module
, &spdk_copy_module_list
, tailq
) {
311 if (copy_engine_module
->config_text
) {
312 copy_engine_module
->config_text(fp
);
317 SPDK_COPY_MODULE_REGISTER(copy_engine_mem_init
, copy_engine_mem_fini
,
318 NULL
, copy_engine_mem_get_ctx_size
)