]>
Commit | Line | Data |
---|---|---|
87dc6f5f JQ |
1 | /* |
2 | * Multifd zlib compression implementation | |
3 | * | |
4 | * Copyright (c) 2020 Red Hat Inc | |
5 | * | |
6 | * Authors: | |
7 | * Juan Quintela <quintela@redhat.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2 or later. | |
10 | * See the COPYING file in the top-level directory. | |
11 | */ | |
12 | ||
13 | #include "qemu/osdep.h" | |
14 | #include <zstd.h> | |
15 | #include "qemu/rcu.h" | |
f5ff5487 | 16 | #include "exec/ramblock.h" |
87dc6f5f JQ |
17 | #include "exec/target_page.h" |
18 | #include "qapi/error.h" | |
19 | #include "migration.h" | |
20 | #include "trace.h" | |
1dfc4b9e | 21 | #include "options.h" |
87dc6f5f JQ |
22 | #include "multifd.h" |
23 | ||
24 | struct zstd_data { | |
25 | /* stream for compression */ | |
26 | ZSTD_CStream *zcs; | |
27 | /* stream for decompression */ | |
28 | ZSTD_DStream *zds; | |
29 | /* buffers */ | |
30 | ZSTD_inBuffer in; | |
31 | ZSTD_outBuffer out; | |
32 | /* compressed buffer */ | |
33 | uint8_t *zbuff; | |
34 | /* size of compressed buffer */ | |
35 | uint32_t zbuff_len; | |
36 | }; | |
37 | ||
38 | /* Multifd zstd compression */ | |
39 | ||
40 | /** | |
41 | * zstd_send_setup: setup send side | |
42 | * | |
43 | * Setup each channel with zstd compression. | |
44 | * | |
45 | * Returns 0 for success or -1 for error | |
46 | * | |
47 | * @p: Params for the channel that we are using | |
48 | * @errp: pointer to an error | |
49 | */ | |
50 | static int zstd_send_setup(MultiFDSendParams *p, Error **errp) | |
51 | { | |
87dc6f5f JQ |
52 | struct zstd_data *z = g_new0(struct zstd_data, 1); |
53 | int res; | |
54 | ||
55 | p->data = z; | |
56 | z->zcs = ZSTD_createCStream(); | |
57 | if (!z->zcs) { | |
58 | g_free(z); | |
04e11404 | 59 | error_setg(errp, "multifd %u: zstd createCStream failed", p->id); |
87dc6f5f JQ |
60 | return -1; |
61 | } | |
62 | ||
63 | res = ZSTD_initCStream(z->zcs, migrate_multifd_zstd_level()); | |
64 | if (ZSTD_isError(res)) { | |
65 | ZSTD_freeCStream(z->zcs); | |
66 | g_free(z); | |
04e11404 | 67 | error_setg(errp, "multifd %u: initCStream failed with error %s", |
87dc6f5f JQ |
68 | p->id, ZSTD_getErrorName(res)); |
69 | return -1; | |
70 | } | |
d8b71d96 | 71 | /* This is the maximum size of the compressed buffer */ |
fc670522 | 72 | z->zbuff_len = ZSTD_compressBound(MULTIFD_PACKET_SIZE); |
87dc6f5f JQ |
73 | z->zbuff = g_try_malloc(z->zbuff_len); |
74 | if (!z->zbuff) { | |
75 | ZSTD_freeCStream(z->zcs); | |
76 | g_free(z); | |
04e11404 | 77 | error_setg(errp, "multifd %u: out of memory for zbuff", p->id); |
87dc6f5f JQ |
78 | return -1; |
79 | } | |
80 | return 0; | |
81 | } | |
82 | ||
83 | /** | |
84 | * zstd_send_cleanup: cleanup send side | |
85 | * | |
86 | * Close the channel and return memory. | |
87 | * | |
88 | * @p: Params for the channel that we are using | |
18ede636 | 89 | * @errp: pointer to an error |
87dc6f5f JQ |
90 | */ |
91 | static void zstd_send_cleanup(MultiFDSendParams *p, Error **errp) | |
92 | { | |
93 | struct zstd_data *z = p->data; | |
94 | ||
95 | ZSTD_freeCStream(z->zcs); | |
96 | z->zcs = NULL; | |
97 | g_free(z->zbuff); | |
98 | z->zbuff = NULL; | |
99 | g_free(p->data); | |
100 | p->data = NULL; | |
101 | } | |
102 | ||
103 | /** | |
104 | * zstd_send_prepare: prepare date to be able to send | |
105 | * | |
106 | * Create a compressed buffer with all the pages that we are going to | |
107 | * send. | |
108 | * | |
109 | * Returns 0 for success or -1 for error | |
110 | * | |
111 | * @p: Params for the channel that we are using | |
18ede636 | 112 | * @errp: pointer to an error |
87dc6f5f | 113 | */ |
02fb8104 | 114 | static int zstd_send_prepare(MultiFDSendParams *p, Error **errp) |
87dc6f5f | 115 | { |
87dc6f5f JQ |
116 | struct zstd_data *z = p->data; |
117 | int ret; | |
118 | uint32_t i; | |
119 | ||
120 | z->out.dst = z->zbuff; | |
121 | z->out.size = z->zbuff_len; | |
122 | z->out.pos = 0; | |
123 | ||
815956f0 | 124 | for (i = 0; i < p->normal_num; i++) { |
87dc6f5f JQ |
125 | ZSTD_EndDirective flush = ZSTD_e_continue; |
126 | ||
815956f0 | 127 | if (i == p->normal_num - 1) { |
87dc6f5f JQ |
128 | flush = ZSTD_e_flush; |
129 | } | |
815956f0 | 130 | z->in.src = p->pages->block->host + p->normal[i]; |
ddec20f8 | 131 | z->in.size = p->page_size; |
87dc6f5f JQ |
132 | z->in.pos = 0; |
133 | ||
134 | /* | |
135 | * Welcome to compressStream2 semantics | |
136 | * | |
137 | * We need to loop while: | |
138 | * - return is > 0 | |
139 | * - there is input available | |
140 | * - there is output space free | |
141 | */ | |
142 | do { | |
143 | ret = ZSTD_compressStream2(z->zcs, &z->out, &z->in, flush); | |
144 | } while (ret > 0 && (z->in.size - z->in.pos > 0) | |
145 | && (z->out.size - z->out.pos > 0)); | |
146 | if (ret > 0 && (z->in.size - z->in.pos > 0)) { | |
04e11404 | 147 | error_setg(errp, "multifd %u: compressStream buffer too small", |
87dc6f5f JQ |
148 | p->id); |
149 | return -1; | |
150 | } | |
151 | if (ZSTD_isError(ret)) { | |
04e11404 | 152 | error_setg(errp, "multifd %u: compressStream error %s", |
87dc6f5f JQ |
153 | p->id, ZSTD_getErrorName(ret)); |
154 | return -1; | |
155 | } | |
156 | } | |
0a818b89 JQ |
157 | p->iov[p->iovs_num].iov_base = z->zbuff; |
158 | p->iov[p->iovs_num].iov_len = z->out.pos; | |
159 | p->iovs_num++; | |
87dc6f5f JQ |
160 | p->next_packet_size = z->out.pos; |
161 | p->flags |= MULTIFD_FLAG_ZSTD; | |
162 | ||
163 | return 0; | |
164 | } | |
165 | ||
87dc6f5f JQ |
166 | /** |
167 | * zstd_recv_setup: setup receive side | |
168 | * | |
169 | * Create the compressed channel and buffer. | |
170 | * | |
171 | * Returns 0 for success or -1 for error | |
172 | * | |
173 | * @p: Params for the channel that we are using | |
174 | * @errp: pointer to an error | |
175 | */ | |
176 | static int zstd_recv_setup(MultiFDRecvParams *p, Error **errp) | |
177 | { | |
87dc6f5f JQ |
178 | struct zstd_data *z = g_new0(struct zstd_data, 1); |
179 | int ret; | |
180 | ||
181 | p->data = z; | |
182 | z->zds = ZSTD_createDStream(); | |
183 | if (!z->zds) { | |
184 | g_free(z); | |
04e11404 | 185 | error_setg(errp, "multifd %u: zstd createDStream failed", p->id); |
87dc6f5f JQ |
186 | return -1; |
187 | } | |
188 | ||
189 | ret = ZSTD_initDStream(z->zds); | |
190 | if (ZSTD_isError(ret)) { | |
191 | ZSTD_freeDStream(z->zds); | |
192 | g_free(z); | |
04e11404 | 193 | error_setg(errp, "multifd %u: initDStream failed with error %s", |
87dc6f5f JQ |
194 | p->id, ZSTD_getErrorName(ret)); |
195 | return -1; | |
196 | } | |
197 | ||
47a17824 JQ |
198 | /* To be safe, we reserve twice the size of the packet */ |
199 | z->zbuff_len = MULTIFD_PACKET_SIZE * 2; | |
87dc6f5f JQ |
200 | z->zbuff = g_try_malloc(z->zbuff_len); |
201 | if (!z->zbuff) { | |
202 | ZSTD_freeDStream(z->zds); | |
203 | g_free(z); | |
04e11404 | 204 | error_setg(errp, "multifd %u: out of memory for zbuff", p->id); |
87dc6f5f JQ |
205 | return -1; |
206 | } | |
207 | return 0; | |
208 | } | |
209 | ||
210 | /** | |
211 | * zstd_recv_cleanup: setup receive side | |
212 | * | |
213 | * For no compression this function does nothing. | |
214 | * | |
215 | * @p: Params for the channel that we are using | |
216 | */ | |
217 | static void zstd_recv_cleanup(MultiFDRecvParams *p) | |
218 | { | |
219 | struct zstd_data *z = p->data; | |
220 | ||
221 | ZSTD_freeDStream(z->zds); | |
222 | z->zds = NULL; | |
223 | g_free(z->zbuff); | |
224 | z->zbuff = NULL; | |
225 | g_free(p->data); | |
226 | p->data = NULL; | |
227 | } | |
228 | ||
229 | /** | |
230 | * zstd_recv_pages: read the data from the channel into actual pages | |
231 | * | |
232 | * Read the compressed buffer, and uncompress it into the actual | |
233 | * pages. | |
234 | * | |
235 | * Returns 0 for success or -1 for error | |
236 | * | |
237 | * @p: Params for the channel that we are using | |
87dc6f5f JQ |
238 | * @errp: pointer to an error |
239 | */ | |
40a4bfe9 | 240 | static int zstd_recv_pages(MultiFDRecvParams *p, Error **errp) |
87dc6f5f JQ |
241 | { |
242 | uint32_t in_size = p->next_packet_size; | |
243 | uint32_t out_size = 0; | |
ddec20f8 | 244 | uint32_t expected_size = p->normal_num * p->page_size; |
87dc6f5f JQ |
245 | uint32_t flags = p->flags & MULTIFD_FLAG_COMPRESSION_MASK; |
246 | struct zstd_data *z = p->data; | |
247 | int ret; | |
248 | int i; | |
249 | ||
250 | if (flags != MULTIFD_FLAG_ZSTD) { | |
04e11404 | 251 | error_setg(errp, "multifd %u: flags received %x flags expected %x", |
87dc6f5f JQ |
252 | p->id, flags, MULTIFD_FLAG_ZSTD); |
253 | return -1; | |
254 | } | |
255 | ret = qio_channel_read_all(p->c, (void *)z->zbuff, in_size, errp); | |
256 | ||
257 | if (ret != 0) { | |
258 | return ret; | |
259 | } | |
260 | ||
261 | z->in.src = z->zbuff; | |
262 | z->in.size = in_size; | |
263 | z->in.pos = 0; | |
264 | ||
cf2d4aa8 | 265 | for (i = 0; i < p->normal_num; i++) { |
faf60935 | 266 | z->out.dst = p->host + p->normal[i]; |
ddec20f8 | 267 | z->out.size = p->page_size; |
87dc6f5f JQ |
268 | z->out.pos = 0; |
269 | ||
270 | /* | |
271 | * Welcome to decompressStream semantics | |
272 | * | |
273 | * We need to loop while: | |
274 | * - return is > 0 | |
275 | * - there is input available | |
276 | * - we haven't put out a full page | |
277 | */ | |
278 | do { | |
279 | ret = ZSTD_decompressStream(z->zds, &z->out, &z->in); | |
280 | } while (ret > 0 && (z->in.size - z->in.pos > 0) | |
ddec20f8 JQ |
281 | && (z->out.pos < p->page_size)); |
282 | if (ret > 0 && (z->out.pos < p->page_size)) { | |
04e11404 | 283 | error_setg(errp, "multifd %u: decompressStream buffer too small", |
87dc6f5f JQ |
284 | p->id); |
285 | return -1; | |
286 | } | |
287 | if (ZSTD_isError(ret)) { | |
04e11404 | 288 | error_setg(errp, "multifd %u: decompressStream returned %s", |
87dc6f5f JQ |
289 | p->id, ZSTD_getErrorName(ret)); |
290 | return ret; | |
291 | } | |
292 | out_size += z->out.pos; | |
293 | } | |
294 | if (out_size != expected_size) { | |
04e11404 | 295 | error_setg(errp, "multifd %u: packet size received %u size expected %u", |
87dc6f5f JQ |
296 | p->id, out_size, expected_size); |
297 | return -1; | |
298 | } | |
299 | return 0; | |
300 | } | |
301 | ||
302 | static MultiFDMethods multifd_zstd_ops = { | |
303 | .send_setup = zstd_send_setup, | |
304 | .send_cleanup = zstd_send_cleanup, | |
305 | .send_prepare = zstd_send_prepare, | |
87dc6f5f JQ |
306 | .recv_setup = zstd_recv_setup, |
307 | .recv_cleanup = zstd_recv_cleanup, | |
308 | .recv_pages = zstd_recv_pages | |
309 | }; | |
310 | ||
311 | static void multifd_zstd_register(void) | |
312 | { | |
313 | multifd_register_ops(MULTIFD_COMPRESSION_ZSTD, &multifd_zstd_ops); | |
314 | } | |
315 | ||
316 | migration_init(multifd_zstd_register); |