2 * Copyright (c) 2017-present, Facebook, Inc.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
10 #include <stdlib.h> /* malloc, free */
12 #define XXH_STATIC_LINKING_ONLY
13 #define XXH_NAMESPACE ZSTD_
16 #define ZSTD_STATIC_LINKING_ONLY
18 #include "zstd_errors.h"
20 #include "zstd_seekable.h"
22 #define CHECK_Z(f) { size_t const ret = (f); if (ret != 0) return ret; }
25 #define ERROR(name) ((size_t)-ZSTD_error_##name)
29 #define MIN(a, b) ((a) < (b) ? (a) : (b))
30 #define MAX(a, b) ((a) > (b) ? (a) : (b))
38 struct ZSTD_frameLog_s
{
39 framelogEntry_t
* entries
;
45 /* for use when streaming out the seek table */
50 struct ZSTD_seekable_CStream_s
{
51 ZSTD_CStream
* cstream
;
52 ZSTD_frameLog framelog
;
57 XXH64_state_t xxhState
;
64 size_t ZSTD_seekable_frameLog_allocVec(ZSTD_frameLog
* fl
)
66 /* allocate some initial space */
67 size_t const FRAMELOG_STARTING_CAPACITY
= 16;
68 fl
->entries
= (framelogEntry_t
*)malloc(
69 sizeof(framelogEntry_t
) * FRAMELOG_STARTING_CAPACITY
);
70 if (fl
->entries
== NULL
) return ERROR(memory_allocation
);
71 fl
->capacity
= FRAMELOG_STARTING_CAPACITY
;
76 size_t ZSTD_seekable_frameLog_freeVec(ZSTD_frameLog
* fl
)
78 if (fl
!= NULL
) free(fl
->entries
);
82 ZSTD_frameLog
* ZSTD_seekable_createFrameLog(int checksumFlag
)
84 ZSTD_frameLog
* fl
= malloc(sizeof(ZSTD_frameLog
));
85 if (fl
== NULL
) return NULL
;
87 if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(fl
))) {
92 fl
->checksumFlag
= checksumFlag
;
94 fl
->seekTableIndex
= 0;
100 size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog
* fl
)
102 ZSTD_seekable_frameLog_freeVec(fl
);
107 ZSTD_seekable_CStream
* ZSTD_seekable_createCStream()
109 ZSTD_seekable_CStream
* zcs
= malloc(sizeof(ZSTD_seekable_CStream
));
111 if (zcs
== NULL
) return NULL
;
113 memset(zcs
, 0, sizeof(*zcs
));
115 zcs
->cstream
= ZSTD_createCStream();
116 if (zcs
->cstream
== NULL
) goto failed1
;
118 if (ZSTD_isError(ZSTD_seekable_frameLog_allocVec(&zcs
->framelog
))) goto failed2
;
123 ZSTD_freeCStream(zcs
->cstream
);
129 size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream
* zcs
)
131 if (zcs
== NULL
) return 0; /* support free on null */
132 ZSTD_freeCStream(zcs
->cstream
);
133 ZSTD_seekable_frameLog_freeVec(&zcs
->framelog
);
139 size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream
* zcs
,
140 int compressionLevel
,
144 zcs
->framelog
.size
= 0;
148 /* make sure maxFrameSize has a reasonable value */
149 if (maxFrameSize
> ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE
) {
150 return ERROR(compressionParameter_unsupported
);
153 zcs
->maxFrameSize
= maxFrameSize
155 : ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE
;
157 zcs
->framelog
.checksumFlag
= checksumFlag
;
158 if (zcs
->framelog
.checksumFlag
) {
159 XXH64_reset(&zcs
->xxhState
, 0);
162 zcs
->framelog
.seekTablePos
= 0;
163 zcs
->framelog
.seekTableIndex
= 0;
164 zcs
->writingSeekTable
= 0;
166 return ZSTD_initCStream(zcs
->cstream
, compressionLevel
);
169 size_t ZSTD_seekable_logFrame(ZSTD_frameLog
* fl
,
170 unsigned compressedSize
,
171 unsigned decompressedSize
,
174 if (fl
->size
== ZSTD_SEEKABLE_MAXFRAMES
)
175 return ERROR(frameIndex_tooLarge
);
177 /* grow the buffer if required */
178 if (fl
->size
== fl
->capacity
) {
179 /* exponential size increase for constant amortized runtime */
180 size_t const newCapacity
= fl
->capacity
* 2;
181 framelogEntry_t
* const newEntries
= realloc(fl
->entries
,
182 sizeof(framelogEntry_t
) * newCapacity
);
184 if (newEntries
== NULL
) return ERROR(memory_allocation
);
186 fl
->entries
= newEntries
;
187 fl
->capacity
= newCapacity
;
190 fl
->entries
[fl
->size
] = (framelogEntry_t
){
191 compressedSize
, decompressedSize
, checksum
198 size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream
* zcs
, ZSTD_outBuffer
* output
)
200 size_t const prevOutPos
= output
->pos
;
202 size_t ret
= ZSTD_endStream(zcs
->cstream
, output
);
204 zcs
->frameCSize
+= output
->pos
- prevOutPos
;
206 /* need to flush before doing the rest */
211 /* store the frame data for later */
212 ret
= ZSTD_seekable_logFrame(
213 &zcs
->framelog
, zcs
->frameCSize
, zcs
->frameDSize
,
214 zcs
->framelog
.checksumFlag
215 ? XXH64_digest(&zcs
->xxhState
) & 0xFFFFFFFFU
219 /* reset for the next frame */
223 ZSTD_resetCStream(zcs
->cstream
, 0);
224 if (zcs
->framelog
.checksumFlag
)
225 XXH64_reset(&zcs
->xxhState
, 0);
230 size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream
* zcs
, ZSTD_outBuffer
* output
, ZSTD_inBuffer
* input
)
232 const BYTE
* const inBase
= (const BYTE
*) input
->src
+ input
->pos
;
233 size_t inLen
= input
->size
- input
->pos
;
235 inLen
= MIN(inLen
, (size_t)(zcs
->maxFrameSize
- zcs
->frameDSize
));
237 /* if we haven't finished flushing the last frame, don't start writing a new one */
239 ZSTD_inBuffer inTmp
= { inBase
, inLen
, 0 };
240 size_t const prevOutPos
= output
->pos
;
242 size_t const ret
= ZSTD_compressStream(zcs
->cstream
, output
, &inTmp
);
244 if (zcs
->framelog
.checksumFlag
) {
245 XXH64_update(&zcs
->xxhState
, inBase
, inTmp
.pos
);
248 zcs
->frameCSize
+= output
->pos
- prevOutPos
;
249 zcs
->frameDSize
+= inTmp
.pos
;
251 input
->pos
+= inTmp
.pos
;
253 if (ZSTD_isError(ret
)) return ret
;
256 if (zcs
->maxFrameSize
== zcs
->frameDSize
) {
257 /* log the frame and start over */
258 size_t const ret
= ZSTD_seekable_endFrame(zcs
, output
);
259 if (ZSTD_isError(ret
)) return ret
;
261 /* get the client ready for the next frame */
262 return (size_t)zcs
->maxFrameSize
;
265 return (size_t)(zcs
->maxFrameSize
- zcs
->frameDSize
);
268 static inline size_t ZSTD_seekable_seekTableSize(const ZSTD_frameLog
* fl
)
270 size_t const sizePerFrame
= 8 + (fl
->checksumFlag
?4:0);
271 size_t const seekTableLen
= ZSTD_skippableHeaderSize
+
272 sizePerFrame
* fl
->size
+
273 ZSTD_seekTableFooterSize
;
278 static inline size_t ZSTD_stwrite32(ZSTD_frameLog
* fl
,
279 ZSTD_outBuffer
* output
, U32
const value
,
282 if (fl
->seekTablePos
< offset
+ 4) {
283 BYTE tmp
[4]; /* so that we can work with buffers too small to write a whole word to */
284 size_t const lenWrite
=
285 MIN(output
->size
- output
->pos
, offset
+ 4 - fl
->seekTablePos
);
286 MEM_writeLE32(tmp
, value
);
287 memcpy((BYTE
*)output
->dst
+ output
->pos
,
288 tmp
+ (fl
->seekTablePos
- offset
), lenWrite
);
289 output
->pos
+= lenWrite
;
290 fl
->seekTablePos
+= lenWrite
;
292 if (lenWrite
< 4) return ZSTD_seekable_seekTableSize(fl
) - fl
->seekTablePos
;
297 size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog
* fl
, ZSTD_outBuffer
* output
)
299 /* seekTableIndex: the current index in the table and
300 * seekTableSize: the amount of the table written so far
302 * This function is written this way so that if it has to return early
303 * because of a small buffer, it can keep going where it left off.
306 size_t const sizePerFrame
= 8 + (fl
->checksumFlag
?4:0);
307 size_t const seekTableLen
= ZSTD_seekable_seekTableSize(fl
);
309 CHECK_Z(ZSTD_stwrite32(fl
, output
, ZSTD_MAGIC_SKIPPABLE_START
| 0xE, 0));
310 CHECK_Z(ZSTD_stwrite32(fl
, output
, seekTableLen
- ZSTD_skippableHeaderSize
,
313 while (fl
->seekTableIndex
< fl
->size
) {
314 CHECK_Z(ZSTD_stwrite32(fl
, output
,
315 fl
->entries
[fl
->seekTableIndex
].cSize
,
316 ZSTD_skippableHeaderSize
+
317 sizePerFrame
* fl
->seekTableIndex
+ 0));
319 CHECK_Z(ZSTD_stwrite32(fl
, output
,
320 fl
->entries
[fl
->seekTableIndex
].dSize
,
321 ZSTD_skippableHeaderSize
+
322 sizePerFrame
* fl
->seekTableIndex
+ 4));
324 if (fl
->checksumFlag
) {
325 CHECK_Z(ZSTD_stwrite32(
326 fl
, output
, fl
->entries
[fl
->seekTableIndex
].checksum
,
327 ZSTD_skippableHeaderSize
+
328 sizePerFrame
* fl
->seekTableIndex
+ 8));
331 fl
->seekTableIndex
++;
334 CHECK_Z(ZSTD_stwrite32(fl
, output
, fl
->size
,
335 seekTableLen
- ZSTD_seekTableFooterSize
));
337 if (output
->size
- output
->pos
< 1) return seekTableLen
- fl
->seekTablePos
;
338 if (fl
->seekTablePos
< seekTableLen
- 4) {
340 sfd
|= (fl
->checksumFlag
) << 7;
342 ((BYTE
*)output
->dst
)[output
->pos
] = sfd
;
347 CHECK_Z(ZSTD_stwrite32(fl
, output
, ZSTD_SEEKABLE_MAGICNUMBER
,
350 if (fl
->seekTablePos
!= seekTableLen
) return ERROR(GENERIC
);
354 size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream
* zcs
, ZSTD_outBuffer
* output
)
356 if (!zcs
->writingSeekTable
&& zcs
->frameDSize
) {
357 const size_t endFrame
= ZSTD_seekable_endFrame(zcs
, output
);
358 if (ZSTD_isError(endFrame
)) return endFrame
;
359 /* return an accurate size hint */
360 if (endFrame
) return endFrame
+ ZSTD_seekable_seekTableSize(&zcs
->framelog
);
363 zcs
->writingSeekTable
= 1;
365 return ZSTD_seekable_writeSeekTable(&zcs
->framelog
, output
);