1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
38 #define MAX_WRITE_BITS_SIZE 8
39 #define FORCE_FLUSH 64
40 #define MIN_OBUF_SIZE 224
41 #define NON_EMPTY_BLOCK_SIZE 6
42 #define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE
44 #define MAX_TOKENS (16 * 1024)
48 #include "igzip_lib.h"
49 #include "repeated_char_result.h"
50 #include "huff_codes.h"
51 #include "encode_df.h"
52 #include "igzip_level_buf_structs.h"
54 extern const uint8_t gzip_hdr
[];
55 extern const uint32_t gzip_hdr_bytes
;
56 extern const uint32_t gzip_trl_bytes
;
57 extern const struct isal_hufftables hufftables_default
;
58 extern const struct isal_hufftables hufftables_static
;
60 extern uint32_t crc32_gzip(uint32_t init_crc
, const unsigned char *buf
, uint64_t len
);
62 static int write_stored_block_stateless(struct isal_zstream
*stream
, uint32_t stored_len
,
65 static int write_gzip_header_stateless(struct isal_zstream
*stream
);
66 static void write_gzip_header(struct isal_zstream
*stream
);
67 static int write_deflate_header_stateless(struct isal_zstream
*stream
);
68 static int write_deflate_header_unaligned_stateless(struct isal_zstream
*stream
);
70 unsigned int detect_repeated_char(uint8_t * buf
, uint32_t size
);
72 #define STORED_BLK_HDR_BZ 5
73 #define STORED_BLK_MAX_BZ 65535
75 void isal_deflate_body(struct isal_zstream
*stream
);
76 void isal_deflate_finish(struct isal_zstream
*stream
);
78 void isal_deflate_icf_body(struct isal_zstream
*stream
);
79 void isal_deflate_icf_finish(struct isal_zstream
*stream
);
80 /*****************************************************************/
82 /* Forward declarations */
83 static inline void reset_match_history(struct isal_zstream
*stream
);
84 void write_header(struct isal_zstream
*stream
, uint8_t * deflate_hdr
,
85 uint32_t deflate_hdr_count
, uint32_t extra_bits_count
, uint32_t next_state
,
86 uint32_t toggle_end_of_stream
);
87 void write_deflate_header(struct isal_zstream
*stream
);
88 void write_trailer(struct isal_zstream
*stream
);
97 struct slver isal_deflate_init_slver_01030081
;
98 struct slver isal_deflate_init_slver
= { 0x0081, 0x03, 0x01 };
100 struct slver isal_deflate_stateless_init_slver_00010084
;
101 struct slver isal_deflate_stateless_init_slver
= { 0x0084, 0x01, 0x00 };
103 struct slver isal_deflate_slver_01030082
;
104 struct slver isal_deflate_slver
= { 0x0082, 0x03, 0x01 };
106 struct slver isal_deflate_stateless_slver_01010083
;
107 struct slver isal_deflate_stateless_slver
= { 0x0083, 0x01, 0x01 };
109 struct slver isal_deflate_set_hufftables_slver_00_01_008b
;
110 struct slver isal_deflate_set_hufftables_slver
= { 0x008b, 0x01, 0x00 };
112 /*****************************************************************/
114 void sync_flush(struct isal_zstream
*stream
)
116 struct isal_zstate
*state
= &stream
->internal_state
;
117 uint64_t bits_to_write
= 0xFFFF0000, bits_len
;
121 if (stream
->avail_out
>= 8) {
122 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
124 flush_size
= (-(state
->bitbuf
.m_bit_count
+ 3)) % 8;
126 bits_to_write
<<= flush_size
+ 3;
127 bits_len
= 32 + flush_size
+ 3;
129 #ifdef USE_BITBUFB /* Write Bits Always */
130 state
->state
= ZSTATE_NEW_HDR
;
131 #else /* Not Write Bits Always */
132 state
->state
= ZSTATE_FLUSH_WRITE_BUFFER
;
136 write_bits(&state
->bitbuf
, bits_to_write
, bits_len
);
138 bytes
= buffer_used(&state
->bitbuf
);
139 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
140 stream
->avail_out
-= bytes
;
141 stream
->total_out
+= bytes
;
143 if (stream
->flush
== FULL_FLUSH
) {
144 /* Clear match history so there are no cross
145 * block length distance pairs */
146 state
->file_start
-= state
->b_bytes_processed
;
147 state
->b_bytes_valid
-= state
->b_bytes_processed
;
148 state
->b_bytes_processed
= 0;
149 reset_match_history(stream
);
154 static void flush_write_buffer(struct isal_zstream
*stream
)
156 struct isal_zstate
*state
= &stream
->internal_state
;
158 if (stream
->avail_out
>= 8) {
159 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
160 flush(&state
->bitbuf
);
161 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
162 bytes
= buffer_used(&state
->bitbuf
);
163 stream
->avail_out
-= bytes
;
164 stream
->total_out
+= bytes
;
165 state
->state
= ZSTATE_NEW_HDR
;
169 static void flush_icf_block(struct isal_zstream
*stream
)
171 struct isal_zstate
*state
= &stream
->internal_state
;
172 struct level_2_buf
*level_buf
= (struct level_2_buf
*)stream
->level_buf
;
173 struct BitBuf2
*write_buf
= &state
->bitbuf
;
174 struct deflate_icf
*icf_buf_encoded_next
;
176 set_buf(write_buf
, stream
->next_out
, stream
->avail_out
);
178 #if defined (USE_BITBUF8) || (USE_BITBUF_ELSE)
179 if (!is_full(write_buf
))
180 flush_bits(write_buf
);
183 icf_buf_encoded_next
= encode_deflate_icf(level_buf
->icf_buf_start
+ state
->count
,
184 level_buf
->icf_buf_next
, write_buf
,
185 &level_buf
->encode_tables
);
187 state
->count
= icf_buf_encoded_next
- level_buf
->icf_buf_start
;
188 stream
->next_out
= buffer_ptr(write_buf
);
189 stream
->total_out
+= buffer_used(write_buf
);
190 stream
->avail_out
-= buffer_used(write_buf
);
192 if (level_buf
->icf_buf_next
<= icf_buf_encoded_next
) {
194 if (stream
->avail_in
== 0 && stream
->end_of_stream
)
195 state
->state
= ZSTATE_TRL
;
196 else if (stream
->avail_in
== 0 && stream
->flush
!= NO_FLUSH
)
197 state
->state
= ZSTATE_SYNC_FLUSH
;
199 state
->state
= ZSTATE_NEW_HDR
;
203 static void init_new_icf_block(struct isal_zstream
*stream
)
205 struct isal_zstate
*state
= &stream
->internal_state
;
206 struct level_2_buf
*level_buf
= (struct level_2_buf
*)stream
->level_buf
;
208 if (stream
->level_buf_size
>=
209 sizeof(struct level_2_buf
) + 100 * sizeof(struct deflate_icf
)) {
210 level_buf
->icf_buf_next
= level_buf
->icf_buf_start
;
211 level_buf
->icf_buf_avail_out
=
212 stream
->level_buf_size
- sizeof(struct level_2_buf
) -
213 sizeof(struct deflate_icf
);
214 memset(&state
->hist
, 0, sizeof(struct isal_mod_hist
));
215 state
->state
= ZSTATE_BODY
;
219 static void create_icf_block_hdr(struct isal_zstream
*stream
)
221 struct isal_zstate
*state
= &stream
->internal_state
;
222 struct level_2_buf
*level_buf
= (struct level_2_buf
*)stream
->level_buf
;
223 struct BitBuf2
*write_buf
= &state
->bitbuf
;
224 struct BitBuf2 write_buf_tmp
;
225 uint32_t out_size
= stream
->avail_out
;
226 uint8_t *end_out
= stream
->next_out
+ out_size
;
227 /* Write EOB in icf_buf */
228 state
->hist
.ll_hist
[256] = 1;
229 level_buf
->icf_buf_next
->lit_len
= 0x100;
230 level_buf
->icf_buf_next
->lit_dist
= NULL_DIST_SYM
;
231 level_buf
->icf_buf_next
->dist_extra
= 0;
232 level_buf
->icf_buf_next
++;
234 state
->has_eob_hdr
= stream
->end_of_stream
&& !stream
->avail_in
;
235 if (end_out
- stream
->next_out
>= ISAL_DEF_MAX_HDR_SIZE
) {
236 /* Determine whether this is the final block */
238 if (stream
->gzip_flag
== IGZIP_GZIP
)
239 write_gzip_header_stateless(stream
);
241 set_buf(write_buf
, stream
->next_out
, stream
->avail_out
);
243 create_hufftables_icf(write_buf
, &level_buf
->encode_tables
, &state
->hist
,
245 state
->state
= ZSTATE_FLUSH_ICF_BUFFER
;
246 stream
->next_out
= buffer_ptr(write_buf
);
247 stream
->total_out
+= buffer_used(write_buf
);
248 stream
->avail_out
-= buffer_used(write_buf
);
250 /* Start writing into temporary buffer */
251 write_buf_tmp
.m_bits
= write_buf
->m_bits
;
252 write_buf_tmp
.m_bit_count
= write_buf
->m_bit_count
;
254 write_buf
->m_bits
= 0;
255 write_buf
->m_bit_count
= 0;
257 set_buf(&write_buf_tmp
, level_buf
->deflate_hdr
, ISAL_DEF_MAX_HDR_SIZE
);
259 create_hufftables_icf(&write_buf_tmp
, &level_buf
->encode_tables
,
260 &state
->hist
, state
->has_eob_hdr
);
262 level_buf
->deflate_hdr_count
= buffer_used(&write_buf_tmp
);
263 level_buf
->deflate_hdr_extra_bits
= write_buf_tmp
.m_bit_count
;
264 flush(&write_buf_tmp
);
266 state
->state
= ZSTATE_HDR
;
270 static void isal_deflate_pass(struct isal_zstream
*stream
)
272 struct isal_zstate
*state
= &stream
->internal_state
;
273 struct isal_hufftables
*hufftables
= stream
->hufftables
;
274 uint8_t *start_in
= stream
->next_in
;
276 if (state
->state
== ZSTATE_NEW_HDR
|| state
->state
== ZSTATE_HDR
) {
277 if (state
->count
== 0)
278 /* Assume the final header is being written since the header
279 * stored in hufftables is the final header. */
280 state
->has_eob_hdr
= 1;
281 write_header(stream
, hufftables
->deflate_hdr
, hufftables
->deflate_hdr_count
,
282 hufftables
->deflate_hdr_extra_bits
, ZSTATE_BODY
,
283 !stream
->end_of_stream
);
286 if (state
->state
== ZSTATE_BODY
)
287 isal_deflate_body(stream
);
289 if (state
->state
== ZSTATE_FLUSH_READ_BUFFER
)
290 isal_deflate_finish(stream
);
292 if (state
->state
== ZSTATE_SYNC_FLUSH
)
295 if (state
->state
== ZSTATE_FLUSH_WRITE_BUFFER
)
296 flush_write_buffer(stream
);
298 if (stream
->gzip_flag
)
299 state
->crc
= crc32_gzip(state
->crc
, start_in
, stream
->next_in
- start_in
);
301 if (state
->state
== ZSTATE_TRL
)
302 write_trailer(stream
);
305 static void isal_deflate_icf_pass(struct isal_zstream
*stream
)
307 uint8_t *start_in
= stream
->next_in
;
308 struct isal_zstate
*state
= &stream
->internal_state
;
309 struct level_2_buf
*level_buf
= (struct level_2_buf
*)stream
->level_buf
;
312 if (state
->state
== ZSTATE_NEW_HDR
)
313 init_new_icf_block(stream
);
315 if (state
->state
== ZSTATE_BODY
)
316 isal_deflate_icf_body(stream
);
318 if (state
->state
== ZSTATE_FLUSH_READ_BUFFER
)
319 isal_deflate_icf_finish(stream
);
321 if (state
->state
== ZSTATE_CREATE_HDR
)
322 create_icf_block_hdr(stream
);
324 if (state
->state
== ZSTATE_HDR
)
325 /* Note that the header may be prepended by the
326 * remaining bits in the previous block, as such the
327 * toggle header flag cannot be used */
328 write_header(stream
, level_buf
->deflate_hdr
,
329 level_buf
->deflate_hdr_count
,
330 level_buf
->deflate_hdr_extra_bits
,
331 ZSTATE_FLUSH_ICF_BUFFER
, 0);
333 if (state
->state
== ZSTATE_FLUSH_ICF_BUFFER
)
334 flush_icf_block(stream
);
336 } while (state
->state
== ZSTATE_NEW_HDR
);
338 if (state
->state
== ZSTATE_SYNC_FLUSH
)
341 if (state
->state
== ZSTATE_FLUSH_WRITE_BUFFER
)
342 flush_write_buffer(stream
);
344 if (stream
->gzip_flag
)
345 state
->crc
= crc32_gzip(state
->crc
, start_in
, stream
->next_in
- start_in
);
347 if (state
->state
== ZSTATE_TRL
)
348 write_trailer(stream
);
351 static void isal_deflate_int(struct isal_zstream
*stream
)
353 struct isal_zstate
*state
= &stream
->internal_state
;
356 /* Move data from temporary output buffer to output buffer */
357 if (state
->state
>= ZSTATE_TMP_OFFSET
) {
358 size
= state
->tmp_out_end
- state
->tmp_out_start
;
359 if (size
> stream
->avail_out
)
360 size
= stream
->avail_out
;
361 memcpy(stream
->next_out
, state
->tmp_out_buff
+ state
->tmp_out_start
, size
);
362 stream
->next_out
+= size
;
363 stream
->avail_out
-= size
;
364 stream
->total_out
+= size
;
365 state
->tmp_out_start
+= size
;
367 if (state
->tmp_out_start
== state
->tmp_out_end
)
368 state
->state
-= ZSTATE_TMP_OFFSET
;
370 if (stream
->avail_out
== 0 || state
->state
== ZSTATE_END
371 // or do not write out empty blocks since the outbuffer was processed
372 || (state
->state
== ZSTATE_NEW_HDR
&& stream
->avail_out
== 0))
375 assert(state
->tmp_out_start
== state
->tmp_out_end
);
377 if (stream
->level
== 0)
378 isal_deflate_pass(stream
);
380 isal_deflate_icf_pass(stream
);
382 /* Fill temporary output buffer then complete filling output buffer */
383 if (stream
->avail_out
> 0 && stream
->avail_out
< 8 && state
->state
!= ZSTATE_NEW_HDR
) {
388 next_out
= stream
->next_out
;
389 avail_out
= stream
->avail_out
;
390 total_out
= stream
->total_out
;
392 stream
->next_out
= state
->tmp_out_buff
;
393 stream
->avail_out
= sizeof(state
->tmp_out_buff
);
394 stream
->total_out
= 0;
396 if (stream
->level
== 0)
397 isal_deflate_pass(stream
);
399 isal_deflate_icf_pass(stream
);
401 state
->tmp_out_start
= 0;
402 state
->tmp_out_end
= stream
->total_out
;
404 stream
->next_out
= next_out
;
405 stream
->avail_out
= avail_out
;
406 stream
->total_out
= total_out
;
407 if (state
->tmp_out_end
) {
408 size
= state
->tmp_out_end
;
409 if (size
> stream
->avail_out
)
410 size
= stream
->avail_out
;
411 memcpy(stream
->next_out
, state
->tmp_out_buff
, size
);
412 stream
->next_out
+= size
;
413 stream
->avail_out
-= size
;
414 stream
->total_out
+= size
;
415 state
->tmp_out_start
+= size
;
416 if (state
->tmp_out_start
!= state
->tmp_out_end
)
417 state
->state
+= ZSTATE_TMP_OFFSET
;
424 static void write_constant_compressed_stateless(struct isal_zstream
*stream
,
425 uint32_t repeated_length
)
427 /* Assumes repeated_length is at least 1.
428 * Assumes the input end_of_stream is either 0 or 1. */
429 struct isal_zstate
*state
= &stream
->internal_state
;
430 uint32_t rep_bits
= ((repeated_length
- 1) / 258) * 2;
431 uint32_t rep_bytes
= rep_bits
/ 8;
432 uint32_t rep_extra
= (repeated_length
- 1) % 258;
434 uint32_t repeated_char
= *stream
->next_in
;
435 uint8_t *start_in
= stream
->next_in
;
437 /* Guarantee there is enough space for the header even in the worst case */
438 if (stream
->avail_out
< HEADER_LENGTH
+ MAX_FIXUP_CODE_LENGTH
+ rep_bytes
+ 8)
441 /* Assumes the repeated char is either 0 or 0xFF. */
442 memcpy(stream
->next_out
, repeated_char_header
[repeated_char
& 1], HEADER_LENGTH
);
444 if (stream
->avail_in
== repeated_length
&& stream
->end_of_stream
> 0) {
445 stream
->next_out
[0] |= 1;
446 state
->has_eob_hdr
= 1;
448 state
->state
= ZSTATE_TRL
;
450 state
->state
= ZSTATE_NEW_HDR
;
453 memset(stream
->next_out
+ HEADER_LENGTH
, 0, rep_bytes
);
454 stream
->avail_out
-= HEADER_LENGTH
+ rep_bytes
;
455 stream
->next_out
+= HEADER_LENGTH
+ rep_bytes
;
456 stream
->total_out
+= HEADER_LENGTH
+ rep_bytes
;
458 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
460 /* These two lines are basically a modified version of init. */
461 state
->bitbuf
.m_bits
= 0;
462 state
->bitbuf
.m_bit_count
= rep_bits
% 8;
464 /* Add smaller repeat codes as necessary. Code280 can describe repeat
465 * lengths of 115-130 bits. Code10 can describe repeat lengths of 10
466 * bits. If more than 230 bits, fill code with two code280s. Else if
467 * more than 115 repeates, fill with code10s until one code280 can
468 * finish the rest of the repeats. Else, fill with code10s and
470 if (rep_extra
> 115) {
471 while (rep_extra
> 130 && rep_extra
< 230) {
472 write_bits(&state
->bitbuf
, CODE_10
, CODE_10_LENGTH
);
476 if (rep_extra
>= 230) {
477 write_bits(&state
->bitbuf
,
478 CODE_280
| ((rep_extra
/ 2 - 115) <<
479 CODE_280_LENGTH
), CODE_280_TOTAL_LENGTH
);
480 rep_extra
-= rep_extra
/ 2;
483 write_bits(&state
->bitbuf
,
484 CODE_280
| ((rep_extra
- 115) << CODE_280_LENGTH
),
485 CODE_280_TOTAL_LENGTH
);
488 while (rep_extra
>= 10) {
490 write_bits(&state
->bitbuf
, CODE_10
, CODE_10_LENGTH
);
494 for (; rep_extra
> 0; rep_extra
--)
495 write_bits(&state
->bitbuf
, CODE_LIT
, CODE_LIT_LENGTH
);
498 write_bits(&state
->bitbuf
, END_OF_BLOCK
, END_OF_BLOCK_LEN
);
500 stream
->next_in
+= repeated_length
;
501 stream
->avail_in
-= repeated_length
;
502 stream
->total_in
+= repeated_length
;
504 bytes
= buffer_used(&state
->bitbuf
);
505 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
506 stream
->avail_out
-= bytes
;
507 stream
->total_out
+= bytes
;
509 if (stream
->gzip_flag
)
510 state
->crc
= crc32_gzip(state
->crc
, start_in
, stream
->next_in
- start_in
);
515 int detect_repeated_char_length(uint8_t * in
, uint32_t length
)
517 /* This currently assumes the first 8 bytes are the same character.
518 * This won't work effectively if the input stream isn't aligned well. */
519 uint8_t *p_8
, *end
= in
+ length
;
520 uint64_t *p_64
= (uint64_t *) in
;
522 uint8_t c
= (uint8_t) w
;
524 for (; (p_64
<= (uint64_t *) (end
- 8)) && (w
== *p_64
); p_64
++) ;
526 p_8
= (uint8_t *) p_64
;
528 for (; (p_8
< end
) && (c
== *p_8
); p_8
++) ;
533 static int isal_deflate_int_stateless(struct isal_zstream
*stream
)
535 uint32_t repeat_length
;
536 struct isal_zstate
*state
= &stream
->internal_state
;
538 if (stream
->gzip_flag
== IGZIP_GZIP
)
539 if (write_gzip_header_stateless(stream
))
540 return STATELESS_OVERFLOW
;
542 if (stream
->avail_in
>= 8
543 && (*(uint64_t *) stream
->next_in
== 0
544 || *(uint64_t *) stream
->next_in
== ~(uint64_t) 0)) {
545 repeat_length
= detect_repeated_char_length(stream
->next_in
, stream
->avail_in
);
547 if (stream
->avail_in
== repeat_length
|| repeat_length
>= MIN_REPEAT_LEN
)
548 write_constant_compressed_stateless(stream
, repeat_length
);
551 if (stream
->level
== 0) {
552 if (state
->state
== ZSTATE_NEW_HDR
|| state
->state
== ZSTATE_HDR
) {
553 write_deflate_header_unaligned_stateless(stream
);
554 if (state
->state
== ZSTATE_NEW_HDR
|| state
->state
== ZSTATE_HDR
)
555 return STATELESS_OVERFLOW
;
557 reset_match_history(stream
);
560 state
->file_start
= stream
->next_in
- stream
->total_in
;
561 isal_deflate_pass(stream
);
563 } else if (stream
->level
== 1) {
564 if (stream
->level_buf
== NULL
|| stream
->level_buf_size
< ISAL_DEF_LVL1_MIN
) {
565 /* Default to internal buffer if invalid size is supplied */
566 stream
->level_buf
= state
->buffer
;
567 stream
->level_buf_size
= sizeof(state
->buffer
);
570 if (state
->state
== ZSTATE_NEW_HDR
|| state
->state
== ZSTATE_HDR
)
571 reset_match_history(stream
);
574 state
->file_start
= stream
->next_in
- stream
->total_in
;
575 isal_deflate_icf_pass(stream
);
578 return ISAL_INVALID_LEVEL
;
580 if (state
->state
== ZSTATE_END
581 || (state
->state
== ZSTATE_NEW_HDR
&& stream
->flush
== FULL_FLUSH
))
584 return STATELESS_OVERFLOW
;
587 static int write_stored_block_stateless(struct isal_zstream
*stream
,
588 uint32_t stored_len
, uint32_t crc32
)
590 uint64_t stored_blk_hdr
;
595 if (stream
->avail_out
< stored_len
)
596 return STATELESS_OVERFLOW
;
598 stream
->avail_out
-= stored_len
;
599 stream
->total_out
+= stored_len
;
600 avail_in
= stream
->avail_in
;
602 if (stream
->gzip_flag
== IGZIP_GZIP
) {
603 memcpy(stream
->next_out
, gzip_hdr
, gzip_hdr_bytes
);
604 stream
->next_out
+= gzip_hdr_bytes
;
605 stream
->gzip_flag
= IGZIP_GZIP_NO_HDR
;
609 if (avail_in
>= STORED_BLK_MAX_BZ
) {
610 stored_blk_hdr
= 0xFFFF00;
611 copy_size
= STORED_BLK_MAX_BZ
;
613 stored_blk_hdr
= ~avail_in
;
614 stored_blk_hdr
<<= 24;
615 stored_blk_hdr
|= (avail_in
& 0xFFFF) << 8;
616 copy_size
= avail_in
;
619 avail_in
-= copy_size
;
621 /* Handle BFINAL bit */
623 if (stream
->flush
== NO_FLUSH
|| stream
->end_of_stream
) {
624 stored_blk_hdr
|= 0x1;
625 stream
->internal_state
.has_eob_hdr
= 1;
628 memcpy(stream
->next_out
, &stored_blk_hdr
, STORED_BLK_HDR_BZ
);
629 stream
->next_out
+= STORED_BLK_HDR_BZ
;
631 memcpy(stream
->next_out
, stream
->next_in
, copy_size
);
632 stream
->next_out
+= copy_size
;
633 stream
->next_in
+= copy_size
;
634 stream
->total_in
+= copy_size
;
635 } while (avail_in
!= 0);
637 if (stream
->gzip_flag
&& stream
->internal_state
.has_eob_hdr
) {
638 gzip_trl
= stream
->avail_in
;
640 gzip_trl
|= crc32
& 0xFFFFFFFF;
641 memcpy(stream
->next_out
, &gzip_trl
, gzip_trl_bytes
);
642 stream
->next_out
+= gzip_trl_bytes
;
645 stream
->avail_in
= 0;
649 static inline void reset_match_history(struct isal_zstream
*stream
)
651 struct isal_zstate
*state
= &stream
->internal_state
;
652 uint16_t *head
= stream
->internal_state
.head
;
657 if (stream
->total_in
== 0)
658 memset(stream
->internal_state
.head
, 0, sizeof(stream
->internal_state
.head
));
660 for (i
= 0; i
< sizeof(state
->head
) / 2; i
++) {
661 head
[i
] = (uint16_t) (stream
->total_in
);
666 void isal_deflate_init(struct isal_zstream
*stream
)
668 struct isal_zstate
*state
= &stream
->internal_state
;
670 stream
->total_in
= 0;
671 stream
->total_out
= 0;
672 stream
->hufftables
= (struct isal_hufftables
*)&hufftables_default
;
674 stream
->level_buf
= NULL
;
675 stream
->level_buf_size
= 0;
676 stream
->end_of_stream
= 0;
677 stream
->flush
= NO_FLUSH
;
678 stream
->gzip_flag
= 0;
680 state
->b_bytes_valid
= 0;
681 state
->b_bytes_processed
= 0;
683 state
->has_eob_hdr
= 0;
685 state
->state
= ZSTATE_NEW_HDR
;
688 state
->tmp_out_start
= 0;
689 state
->tmp_out_end
= 0;
691 state
->file_start
= stream
->next_in
;
693 init(&state
->bitbuf
);
697 memset(state
->head
, 0, sizeof(state
->head
));
702 int isal_deflate_set_hufftables(struct isal_zstream
*stream
,
703 struct isal_hufftables
*hufftables
, int type
)
705 if (stream
->internal_state
.state
!= ZSTATE_NEW_HDR
)
706 return ISAL_INVALID_OPERATION
;
709 case IGZIP_HUFFTABLE_DEFAULT
:
710 stream
->hufftables
= (struct isal_hufftables
*)&hufftables_default
;
712 case IGZIP_HUFFTABLE_STATIC
:
713 stream
->hufftables
= (struct isal_hufftables
*)&hufftables_static
;
715 case IGZIP_HUFFTABLE_CUSTOM
:
716 if (hufftables
!= NULL
) {
717 stream
->hufftables
= hufftables
;
721 return ISAL_INVALID_OPERATION
;
727 void isal_deflate_stateless_init(struct isal_zstream
*stream
)
729 stream
->total_in
= 0;
730 stream
->total_out
= 0;
731 stream
->hufftables
= (struct isal_hufftables
*)&hufftables_default
;
733 stream
->level_buf
= NULL
;
734 stream
->level_buf_size
= 0;
735 stream
->end_of_stream
= 0;
736 stream
->flush
= NO_FLUSH
;
737 stream
->gzip_flag
= 0;
738 stream
->internal_state
.state
= ZSTATE_NEW_HDR
;
742 int isal_deflate_stateless(struct isal_zstream
*stream
)
744 uint8_t *next_in
= stream
->next_in
;
745 const uint32_t avail_in
= stream
->avail_in
;
746 const uint32_t total_in
= stream
->total_in
;
748 uint8_t *next_out
= stream
->next_out
;
749 const uint32_t avail_out
= stream
->avail_out
;
750 const uint32_t total_out
= stream
->total_out
;
751 const uint32_t gzip_flag
= stream
->gzip_flag
;
756 /* Final block has already been written */
757 stream
->internal_state
.has_eob_hdr
= 0;
758 init(&stream
->internal_state
.bitbuf
);
759 stream
->internal_state
.state
= ZSTATE_NEW_HDR
;
760 stream
->internal_state
.crc
= 0;
762 if (stream
->flush
== NO_FLUSH
)
763 stream
->end_of_stream
= 1;
765 if (stream
->flush
!= NO_FLUSH
&& stream
->flush
!= FULL_FLUSH
)
766 return INVALID_FLUSH
;
768 if (stream
->level
!= 0 && stream
->level
!= 1)
769 return ISAL_INVALID_LEVEL
;
772 stored_len
= STORED_BLK_HDR_BZ
;
775 STORED_BLK_HDR_BZ
* ((avail_in
+ STORED_BLK_MAX_BZ
- 1) /
776 STORED_BLK_MAX_BZ
) + avail_in
;
779 at least 1 byte compressed data in the case of empty dynamic block which only
783 if (stream
->gzip_flag
== IGZIP_GZIP
)
784 stored_len
+= gzip_hdr_bytes
+ gzip_trl_bytes
;
786 else if (stream
->gzip_flag
== IGZIP_GZIP_NO_HDR
)
787 stored_len
+= gzip_trl_bytes
;
790 the output buffer should be no less than 8 bytes
791 while empty stored deflate block is 5 bytes only
793 if (stream
->avail_out
< 8)
794 return STATELESS_OVERFLOW
;
796 if (isal_deflate_int_stateless(stream
) == COMP_OK
)
799 if (stream
->flush
== FULL_FLUSH
) {
800 stream
->internal_state
.file_start
=
801 (uint8_t *) & stream
->internal_state
.buffer
;
802 reset_match_history(stream
);
804 stream
->internal_state
.has_eob_hdr
= 0;
807 if (avail_out
< stored_len
)
808 return STATELESS_OVERFLOW
;
810 stream
->next_in
= next_in
;
811 stream
->avail_in
= avail_in
;
812 stream
->total_in
= total_in
;
814 stream
->next_out
= next_out
;
815 stream
->avail_out
= avail_out
;
816 stream
->total_out
= total_out
;
818 stream
->gzip_flag
= gzip_flag
;
820 if (stream
->gzip_flag
)
821 crc32
= crc32_gzip(0x0, next_in
, avail_in
);
823 return write_stored_block_stateless(stream
, stored_len
, crc32
);
826 int isal_deflate(struct isal_zstream
*stream
)
828 struct isal_zstate
*state
= &stream
->internal_state
;
831 uint32_t avail_in
, avail_in_start
;
832 uint32_t flush_type
= stream
->flush
;
833 uint32_t end_of_stream
= stream
->end_of_stream
;
835 uint8_t *copy_down_src
= NULL
;
836 uint64_t copy_down_size
= 0;
837 int32_t processed
= -(state
->b_bytes_valid
- state
->b_bytes_processed
);
839 if (stream
->flush
>= 3)
840 return INVALID_FLUSH
;
842 next_in
= stream
->next_in
;
843 avail_in
= stream
->avail_in
;
844 stream
->total_in
-= state
->b_bytes_valid
- state
->b_bytes_processed
;
848 if (size
> sizeof(state
->buffer
) - state
->b_bytes_valid
) {
849 size
= sizeof(state
->buffer
) - state
->b_bytes_valid
;
850 stream
->flush
= NO_FLUSH
;
851 stream
->end_of_stream
= 0;
853 memcpy(&state
->buffer
[state
->b_bytes_valid
], next_in
, size
);
857 state
->b_bytes_valid
+= size
;
859 stream
->next_in
= &state
->buffer
[state
->b_bytes_processed
];
860 stream
->avail_in
= state
->b_bytes_valid
- state
->b_bytes_processed
;
861 state
->file_start
= stream
->next_in
- stream
->total_in
;
862 processed
+= stream
->avail_in
;
864 if (stream
->avail_in
> IGZIP_HIST_SIZE
865 || stream
->end_of_stream
|| stream
->flush
!= NO_FLUSH
) {
866 avail_in_start
= stream
->avail_in
;
867 isal_deflate_int(stream
);
868 state
->b_bytes_processed
+= avail_in_start
- stream
->avail_in
;
870 if (state
->b_bytes_processed
> IGZIP_HIST_SIZE
) {
872 &state
->buffer
[state
->b_bytes_processed
- IGZIP_HIST_SIZE
];
874 state
->b_bytes_valid
- state
->b_bytes_processed
+
876 memmove(state
->buffer
, copy_down_src
, copy_down_size
);
878 state
->b_bytes_valid
-= copy_down_src
- state
->buffer
;
879 state
->b_bytes_processed
-= copy_down_src
- state
->buffer
;
884 stream
->flush
= flush_type
;
885 stream
->end_of_stream
= end_of_stream
;
886 processed
-= stream
->avail_in
;
887 } while (processed
< IGZIP_HIST_SIZE
+ ISAL_LOOK_AHEAD
&& avail_in
> 0
888 && stream
->avail_out
> 0);
890 if (processed
>= IGZIP_HIST_SIZE
+ ISAL_LOOK_AHEAD
) {
891 stream
->next_in
= next_in
- stream
->avail_in
;
892 stream
->avail_in
= avail_in
+ stream
->avail_in
;
894 state
->file_start
= stream
->next_in
- stream
->total_in
;
896 if (stream
->avail_in
> 0 && stream
->avail_out
> 0)
897 isal_deflate_int(stream
);
899 size
= stream
->avail_in
;
900 if (stream
->avail_in
> IGZIP_HIST_SIZE
)
903 memmove(state
->buffer
, stream
->next_in
- IGZIP_HIST_SIZE
,
904 size
+ IGZIP_HIST_SIZE
);
905 state
->b_bytes_processed
= IGZIP_HIST_SIZE
;
906 state
->b_bytes_valid
= size
+ IGZIP_HIST_SIZE
;
908 stream
->next_in
+= size
;
909 stream
->avail_in
-= size
;
910 stream
->total_in
+= size
;
913 stream
->total_in
+= state
->b_bytes_valid
- state
->b_bytes_processed
;
914 stream
->next_in
= next_in
;
915 stream
->avail_in
= avail_in
;
916 state
->file_start
= stream
->next_in
- stream
->total_in
;
923 static int write_gzip_header_stateless(struct isal_zstream
*stream
)
925 if (gzip_hdr_bytes
>= stream
->avail_out
)
926 return STATELESS_OVERFLOW
;
928 stream
->avail_out
-= gzip_hdr_bytes
;
929 stream
->total_out
+= gzip_hdr_bytes
;
931 memcpy(stream
->next_out
, gzip_hdr
, gzip_hdr_bytes
);
933 stream
->next_out
+= gzip_hdr_bytes
;
934 stream
->gzip_flag
= IGZIP_GZIP_NO_HDR
;
939 static void write_gzip_header(struct isal_zstream
*stream
)
941 struct isal_zstate
*state
= &stream
->internal_state
;
942 int bytes_to_write
= gzip_hdr_bytes
;
944 bytes_to_write
-= state
->count
;
946 if (bytes_to_write
> stream
->avail_out
)
947 bytes_to_write
= stream
->avail_out
;
949 memcpy(stream
->next_out
, gzip_hdr
+ state
->count
, bytes_to_write
);
950 state
->count
+= bytes_to_write
;
952 if (state
->count
== gzip_hdr_bytes
) {
954 stream
->gzip_flag
= IGZIP_GZIP_NO_HDR
;
957 stream
->avail_out
-= bytes_to_write
;
958 stream
->total_out
+= bytes_to_write
;
959 stream
->next_out
+= bytes_to_write
;
963 static int write_deflate_header_stateless(struct isal_zstream
*stream
)
965 struct isal_zstate
*state
= &stream
->internal_state
;
966 struct isal_hufftables
*hufftables
= stream
->hufftables
;
967 uint64_t hdr_extra_bits
= hufftables
->deflate_hdr
[hufftables
->deflate_hdr_count
];
970 if (hufftables
->deflate_hdr_count
+ 8 >= stream
->avail_out
)
971 return STATELESS_OVERFLOW
;
973 memcpy(stream
->next_out
, hufftables
->deflate_hdr
, hufftables
->deflate_hdr_count
);
975 if (stream
->end_of_stream
== 0) {
976 if (hufftables
->deflate_hdr_count
> 0)
977 *stream
->next_out
-= 1;
981 state
->has_eob_hdr
= 1;
983 stream
->avail_out
-= hufftables
->deflate_hdr_count
;
984 stream
->total_out
+= hufftables
->deflate_hdr_count
;
985 stream
->next_out
+= hufftables
->deflate_hdr_count
;
987 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
989 write_bits(&state
->bitbuf
, hdr_extra_bits
, hufftables
->deflate_hdr_extra_bits
);
991 count
= buffer_used(&state
->bitbuf
);
992 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
993 stream
->avail_out
-= count
;
994 stream
->total_out
+= count
;
996 state
->state
= ZSTATE_BODY
;
1001 static int write_deflate_header_unaligned_stateless(struct isal_zstream
*stream
)
1003 struct isal_zstate
*state
= &stream
->internal_state
;
1004 struct isal_hufftables
*hufftables
= stream
->hufftables
;
1007 uint64_t *header_next
;
1008 uint64_t *header_end
;
1009 uint64_t header_bits
;
1011 if (state
->bitbuf
.m_bit_count
== 0)
1012 return write_deflate_header_stateless(stream
);
1014 if (hufftables
->deflate_hdr_count
+ 16 >= stream
->avail_out
)
1015 return STATELESS_OVERFLOW
;
1017 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
1019 header_next
= (uint64_t *) hufftables
->deflate_hdr
;
1020 header_end
= header_next
+ hufftables
->deflate_hdr_count
/ 8;
1022 header_bits
= *header_next
;
1024 if (stream
->end_of_stream
== 0)
1027 state
->has_eob_hdr
= 1;
1031 /* Write out Complete Header bits */
1032 for (; header_next
<= header_end
; header_next
++) {
1033 write_bits(&state
->bitbuf
, header_bits
, 32);
1035 write_bits(&state
->bitbuf
, header_bits
, 32);
1036 header_bits
= *header_next
;
1040 (hufftables
->deflate_hdr_count
& 0x7) * 8 + hufftables
->deflate_hdr_extra_bits
;
1042 if (bit_count
> MAX_BITBUF_BIT_WRITE
) {
1043 write_bits(&state
->bitbuf
, header_bits
, MAX_BITBUF_BIT_WRITE
);
1044 header_bits
>>= MAX_BITBUF_BIT_WRITE
;
1045 bit_count
-= MAX_BITBUF_BIT_WRITE
;
1049 write_bits(&state
->bitbuf
, header_bits
, bit_count
);
1051 /* check_space flushes extra bytes in bitbuf. Required because
1052 * write_bits_always fails when the next commit makes the buffer
1053 * length exceed 64 bits */
1054 check_space(&state
->bitbuf
, FORCE_FLUSH
);
1056 count
= buffer_used(&state
->bitbuf
);
1057 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
1058 stream
->avail_out
-= count
;
1059 stream
->total_out
+= count
;
1061 state
->state
= ZSTATE_BODY
;
1066 /* Toggle end of stream only works when deflate header is aligned */
1067 void write_header(struct isal_zstream
*stream
, uint8_t * deflate_hdr
,
1068 uint32_t deflate_hdr_count
, uint32_t extra_bits_count
, uint32_t next_state
,
1069 uint32_t toggle_end_of_stream
)
1071 struct isal_zstate
*state
= &stream
->internal_state
;
1072 uint32_t hdr_extra_bits
= deflate_hdr
[deflate_hdr_count
];
1074 state
->state
= ZSTATE_HDR
;
1076 if (state
->bitbuf
.m_bit_count
!= 0) {
1077 if (stream
->avail_out
< 8)
1079 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
1080 flush(&state
->bitbuf
);
1081 count
= buffer_used(&state
->bitbuf
);
1082 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
1083 stream
->avail_out
-= count
;
1084 stream
->total_out
+= count
;
1087 if (stream
->gzip_flag
== IGZIP_GZIP
)
1088 write_gzip_header(stream
);
1090 count
= deflate_hdr_count
- state
->count
;
1093 if (count
> stream
->avail_out
)
1094 count
= stream
->avail_out
;
1096 memcpy(stream
->next_out
, deflate_hdr
+ state
->count
, count
);
1098 if (toggle_end_of_stream
&& state
->count
== 0 && count
> 0) {
1099 /* Assumes the final block bit is the first bit */
1100 *stream
->next_out
^= 1;
1101 state
->has_eob_hdr
= !state
->has_eob_hdr
;
1104 stream
->next_out
+= count
;
1105 stream
->avail_out
-= count
;
1106 stream
->total_out
+= count
;
1107 state
->count
+= count
;
1109 count
= deflate_hdr_count
- state
->count
;
1110 } else if (toggle_end_of_stream
&& deflate_hdr_count
== 0) {
1111 /* Assumes the final block bit is the first bit */
1112 hdr_extra_bits
^= 1;
1113 state
->has_eob_hdr
= !state
->has_eob_hdr
;
1116 if ((count
== 0) && (stream
->avail_out
>= 8)) {
1118 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
1120 write_bits(&state
->bitbuf
, hdr_extra_bits
, extra_bits_count
);
1122 state
->state
= next_state
;
1125 count
= buffer_used(&state
->bitbuf
);
1126 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
1127 stream
->avail_out
-= count
;
1128 stream
->total_out
+= count
;
1133 void write_trailer(struct isal_zstream
*stream
)
1135 struct isal_zstate
*state
= &stream
->internal_state
;
1137 uint32_t crc
= state
->crc
;
1139 if (stream
->avail_out
>= 8) {
1140 set_buf(&state
->bitbuf
, stream
->next_out
, stream
->avail_out
);
1142 /* the flush() will pad to the next byte and write up to 8 bytes
1143 * to the output stream/buffer.
1145 if (!state
->has_eob_hdr
) {
1146 /* If the final header has not been written, write a
1147 * final block. This block is a static huffman block
1148 * which only contains the end of block symbol. The code
1149 * that happens to do this is the fist 10 bits of
1151 state
->has_eob_hdr
= 1;
1152 write_bits(&state
->bitbuf
, 0x003, 10);
1153 if (is_full(&state
->bitbuf
)) {
1154 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
1155 bytes
= buffer_used(&state
->bitbuf
);
1156 stream
->avail_out
-= bytes
;
1157 stream
->total_out
+= bytes
;
1162 flush(&state
->bitbuf
);
1163 stream
->next_out
= buffer_ptr(&state
->bitbuf
);
1164 bytes
= buffer_used(&state
->bitbuf
);
1166 if (stream
->gzip_flag
) {
1167 if (!is_full(&state
->bitbuf
)) {
1168 *(uint64_t *) stream
->next_out
=
1169 ((uint64_t) stream
->total_in
<< 32) | crc
;
1170 stream
->next_out
+= 8;
1172 state
->state
= ZSTATE_END
;
1175 state
->state
= ZSTATE_END
;
1177 stream
->avail_out
-= bytes
;
1178 stream
->total_out
+= bytes
;