1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2 ; Copyright(c) 2011-2016 Intel Corporation All rights reserved.
4 ; Redistribution and use in source and binary forms, with or without
5 ; modification, are permitted provided that the following conditions
7 ; * Redistributions of source code must retain the above copyright
8 ; notice, this list of conditions and the following disclaimer.
9 ; * Redistributions in binary form must reproduce the above copyright
10 ; notice, this list of conditions and the following disclaimer in
11 ; the documentation and/or other materials provided with the
13 ; * Neither the name of Intel Corporation nor the names of its
14 ; contributors may be used to endorse or promote products derived
15 ; from this software without specific prior written permission.
17 ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 ; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 ; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 ; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 ; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 ; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 ; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 ; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 ; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
29 %include "options.asm"
31 %include "lz0a_const.asm"
32 %include "data_struct2.asm"
33 %include "bitbuf2.asm"
34 %include "huffman.asm"
35 %include "igzip_compare_types.asm"
36 %include "reg_sizes.asm"
40 %define LAST_BYTES_COUNT 3 ; Bytes to prevent reading out of array bounds
41 %define LA_STATELESS 264 ; Max number of bytes read in loop2 rounded up to 8 byte boundary
53 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
54 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
55 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
75 %define file_start rdi
77 %define m_bit_count rbp
99 %define hufftables r15
101 ;; GPR r8 & r15 can be used
103 %define xtmp0 xmm0 ; tmp
104 %define xtmp1 xmm1 ; tmp
106 %define ytmp0 ymm0 ; tmp
107 %define ytmp1 ymm1 ; tmp
110 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
111 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
112 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
115 blen_mem_offset equ 0 ; local variable (8 bytes)
116 f_end_i_mem_offset equ 8
117 gpr_save_mem_offset equ 16 ; gpr save area (8*8 bytes)
118 xmm_save_mem_offset equ 16 + 8*8 ; xmm save area (4*16 bytes) (16 byte aligned)
119 stack_size equ 2*8 + 8*8 + 4*16 + 8
120 ;;; 8 because stack address is odd multiple of 8 after a function call and
121 ;;; we want it aligned to 16 bytes
123 ; void isal_deflate_body_stateless ( isal_zstream *stream )
124 ; arg 1: rcx: addr of stream
125 global isal_deflate_body_stateless_ %+ ARCH
126 isal_deflate_body_stateless_ %+ ARCH %+ :
127 %ifidn __OUTPUT_FORMAT__, elf64
131 ;; do nothing if (avail_in == 0)
132 cmp dword [rcx + _avail_in], 0
146 mov [rsp + gpr_save_mem_offset + 0*8], rbx
147 mov [rsp + gpr_save_mem_offset + 1*8], rsi
148 mov [rsp + gpr_save_mem_offset + 2*8], rdi
149 mov [rsp + gpr_save_mem_offset + 3*8], rbp
150 mov [rsp + gpr_save_mem_offset + 4*8], r12
151 mov [rsp + gpr_save_mem_offset + 5*8], r13
152 mov [rsp + gpr_save_mem_offset + 6*8], r14
153 mov [rsp + gpr_save_mem_offset + 7*8], r15
156 mov dword [stream + _internal_state_has_eob], 0
158 ; state->bitbuf.set_buf(stream->next_out, stream->avail_out);
159 mov m_out_buf, [stream + _next_out]
160 mov [stream + _internal_state_bitbuf_m_out_start], m_out_buf
161 mov tmp1 %+ d, [stream + _avail_out]
166 mov [stream + _internal_state_bitbuf_m_out_end], tmp1
168 mov m_bits, [stream + _internal_state_bitbuf_m_bits]
169 mov m_bit_count %+ d, [stream + _internal_state_bitbuf_m_bit_count]
170 mov hufftables, [stream + _hufftables]
171 ; state->b_bytes_valid = stream->avail_in;
172 mov f_end_i %+ d, [stream + _avail_in]
173 mov [stream + _internal_state_b_bytes_valid], f_end_i %+ d
176 mov file_start, [stream + _next_in]
177 mov [stream + _internal_state_file_start], file_start
180 sub f_end_i, LA_STATELESS
181 mov [rsp + f_end_i_mem_offset], f_end_i
182 ; if (f_end_i <= 0) continue;
186 ; for (f_i = f_start_i; f_i < f_end_i; f_i++) {
187 MARK __stateless_compute_hash_ %+ ARCH
188 mov curr_data %+ d, [file_start + f_i]
190 cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end]
193 ;; Encode first byte in the stream as a literal
194 compute_hash hash, curr_data
195 and hash %+ d, HASH_MASK
196 mov [stream + _internal_state_head + 2 * hash], f_i %+ w
198 get_lit_code curr_data, code2, code_len2, hufftables
205 xor hash2 %+ d, hash2 %+ d
206 crc32 hash2 %+ d, curr_data2 %+ d
208 ; hash = compute_hash(state->file_start + f_i) & HASH_MASK;
209 and hash %+ d, HASH_MASK
210 and hash2 %+ d, HASH_MASK
212 ; if (state->bitbuf.is_full()) {
213 cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end]
220 lea tmp1, [file_start + f_i]
223 mov dist %+ w, f_i %+ w
224 sub dist %+ w, word [stream + _internal_state_head + 2 * hash]
226 ; state->head[hash] = (uint16_t) f_i;
227 mov [stream + _internal_state_head + 2 * hash], f_i %+ w
231 mov dist2 %+ w, f_i %+ w
232 sub dist2 %+ w, word [stream + _internal_state_head + 2 * hash2]
235 ; state->head[hash2] = (uint16_t) f_i;
236 mov [stream + _internal_state_head + 2 * hash2], f_i %+ w
242 ; if ((dist-1) < (D-1)) {
248 cmp dist2 %+ d, (D-1)
252 MARK __stateless_compare_ %+ ARCH
253 ; len = compare258(state->file_start + f_i,
254 ; state->file_start + f_i - dist);
256 ;; Specutively load distance code (except for when large windows are used)
257 get_packed_dist_code dist, code2, hufftables
259 ;; Check for long len/dist match (>7) with first literal
269 lea tmp1, [file_start + f_i]
273 ;; Specutively load distance code (except for when large windows are used)
274 get_packed_dist_code dist2, code4, hufftables
276 ;; Check for len/dist match (>7) with second literal
282 ;; Check for len/dist match for first literal
284 jz len_dist_lit_huffman_pre
288 jmp len_dist_huffman_pre
291 MARK __stateless_len_dist_lit_huffman_ %+ ARCH
292 len_dist_lit_huffman_pre:
293 movzx tmp1, curr_data %+ b
294 get_lit_code tmp1, code3, code_len3, hufftables
296 ;; Specutively load the code for the first literal
297 movzx tmp1, curr_data %+ b
298 get_lit_code tmp1, code3, rcx, hufftables
300 ;; Check for len/dist match for first literal
302 jz len_dist_huffman_pre
304 ;; Specutively load the code for the second literal
307 get_lit_code curr_data, code2, code_len2, hufftables
313 ;; Check for len/dist match for second literal
317 MARK __stateless_len_dist_lit_huffman_ %+ ARCH
318 len_dist_lit_huffman_pre:
325 len_dist_lit_huffman:
326 %ifndef LONGER_HUFFTABLE
328 get_dist_code tmp4, code4, code_len2, hufftables ;; clobbers dist, rcx
330 unpack_dist_code code4, code_len2
332 get_len_code len2, code, rcx, hufftables ;; rcx is code_len
335 shlx code4, code4, rcx
345 shlx code4, code4, rcx
353 ;; Setup for updating hash
354 lea tmp3, [f_i + 1] ; tmp3 <= k
357 ; hash = compute_hash(state->file_start + k) & HASH_MASK;
358 mov tmp5 %+ d, [file_start + tmp3]
362 compute_hash hash, tmp5
363 and hash %+ d, HASH_MASK
365 ; state->head[hash] = k;
366 mov [stream + _internal_state_head + 2 * hash], tmp3 %+ w
370 jmp update_hash_for_symbol
371 ;; encode as dist/len
373 MARK __stateless_len_dist_huffman_ %+ ARCH
374 len_dist_huffman_pre:
381 ; get_dist_code(dist, &code2, &code_len2);
382 %ifndef LONGER_HUFFTABLE
383 mov tmp3, dist ; since code2 and dist are rbx
384 get_dist_code tmp3, code2, code_len2, hufftables ;; clobbers dist, rcx
386 unpack_dist_code code2, code_len2
388 ; get_len_code(len, &code, &code_len);
389 get_len_code len, code, rcx, hufftables ;; rcx is code_len
393 ; code_len2 += code_len
395 shlx code2, code2, rcx
402 ;; Setup for updateing hash
403 lea tmp3, [f_i + 2] ; tmp3 <= k
405 mov tmp7 %+ d, [file_start + tmp3]
407 MARK __stateless_update_hash_for_symbol_ %+ ARCH
408 update_hash_for_symbol:
409 mov curr_data %+ d, [file_start + f_i]
410 mov curr_data2, curr_data
411 compute_hash hash, curr_data
412 %ifdef LIMIT_HASH_UPDATE
413 ; only update hash twice, first hash was already calculated.
415 ; hash = compute_hash(state->file_start + k) & HASH_MASK;
416 compute_hash hash2, tmp7
417 and hash2 %+ d, HASH_MASK
418 ; state->head[hash] = k;
419 mov [stream + _internal_state_head + 2 * hash2], tmp3 %+ w
423 ; hash = compute_hash(state->file_start + k) & HASH_MASK;
424 mov tmp7 %+ d, [file_start + tmp3]
425 compute_hash hash2, tmp7
426 and hash2 %+ d, HASH_MASK
427 ; state->head[hash] = k;
428 mov [stream + _internal_state_head + 2 * hash2], tmp3 %+ w
435 MARK __stateless_write_len_dist_bits_ %+ ARCH
436 mov f_end_i, [rsp + f_end_i_mem_offset]
437 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
445 MARK __stateless_write_lit_bits_ %+ ARCH
448 movzx tmp1, curr_data %+ b
449 get_lit_code tmp1, code3, rcx, hufftables
453 get_lit_code curr_data, code2, code_len2, hufftables
455 ;; Calculate code associated with both literals
456 shlx code2, code2, rcx
461 mov f_end_i, [rsp + f_end_i_mem_offset]
463 mov curr_data %+ d, [file_start + f_i]
464 mov curr_data2, curr_data
466 compute_hash hash, curr_data
468 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
474 MARK __stateless_end_loops_ %+ ARCH
476 ;; Handle the last bytes (at most LA_statless bytes)
477 add f_end_i, LA_STATELESS - LAST_BYTES_COUNT
479 jge end_loop_2_finish
482 ;; Check for space in out buffer
483 cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end]
486 mov curr_data %+ d, [file_start + f_i]
487 compute_hash hash, curr_data
488 and hash %+ d, HASH_MASK
490 ;; Calculate possible distance for length/dist pair.
492 mov dist %+ w, f_i %+ w
493 sub dist %+ w, word [stream + _internal_state_head + 2 * hash]
494 mov [stream + _internal_state_head + 2 * hash], f_i %+ w
496 ;; Check if look back distance is valid (the dec is to handle when dist = 0)
499 jae encode_literal_finish
502 ;; Check if look back distance is a match
503 lea tmp6, [f_end_i + LAST_BYTES_COUNT]
505 lea tmp1, [file_start + f_i]
508 compare tmp6, tmp1, tmp2, len, tmp3
510 ;; Limit len to maximum value of 258
514 cmp len, SHORTEST_MATCH
515 jb encode_literal_finish
517 ;; Encode len/dist pair
518 %ifndef LONGER_HUFFTABLE
520 get_dist_code tmp3, code2, code_len2, hufftables ;; clobbers dist, rcx
522 get_dist_code dist, code2, code_len2, hufftables ;; clobbers dist, rcx
524 get_len_code len, code, rcx, hufftables ;; rcx is code_len
526 ;; Combine length and distance code for writing it out
528 shlx code2, code2, rcx
534 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
536 ;; Setup for next loop
540 jmp end_loop_2_finish
542 encode_literal_finish:
544 and curr_data %+ d, 0xFF
545 get_lit_code curr_data, code2, code_len2, hufftables
546 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
548 ;; Setup for next loop
553 cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end]
556 ;; Check if any bytes left (at most LAST_BYTES_COUNT bytes)
557 add f_end_i, LAST_BYTES_COUNT
561 ;; Handle encoding last few bytes by encoding them as literals
562 xor curr_data, curr_data
564 movzx curr_data, byte [file_start + f_i]
565 get_lit_code curr_data, code2, code_len2, hufftables
566 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
568 cmp m_out_buf, [stream + _internal_state_bitbuf_m_out_end]
576 ;; Write out end of block
577 get_lit_code 256, code2, code_len2, hufftables
578 write_bits m_bits, m_bit_count, code2, code_len2, m_out_buf, tmp3
579 mov dword [stream + _internal_state_has_eob], 1
582 ;; update input buffer
583 add [stream + _total_in], f_i %+ d
584 add [stream + _next_in], f_i
585 sub [stream + _avail_in], f_i %+ d
587 ;; update output buffer
588 mov [stream + _next_out], m_out_buf
589 sub m_out_buf, [stream + _internal_state_bitbuf_m_out_start]
590 sub [stream + _avail_out], m_out_buf %+ d
591 add [stream + _total_out], m_out_buf %+ d
593 mov [stream + _internal_state_bitbuf_m_bits], m_bits
594 mov [stream + _internal_state_bitbuf_m_bit_count], m_bit_count %+ d
596 mov rbx, [rsp + gpr_save_mem_offset + 0*8]
597 mov rsi, [rsp + gpr_save_mem_offset + 1*8]
598 mov rdi, [rsp + gpr_save_mem_offset + 2*8]
599 mov rbp, [rsp + gpr_save_mem_offset + 3*8]
600 mov r12, [rsp + gpr_save_mem_offset + 4*8]
601 mov r13, [rsp + gpr_save_mem_offset + 5*8]
602 mov r14, [rsp + gpr_save_mem_offset + 6*8]
603 mov r15, [rsp + gpr_save_mem_offset + 7*8]
613 MARK __stateless_compare_loops_ %+ ARCH
615 %if (COMPARE_TYPE == 1)
616 compare250 tmp1, tmp2, len, tmp3
617 %elif (COMPARE_TYPE == 2)
618 compare250_x tmp1, tmp2, len, tmp3, xtmp0, xtmp1
619 %elif (COMPARE_TYPE == 3)
620 compare250_y tmp1, tmp2, len, tmp3, ytmp0, ytmp1
622 %error Unknown Compare type COMPARE_TYPE
628 %if (COMPARE_TYPE == 1)
629 compare250 tmp1, tmp2, len2, tmp3
630 %elif (COMPARE_TYPE == 2)
631 compare250_x tmp1, tmp2, len2, tmp3, xtmp0, xtmp1
632 %elif (COMPARE_TYPE == 3)
633 compare250_y tmp1, tmp2, len2, tmp3, ytmp0, ytmp1
635 %error Unknown Compare type COMPARE_TYPE
639 get_lit_code curr_data, code3, code_len3, hufftables
640 jmp len_dist_lit_huffman