]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | /********************************************************************** |
2 | Copyright(c) 2011-2017 Intel Corporation All rights reserved. | |
3 | ||
4 | Redistribution and use in source and binary forms, with or without | |
5 | modification, are permitted provided that the following conditions | |
6 | are met: | |
7 | * Redistributions of source code must retain the above copyright | |
8 | notice, this list of conditions and the following disclaimer. | |
9 | * Redistributions in binary form must reproduce the above copyright | |
10 | notice, this list of conditions and the following disclaimer in | |
11 | the documentation and/or other materials provided with the | |
12 | distribution. | |
13 | * Neither the name of Intel Corporation nor the names of its | |
14 | contributors may be used to endorse or promote products derived | |
15 | from this software without specific prior written permission. | |
16 | ||
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
28 | **********************************************************************/ | |
29 | ||
30 | #include <stdlib.h> | |
31 | #include <stdint.h> | |
32 | #include <stdarg.h> | |
33 | #include <stdio.h> | |
34 | #include <assert.h> | |
35 | #include "rolling_hashx.h" | |
36 | #include "sha256_mb.h" | |
37 | #include "test.h" | |
38 | ||
39 | #define MAX_BUFFER_SIZE 128*1024*1024 | |
40 | #define HASH_POOL_SIZE SHA256_MAX_LANES | |
41 | ||
42 | #ifndef TEST_SEED | |
43 | # define TEST_SEED 0x1234 | |
44 | #endif | |
45 | ||
46 | #define FILTER_BITS 10 | |
47 | #define FILTER_SIZE (1 << FILTER_BITS) | |
48 | #define FILTER_MASK (FILTER_SIZE - 1) | |
49 | ||
50 | #define BITS_TO_INDEX_LONG 6 | |
51 | #define MASK_TO_INDEX_LONG ((1 << BITS_TO_INDEX_LONG) - 1) | |
52 | ||
53 | // Globals | |
54 | SHA256_HASH_CTX ctxpool[SHA256_MAX_LANES], *last_ctx; | |
55 | SHA256_HASH_CTX_MGR mb_hash_mgr; | |
56 | uint64_t filter_table[FILTER_SIZE]; | |
57 | unsigned long chunks_created = 0; | |
58 | unsigned long filter_hits = 0; | |
59 | ||
60 | // Example function to run on each chunk | |
61 | ||
62 | void run_fragment(SHA256_HASH_CTX * ctx) | |
63 | { | |
64 | uint64_t lookup, set_hash; | |
65 | unsigned int lookup_hash; | |
66 | uint32_t idx; | |
67 | ||
68 | chunks_created++; | |
69 | ||
70 | // Run a simple lookup filter on chunk using digest | |
71 | lookup_hash = ctx->job.result_digest[0] & FILTER_MASK; | |
72 | lookup = filter_table[lookup_hash]; | |
73 | ||
74 | idx = ctx->job.result_digest[1]; | |
75 | ||
76 | set_hash = 1 << (idx & MASK_TO_INDEX_LONG) | | |
77 | 1 << ((idx >> BITS_TO_INDEX_LONG) & MASK_TO_INDEX_LONG) | | |
78 | 1 << ((idx >> (2 * BITS_TO_INDEX_LONG)) & MASK_TO_INDEX_LONG); | |
79 | ||
80 | if ((lookup & set_hash) == set_hash) | |
81 | filter_hits++; | |
82 | else | |
83 | filter_table[lookup_hash] = lookup | set_hash; | |
84 | } | |
85 | ||
86 | void setup_chunk_processing(void) | |
87 | { | |
88 | int i; | |
89 | ||
90 | sha256_ctx_mgr_init(&mb_hash_mgr); | |
91 | ||
92 | for (i = 0; i < HASH_POOL_SIZE; i++) | |
93 | hash_ctx_init(&ctxpool[i]); | |
94 | ||
95 | last_ctx = &ctxpool[0]; | |
96 | } | |
97 | ||
98 | SHA256_HASH_CTX *get_next_job_ctx(void) | |
99 | { | |
100 | int i; | |
101 | SHA256_HASH_CTX *ctx; | |
102 | ||
103 | if (last_ctx && hash_ctx_complete(last_ctx)) | |
104 | return last_ctx; | |
105 | ||
106 | for (i = 0; i < HASH_POOL_SIZE; i++) { | |
107 | if (hash_ctx_complete(&ctxpool[i])) | |
108 | return &ctxpool[i]; | |
109 | } | |
110 | ctx = sha256_ctx_mgr_flush(&mb_hash_mgr); | |
111 | assert(ctx != NULL); | |
112 | return ctx; | |
113 | } | |
114 | ||
115 | void put_next_job_ctx(SHA256_HASH_CTX * ctx) | |
116 | { | |
117 | if (ctx && hash_ctx_complete(ctx)) | |
118 | last_ctx = ctx; | |
119 | ||
120 | run_fragment(ctx); | |
121 | } | |
122 | ||
123 | void process_chunk(uint8_t * buff, int len) | |
124 | { | |
125 | SHA256_HASH_CTX *ctx; | |
126 | ||
127 | ctx = get_next_job_ctx(); | |
128 | ctx = sha256_ctx_mgr_submit(&mb_hash_mgr, ctx, buff, len, HASH_ENTIRE); | |
129 | ||
130 | if (ctx) | |
131 | put_next_job_ctx(ctx); | |
132 | } | |
133 | ||
134 | void finish_chunk_processing(void) | |
135 | { | |
136 | SHA256_HASH_CTX *ctx; | |
137 | ||
138 | while ((ctx = sha256_ctx_mgr_flush(&mb_hash_mgr)) != NULL) | |
139 | run_fragment(ctx); | |
140 | } | |
141 | ||
142 | int main(void) | |
143 | { | |
144 | int i, w; | |
145 | uint8_t *buffer, *p; | |
146 | uint32_t mask, trigger, offset = 0; | |
147 | uint32_t min_chunk, max_chunk, mean_chunk; | |
148 | long remain; | |
149 | struct rh_state2 state; | |
150 | struct perf start, stop; | |
151 | ||
152 | // Chunking parameters | |
153 | w = 32; | |
154 | min_chunk = 1024; | |
155 | mean_chunk = 4 * 1024; | |
156 | max_chunk = 32 * 1024; | |
157 | mask = rolling_hashx_mask_gen(mean_chunk, 0); | |
158 | trigger = rand() & mask; | |
159 | ||
160 | printf("chunk and hash test w=%d, min=%d, target_ave=%d, max=%d:\n", w, min_chunk, | |
161 | mean_chunk, max_chunk); | |
162 | ||
163 | if (min_chunk < w || min_chunk > max_chunk) { | |
164 | printf(" Improper parameters selected\n"); | |
165 | return -1; | |
166 | } | |
167 | ||
168 | if ((buffer = malloc(MAX_BUFFER_SIZE)) == NULL) { | |
169 | printf("cannot allocate mem\n"); | |
170 | return -1; | |
171 | } | |
172 | // Initialize buffer with random data | |
173 | srand(TEST_SEED); | |
174 | for (i = 0; i < MAX_BUFFER_SIZE; i++) | |
175 | buffer[i] = rand(); | |
176 | ||
177 | // Start chunking test with multi-buffer hashing of results | |
178 | perf_start(&start); | |
179 | ||
180 | rolling_hash2_init(&state, w); | |
181 | setup_chunk_processing(); | |
182 | ||
183 | p = buffer; | |
184 | remain = MAX_BUFFER_SIZE; | |
185 | ||
186 | while (remain > max_chunk) { | |
187 | // Skip to min chunk | |
188 | rolling_hash2_reset(&state, p + min_chunk - w); | |
189 | rolling_hash2_run(&state, p + min_chunk, max_chunk - min_chunk, | |
190 | mask, trigger, &offset); | |
191 | ||
192 | process_chunk(p, min_chunk + offset); | |
193 | ||
194 | p += offset + min_chunk; | |
195 | remain -= (offset + min_chunk); | |
196 | } | |
197 | ||
198 | while (remain > min_chunk) { | |
199 | rolling_hash2_reset(&state, p + min_chunk - w); | |
200 | rolling_hash2_run(&state, p + min_chunk, remain - min_chunk, | |
201 | mask, trigger, &offset); | |
202 | ||
203 | process_chunk(p, min_chunk + offset); | |
204 | ||
205 | p += offset + min_chunk; | |
206 | remain -= (offset + min_chunk); | |
207 | } | |
208 | ||
209 | if (remain > 0) | |
210 | process_chunk(p, remain); | |
211 | ||
212 | finish_chunk_processing(); | |
213 | perf_stop(&stop); | |
214 | ||
215 | printf("chunking_with_mb_hash: "); | |
216 | perf_print(stop, start, MAX_BUFFER_SIZE); | |
217 | ||
218 | printf(" found %ld chunks, ave_len=%ld, filter hits=%ld\n", chunks_created, | |
219 | MAX_BUFFER_SIZE / chunks_created, filter_hits); | |
220 | ||
221 | return 0; | |
222 | } |