]>
Commit | Line | Data |
---|---|---|
11b7501a SB |
1 | /* Copyright 2015 Google Inc. All Rights Reserved.\r |
2 | \r | |
3 | Distributed under MIT license.\r | |
4 | See file LICENSE for detail or copy at https://opensource.org/licenses/MIT\r | |
5 | */\r | |
6 | \r | |
7 | /* Algorithms for distributing the literals and commands of a metablock between\r | |
8 | block types and contexts. */\r | |
9 | \r | |
10 | #include "./metablock.h"\r | |
11 | \r | |
12 | #include "../common/constants.h"\r | |
13 | #include "../common/types.h"\r | |
14 | #include "./bit_cost.h"\r | |
15 | #include "./block_splitter.h"\r | |
16 | #include "./cluster.h"\r | |
17 | #include "./context.h"\r | |
18 | #include "./entropy_encode.h"\r | |
19 | #include "./histogram.h"\r | |
20 | #include "./memory.h"\r | |
21 | #include "./port.h"\r | |
22 | #include "./quality.h"\r | |
23 | \r | |
24 | #if defined(__cplusplus) || defined(c_plusplus)\r | |
25 | extern "C" {\r | |
26 | #endif\r | |
27 | \r | |
28 | void BrotliBuildMetaBlock(MemoryManager* m,\r | |
29 | const uint8_t* ringbuffer,\r | |
30 | const size_t pos,\r | |
31 | const size_t mask,\r | |
32 | const BrotliEncoderParams* params,\r | |
33 | uint8_t prev_byte,\r | |
34 | uint8_t prev_byte2,\r | |
35 | const Command* cmds,\r | |
36 | size_t num_commands,\r | |
37 | ContextType literal_context_mode,\r | |
38 | MetaBlockSplit* mb) {\r | |
39 | /* Histogram ids need to fit in one byte. */\r | |
40 | static const size_t kMaxNumberOfHistograms = 256;\r | |
41 | HistogramDistance* distance_histograms;\r | |
42 | HistogramLiteral* literal_histograms;\r | |
43 | ContextType* literal_context_modes;\r | |
44 | size_t num_literal_contexts;\r | |
45 | size_t num_distance_contexts;\r | |
46 | size_t i;\r | |
47 | \r | |
48 | BrotliSplitBlock(m, cmds, num_commands,\r | |
49 | ringbuffer, pos, mask, params,\r | |
50 | &mb->literal_split,\r | |
51 | &mb->command_split,\r | |
52 | &mb->distance_split);\r | |
53 | if (BROTLI_IS_OOM(m)) return;\r | |
54 | \r | |
55 | literal_context_modes =\r | |
56 | BROTLI_ALLOC(m, ContextType, mb->literal_split.num_types);\r | |
57 | if (BROTLI_IS_OOM(m)) return;\r | |
58 | for (i = 0; i < mb->literal_split.num_types; ++i) {\r | |
59 | literal_context_modes[i] = literal_context_mode;\r | |
60 | }\r | |
61 | \r | |
62 | num_literal_contexts =\r | |
63 | mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;\r | |
64 | num_distance_contexts =\r | |
65 | mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;\r | |
66 | literal_histograms = BROTLI_ALLOC(m, HistogramLiteral, num_literal_contexts);\r | |
67 | if (BROTLI_IS_OOM(m)) return;\r | |
68 | ClearHistogramsLiteral(literal_histograms, num_literal_contexts);\r | |
69 | \r | |
70 | assert(mb->command_histograms == 0);\r | |
71 | mb->command_histograms_size = mb->command_split.num_types;\r | |
72 | mb->command_histograms =\r | |
73 | BROTLI_ALLOC(m, HistogramCommand, mb->command_histograms_size);\r | |
74 | if (BROTLI_IS_OOM(m)) return;\r | |
75 | ClearHistogramsCommand(mb->command_histograms, mb->command_histograms_size);\r | |
76 | distance_histograms =\r | |
77 | BROTLI_ALLOC(m, HistogramDistance, num_distance_contexts);\r | |
78 | if (BROTLI_IS_OOM(m)) return;\r | |
79 | ClearHistogramsDistance(distance_histograms, num_distance_contexts);\r | |
80 | BrotliBuildHistogramsWithContext(cmds, num_commands,\r | |
81 | &mb->literal_split, &mb->command_split, &mb->distance_split,\r | |
82 | ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes,\r | |
83 | literal_histograms, mb->command_histograms, distance_histograms);\r | |
84 | BROTLI_FREE(m, literal_context_modes);\r | |
85 | \r | |
86 | assert(mb->literal_context_map == 0);\r | |
87 | mb->literal_context_map_size =\r | |
88 | mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;\r | |
89 | mb->literal_context_map =\r | |
90 | BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);\r | |
91 | if (BROTLI_IS_OOM(m)) return;\r | |
92 | assert(mb->literal_histograms == 0);\r | |
93 | mb->literal_histograms_size = mb->literal_context_map_size;\r | |
94 | mb->literal_histograms =\r | |
95 | BROTLI_ALLOC(m, HistogramLiteral, mb->literal_histograms_size);\r | |
96 | if (BROTLI_IS_OOM(m)) return;\r | |
97 | BrotliClusterHistogramsLiteral(m, literal_histograms,\r | |
98 | mb->literal_context_map_size,\r | |
99 | kMaxNumberOfHistograms,\r | |
100 | mb->literal_histograms,\r | |
101 | &mb->literal_histograms_size,\r | |
102 | mb->literal_context_map);\r | |
103 | if (BROTLI_IS_OOM(m)) return;\r | |
104 | BROTLI_FREE(m, literal_histograms);\r | |
105 | \r | |
106 | assert(mb->distance_context_map == 0);\r | |
107 | mb->distance_context_map_size =\r | |
108 | mb->distance_split.num_types << BROTLI_DISTANCE_CONTEXT_BITS;\r | |
109 | mb->distance_context_map =\r | |
110 | BROTLI_ALLOC(m, uint32_t, mb->distance_context_map_size);\r | |
111 | if (BROTLI_IS_OOM(m)) return;\r | |
112 | assert(mb->distance_histograms == 0);\r | |
113 | mb->distance_histograms_size = mb->distance_context_map_size;\r | |
114 | mb->distance_histograms =\r | |
115 | BROTLI_ALLOC(m, HistogramDistance, mb->distance_histograms_size);\r | |
116 | if (BROTLI_IS_OOM(m)) return;\r | |
117 | BrotliClusterHistogramsDistance(m, distance_histograms,\r | |
118 | mb->distance_context_map_size,\r | |
119 | kMaxNumberOfHistograms,\r | |
120 | mb->distance_histograms,\r | |
121 | &mb->distance_histograms_size,\r | |
122 | mb->distance_context_map);\r | |
123 | if (BROTLI_IS_OOM(m)) return;\r | |
124 | BROTLI_FREE(m, distance_histograms);\r | |
125 | }\r | |
126 | \r | |
127 | #define FN(X) X ## Literal\r | |
128 | #include "./metablock_inc.h" /* NOLINT(build/include) */\r | |
129 | #undef FN\r | |
130 | \r | |
131 | #define FN(X) X ## Command\r | |
132 | #include "./metablock_inc.h" /* NOLINT(build/include) */\r | |
133 | #undef FN\r | |
134 | \r | |
135 | #define FN(X) X ## Distance\r | |
136 | #include "./metablock_inc.h" /* NOLINT(build/include) */\r | |
137 | #undef FN\r | |
138 | \r | |
139 | void BrotliBuildMetaBlockGreedy(MemoryManager* m,\r | |
140 | const uint8_t* ringbuffer,\r | |
141 | size_t pos,\r | |
142 | size_t mask,\r | |
143 | const Command *commands,\r | |
144 | size_t n_commands,\r | |
145 | MetaBlockSplit* mb) {\r | |
146 | BlockSplitterLiteral lit_blocks;\r | |
147 | BlockSplitterCommand cmd_blocks;\r | |
148 | BlockSplitterDistance dist_blocks;\r | |
149 | size_t num_literals = 0;\r | |
150 | size_t i;\r | |
151 | for (i = 0; i < n_commands; ++i) {\r | |
152 | num_literals += commands[i].insert_len_;\r | |
153 | }\r | |
154 | \r | |
155 | InitBlockSplitterLiteral(m, &lit_blocks, 256, 512, 400.0, num_literals,\r | |
156 | &mb->literal_split, &mb->literal_histograms,\r | |
157 | &mb->literal_histograms_size);\r | |
158 | if (BROTLI_IS_OOM(m)) return;\r | |
159 | InitBlockSplitterCommand(m, &cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS, 1024,\r | |
160 | 500.0, n_commands, &mb->command_split, &mb->command_histograms,\r | |
161 | &mb->command_histograms_size);\r | |
162 | if (BROTLI_IS_OOM(m)) return;\r | |
163 | InitBlockSplitterDistance(m, &dist_blocks, 64, 512, 100.0, n_commands,\r | |
164 | &mb->distance_split, &mb->distance_histograms,\r | |
165 | &mb->distance_histograms_size);\r | |
166 | if (BROTLI_IS_OOM(m)) return;\r | |
167 | \r | |
168 | for (i = 0; i < n_commands; ++i) {\r | |
169 | const Command cmd = commands[i];\r | |
170 | size_t j;\r | |
171 | BlockSplitterAddSymbolCommand(&cmd_blocks, cmd.cmd_prefix_);\r | |
172 | for (j = cmd.insert_len_; j != 0; --j) {\r | |
173 | BlockSplitterAddSymbolLiteral(&lit_blocks, ringbuffer[pos & mask]);\r | |
174 | ++pos;\r | |
175 | }\r | |
176 | pos += CommandCopyLen(&cmd);\r | |
177 | if (CommandCopyLen(&cmd) && cmd.cmd_prefix_ >= 128) {\r | |
178 | BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_);\r | |
179 | }\r | |
180 | }\r | |
181 | \r | |
182 | BlockSplitterFinishBlockLiteral(&lit_blocks, /* is_final = */ BROTLI_TRUE);\r | |
183 | BlockSplitterFinishBlockCommand(&cmd_blocks, /* is_final = */ BROTLI_TRUE);\r | |
184 | BlockSplitterFinishBlockDistance(&dist_blocks, /* is_final = */ BROTLI_TRUE);\r | |
185 | }\r | |
186 | \r | |
187 | /* Greedy block splitter for one block category (literal, command or distance).\r | |
188 | Gathers histograms for all context buckets. */\r | |
189 | typedef struct ContextBlockSplitter {\r | |
190 | /* Alphabet size of particular block category. */\r | |
191 | size_t alphabet_size_;\r | |
192 | size_t num_contexts_;\r | |
193 | size_t max_block_types_;\r | |
194 | /* We collect at least this many symbols for each block. */\r | |
195 | size_t min_block_size_;\r | |
196 | /* We merge histograms A and B if\r | |
197 | entropy(A+B) < entropy(A) + entropy(B) + split_threshold_,\r | |
198 | where A is the current histogram and B is the histogram of the last or the\r | |
199 | second last block type. */\r | |
200 | double split_threshold_;\r | |
201 | \r | |
202 | size_t num_blocks_;\r | |
203 | BlockSplit* split_; /* not owned */\r | |
204 | HistogramLiteral* histograms_; /* not owned */\r | |
205 | size_t* histograms_size_; /* not owned */\r | |
206 | \r | |
207 | /* The number of symbols that we want to collect before deciding on whether\r | |
208 | or not to merge the block with a previous one or emit a new block. */\r | |
209 | size_t target_block_size_;\r | |
210 | /* The number of symbols in the current histogram. */\r | |
211 | size_t block_size_;\r | |
212 | /* Offset of the current histogram. */\r | |
213 | size_t curr_histogram_ix_;\r | |
214 | /* Offset of the histograms of the previous two block types. */\r | |
215 | size_t last_histogram_ix_[2];\r | |
216 | /* Entropy of the previous two block types. */\r | |
217 | double* last_entropy_;\r | |
218 | /* The number of times we merged the current block with the last one. */\r | |
219 | size_t merge_last_count_;\r | |
220 | } ContextBlockSplitter;\r | |
221 | \r | |
222 | static void InitContextBlockSplitter(\r | |
223 | MemoryManager* m, ContextBlockSplitter* self, size_t alphabet_size,\r | |
224 | size_t num_contexts, size_t min_block_size, double split_threshold,\r | |
225 | size_t num_symbols, BlockSplit* split, HistogramLiteral** histograms,\r | |
226 | size_t* histograms_size) {\r | |
227 | size_t max_num_blocks = num_symbols / min_block_size + 1;\r | |
228 | size_t max_num_types;\r | |
229 | \r | |
230 | self->alphabet_size_ = alphabet_size;\r | |
231 | self->num_contexts_ = num_contexts;\r | |
232 | self->max_block_types_ = BROTLI_MAX_NUMBER_OF_BLOCK_TYPES / num_contexts;\r | |
233 | self->min_block_size_ = min_block_size;\r | |
234 | self->split_threshold_ = split_threshold;\r | |
235 | self->num_blocks_ = 0;\r | |
236 | self->split_ = split;\r | |
237 | self->histograms_size_ = histograms_size;\r | |
238 | self->target_block_size_ = min_block_size;\r | |
239 | self->block_size_ = 0;\r | |
240 | self->curr_histogram_ix_ = 0;\r | |
241 | self->merge_last_count_ = 0;\r | |
242 | \r | |
243 | /* We have to allocate one more histogram than the maximum number of block\r | |
244 | types for the current histogram when the meta-block is too big. */\r | |
245 | max_num_types =\r | |
246 | BROTLI_MIN(size_t, max_num_blocks, self->max_block_types_ + 1);\r | |
247 | BROTLI_ENSURE_CAPACITY(m, uint8_t,\r | |
248 | split->types, split->types_alloc_size, max_num_blocks);\r | |
249 | BROTLI_ENSURE_CAPACITY(m, uint32_t,\r | |
250 | split->lengths, split->lengths_alloc_size, max_num_blocks);\r | |
251 | if (BROTLI_IS_OOM(m)) return;\r | |
252 | split->num_blocks = max_num_blocks;\r | |
253 | self->last_entropy_ = BROTLI_ALLOC(m, double, 2 * num_contexts);\r | |
254 | if (BROTLI_IS_OOM(m)) return;\r | |
255 | assert(*histograms == 0);\r | |
256 | *histograms_size = max_num_types * num_contexts;\r | |
257 | *histograms = BROTLI_ALLOC(m, HistogramLiteral, *histograms_size);\r | |
258 | self->histograms_ = *histograms;\r | |
259 | if (BROTLI_IS_OOM(m)) return;\r | |
260 | /* Clear only current historgram. */\r | |
261 | ClearHistogramsLiteral(&self->histograms_[0], num_contexts);\r | |
262 | self->last_histogram_ix_[0] = self->last_histogram_ix_[1] = 0;\r | |
263 | }\r | |
264 | \r | |
265 | static void CleanupContextBlockSplitter(\r | |
266 | MemoryManager* m, ContextBlockSplitter* self) {\r | |
267 | BROTLI_FREE(m, self->last_entropy_);\r | |
268 | }\r | |
269 | \r | |
270 | /* Does either of three things:\r | |
271 | (1) emits the current block with a new block type;\r | |
272 | (2) emits the current block with the type of the second last block;\r | |
273 | (3) merges the current block with the last block. */\r | |
274 | static void ContextBlockSplitterFinishBlock(\r | |
275 | MemoryManager* m, ContextBlockSplitter* self, BROTLI_BOOL is_final) {\r | |
276 | BlockSplit* split = self->split_;\r | |
277 | const size_t num_contexts = self->num_contexts_;\r | |
278 | double* last_entropy = self->last_entropy_;\r | |
279 | HistogramLiteral* histograms = self->histograms_;\r | |
280 | \r | |
281 | if (self->block_size_ < self->min_block_size_) {\r | |
282 | self->block_size_ = self->min_block_size_;\r | |
283 | }\r | |
284 | if (self->num_blocks_ == 0) {\r | |
285 | size_t i;\r | |
286 | /* Create first block. */\r | |
287 | split->lengths[0] = (uint32_t)self->block_size_;\r | |
288 | split->types[0] = 0;\r | |
289 | \r | |
290 | for (i = 0; i < num_contexts; ++i) {\r | |
291 | last_entropy[i] =\r | |
292 | BitsEntropy(histograms[i].data_, self->alphabet_size_);\r | |
293 | last_entropy[num_contexts + i] = last_entropy[i];\r | |
294 | }\r | |
295 | ++self->num_blocks_;\r | |
296 | ++split->num_types;\r | |
297 | self->curr_histogram_ix_ += num_contexts;\r | |
298 | if (self->curr_histogram_ix_ < *self->histograms_size_) {\r | |
299 | ClearHistogramsLiteral(\r | |
300 | &self->histograms_[self->curr_histogram_ix_], self->num_contexts_);\r | |
301 | }\r | |
302 | self->block_size_ = 0;\r | |
303 | } else if (self->block_size_ > 0) {\r | |
304 | /* Try merging the set of histograms for the current block type with the\r | |
305 | respective set of histograms for the last and second last block types.\r | |
306 | Decide over the split based on the total reduction of entropy across\r | |
307 | all contexts. */\r | |
308 | double* entropy = BROTLI_ALLOC(m, double, num_contexts);\r | |
309 | HistogramLiteral* combined_histo =\r | |
310 | BROTLI_ALLOC(m, HistogramLiteral, 2 * num_contexts);\r | |
311 | double* combined_entropy = BROTLI_ALLOC(m, double, 2 * num_contexts);\r | |
312 | double diff[2] = { 0.0 };\r | |
313 | size_t i;\r | |
314 | if (BROTLI_IS_OOM(m)) return;\r | |
315 | for (i = 0; i < num_contexts; ++i) {\r | |
316 | size_t curr_histo_ix = self->curr_histogram_ix_ + i;\r | |
317 | size_t j;\r | |
318 | entropy[i] = BitsEntropy(histograms[curr_histo_ix].data_,\r | |
319 | self->alphabet_size_);\r | |
320 | for (j = 0; j < 2; ++j) {\r | |
321 | size_t jx = j * num_contexts + i;\r | |
322 | size_t last_histogram_ix = self->last_histogram_ix_[j] + i;\r | |
323 | combined_histo[jx] = histograms[curr_histo_ix];\r | |
324 | HistogramAddHistogramLiteral(&combined_histo[jx],\r | |
325 | &histograms[last_histogram_ix]);\r | |
326 | combined_entropy[jx] = BitsEntropy(\r | |
327 | &combined_histo[jx].data_[0], self->alphabet_size_);\r | |
328 | diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx];\r | |
329 | }\r | |
330 | }\r | |
331 | \r | |
332 | if (split->num_types < self->max_block_types_ &&\r | |
333 | diff[0] > self->split_threshold_ &&\r | |
334 | diff[1] > self->split_threshold_) {\r | |
335 | /* Create new block. */\r | |
336 | split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;\r | |
337 | split->types[self->num_blocks_] = (uint8_t)split->num_types;\r | |
338 | self->last_histogram_ix_[1] = self->last_histogram_ix_[0];\r | |
339 | self->last_histogram_ix_[0] = split->num_types * num_contexts;\r | |
340 | for (i = 0; i < num_contexts; ++i) {\r | |
341 | last_entropy[num_contexts + i] = last_entropy[i];\r | |
342 | last_entropy[i] = entropy[i];\r | |
343 | }\r | |
344 | ++self->num_blocks_;\r | |
345 | ++split->num_types;\r | |
346 | self->curr_histogram_ix_ += num_contexts;\r | |
347 | if (self->curr_histogram_ix_ < *self->histograms_size_) {\r | |
348 | ClearHistogramsLiteral(\r | |
349 | &self->histograms_[self->curr_histogram_ix_], self->num_contexts_);\r | |
350 | }\r | |
351 | self->block_size_ = 0;\r | |
352 | self->merge_last_count_ = 0;\r | |
353 | self->target_block_size_ = self->min_block_size_;\r | |
354 | } else if (diff[1] < diff[0] - 20.0) {\r | |
355 | /* Combine this block with second last block. */\r | |
356 | split->lengths[self->num_blocks_] = (uint32_t)self->block_size_;\r | |
357 | split->types[self->num_blocks_] = split->types[self->num_blocks_ - 2];\r | |
358 | BROTLI_SWAP(size_t, self->last_histogram_ix_, 0, 1);\r | |
359 | for (i = 0; i < num_contexts; ++i) {\r | |
360 | histograms[self->last_histogram_ix_[0] + i] =\r | |
361 | combined_histo[num_contexts + i];\r | |
362 | last_entropy[num_contexts + i] = last_entropy[i];\r | |
363 | last_entropy[i] = combined_entropy[num_contexts + i];\r | |
364 | HistogramClearLiteral(&histograms[self->curr_histogram_ix_ + i]);\r | |
365 | }\r | |
366 | ++self->num_blocks_;\r | |
367 | self->block_size_ = 0;\r | |
368 | self->merge_last_count_ = 0;\r | |
369 | self->target_block_size_ = self->min_block_size_;\r | |
370 | } else {\r | |
371 | /* Combine this block with last block. */\r | |
372 | split->lengths[self->num_blocks_ - 1] += (uint32_t)self->block_size_;\r | |
373 | for (i = 0; i < num_contexts; ++i) {\r | |
374 | histograms[self->last_histogram_ix_[0] + i] = combined_histo[i];\r | |
375 | last_entropy[i] = combined_entropy[i];\r | |
376 | if (split->num_types == 1) {\r | |
377 | last_entropy[num_contexts + i] = last_entropy[i];\r | |
378 | }\r | |
379 | HistogramClearLiteral(&histograms[self->curr_histogram_ix_ + i]);\r | |
380 | }\r | |
381 | self->block_size_ = 0;\r | |
382 | if (++self->merge_last_count_ > 1) {\r | |
383 | self->target_block_size_ += self->min_block_size_;\r | |
384 | }\r | |
385 | }\r | |
386 | BROTLI_FREE(m, combined_entropy);\r | |
387 | BROTLI_FREE(m, combined_histo);\r | |
388 | BROTLI_FREE(m, entropy);\r | |
389 | }\r | |
390 | if (is_final) {\r | |
391 | *self->histograms_size_ = split->num_types * num_contexts;\r | |
392 | split->num_blocks = self->num_blocks_;\r | |
393 | }\r | |
394 | }\r | |
395 | \r | |
396 | /* Adds the next symbol to the current block type and context. When the\r | |
397 | current block reaches the target size, decides on merging the block. */\r | |
398 | static void ContextBlockSplitterAddSymbol(MemoryManager* m,\r | |
399 | ContextBlockSplitter* self, size_t symbol, size_t context) {\r | |
400 | HistogramAddLiteral(&self->histograms_[self->curr_histogram_ix_ + context],\r | |
401 | symbol);\r | |
402 | ++self->block_size_;\r | |
403 | if (self->block_size_ == self->target_block_size_) {\r | |
404 | ContextBlockSplitterFinishBlock(m, self, /* is_final = */ BROTLI_FALSE);\r | |
405 | if (BROTLI_IS_OOM(m)) return;\r | |
406 | }\r | |
407 | }\r | |
408 | \r | |
409 | void BrotliBuildMetaBlockGreedyWithContexts(MemoryManager* m,\r | |
410 | const uint8_t* ringbuffer,\r | |
411 | size_t pos,\r | |
412 | size_t mask,\r | |
413 | uint8_t prev_byte,\r | |
414 | uint8_t prev_byte2,\r | |
415 | ContextType literal_context_mode,\r | |
416 | size_t num_contexts,\r | |
417 | const uint32_t* static_context_map,\r | |
418 | const Command *commands,\r | |
419 | size_t n_commands,\r | |
420 | MetaBlockSplit* mb) {\r | |
421 | ContextBlockSplitter lit_blocks;\r | |
422 | BlockSplitterCommand cmd_blocks;\r | |
423 | BlockSplitterDistance dist_blocks;\r | |
424 | size_t num_literals = 0;\r | |
425 | size_t i;\r | |
426 | for (i = 0; i < n_commands; ++i) {\r | |
427 | num_literals += commands[i].insert_len_;\r | |
428 | }\r | |
429 | \r | |
430 | InitContextBlockSplitter(m, &lit_blocks, 256, num_contexts, 512, 400.0,\r | |
431 | num_literals, &mb->literal_split, &mb->literal_histograms,\r | |
432 | &mb->literal_histograms_size);\r | |
433 | if (BROTLI_IS_OOM(m)) return;\r | |
434 | InitBlockSplitterCommand(m, &cmd_blocks, BROTLI_NUM_COMMAND_SYMBOLS, 1024,\r | |
435 | 500.0, n_commands, &mb->command_split, &mb->command_histograms,\r | |
436 | &mb->command_histograms_size);\r | |
437 | if (BROTLI_IS_OOM(m)) return;\r | |
438 | InitBlockSplitterDistance(m, &dist_blocks, 64, 512, 100.0, n_commands,\r | |
439 | &mb->distance_split, &mb->distance_histograms,\r | |
440 | &mb->distance_histograms_size);\r | |
441 | if (BROTLI_IS_OOM(m)) return;\r | |
442 | \r | |
443 | for (i = 0; i < n_commands; ++i) {\r | |
444 | const Command cmd = commands[i];\r | |
445 | size_t j;\r | |
446 | BlockSplitterAddSymbolCommand(&cmd_blocks, cmd.cmd_prefix_);\r | |
447 | for (j = cmd.insert_len_; j != 0; --j) {\r | |
448 | size_t context = Context(prev_byte, prev_byte2, literal_context_mode);\r | |
449 | uint8_t literal = ringbuffer[pos & mask];\r | |
450 | ContextBlockSplitterAddSymbol(\r | |
451 | m, &lit_blocks, literal, static_context_map[context]);\r | |
452 | prev_byte2 = prev_byte;\r | |
453 | if (BROTLI_IS_OOM(m)) return;\r | |
454 | prev_byte = literal;\r | |
455 | ++pos;\r | |
456 | }\r | |
457 | pos += CommandCopyLen(&cmd);\r | |
458 | if (CommandCopyLen(&cmd)) {\r | |
459 | prev_byte2 = ringbuffer[(pos - 2) & mask];\r | |
460 | prev_byte = ringbuffer[(pos - 1) & mask];\r | |
461 | if (cmd.cmd_prefix_ >= 128) {\r | |
462 | BlockSplitterAddSymbolDistance(&dist_blocks, cmd.dist_prefix_);\r | |
463 | }\r | |
464 | }\r | |
465 | }\r | |
466 | \r | |
467 | ContextBlockSplitterFinishBlock(m, &lit_blocks, /* is_final = */ BROTLI_TRUE);\r | |
468 | if (BROTLI_IS_OOM(m)) return;\r | |
469 | CleanupContextBlockSplitter(m, &lit_blocks);\r | |
470 | BlockSplitterFinishBlockCommand(&cmd_blocks, /* is_final = */ BROTLI_TRUE);\r | |
471 | BlockSplitterFinishBlockDistance(&dist_blocks, /* is_final = */ BROTLI_TRUE);\r | |
472 | \r | |
473 | assert(mb->literal_context_map == 0);\r | |
474 | mb->literal_context_map_size =\r | |
475 | mb->literal_split.num_types << BROTLI_LITERAL_CONTEXT_BITS;\r | |
476 | mb->literal_context_map =\r | |
477 | BROTLI_ALLOC(m, uint32_t, mb->literal_context_map_size);\r | |
478 | if (BROTLI_IS_OOM(m)) return;\r | |
479 | \r | |
480 | for (i = 0; i < mb->literal_split.num_types; ++i) {\r | |
481 | size_t j;\r | |
482 | for (j = 0; j < (1u << BROTLI_LITERAL_CONTEXT_BITS); ++j) {\r | |
483 | mb->literal_context_map[(i << BROTLI_LITERAL_CONTEXT_BITS) + j] =\r | |
484 | (uint32_t)(i * num_contexts) + static_context_map[j];\r | |
485 | }\r | |
486 | }\r | |
487 | }\r | |
488 | \r | |
489 | void BrotliOptimizeHistograms(size_t num_direct_distance_codes,\r | |
490 | size_t distance_postfix_bits,\r | |
491 | MetaBlockSplit* mb) {\r | |
492 | uint8_t good_for_rle[BROTLI_NUM_COMMAND_SYMBOLS];\r | |
493 | size_t num_distance_codes;\r | |
494 | size_t i;\r | |
495 | for (i = 0; i < mb->literal_histograms_size; ++i) {\r | |
496 | BrotliOptimizeHuffmanCountsForRle(256, mb->literal_histograms[i].data_,\r | |
497 | good_for_rle);\r | |
498 | }\r | |
499 | for (i = 0; i < mb->command_histograms_size; ++i) {\r | |
500 | BrotliOptimizeHuffmanCountsForRle(BROTLI_NUM_COMMAND_SYMBOLS,\r | |
501 | mb->command_histograms[i].data_,\r | |
502 | good_for_rle);\r | |
503 | }\r | |
504 | num_distance_codes = BROTLI_NUM_DISTANCE_SHORT_CODES +\r | |
505 | num_direct_distance_codes + (48u << distance_postfix_bits);\r | |
506 | for (i = 0; i < mb->distance_histograms_size; ++i) {\r | |
507 | BrotliOptimizeHuffmanCountsForRle(num_distance_codes,\r | |
508 | mb->distance_histograms[i].data_,\r | |
509 | good_for_rle);\r | |
510 | }\r | |
511 | }\r | |
512 | \r | |
513 | #if defined(__cplusplus) || defined(c_plusplus)\r | |
514 | } /* extern "C" */\r | |
515 | #endif\r |