]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | ||
4 | #include "include/stringify.h" | |
5 | #include "CrushTester.h" | |
6 | #include "CrushTreeDumper.h" | |
7 | #include "include/ceph_features.h" | |
8 | ||
9 | #include <algorithm> | |
10 | #include <stdlib.h> | |
11 | #include <boost/lexical_cast.hpp> | |
12 | // to workaround https://svn.boost.org/trac/boost/ticket/9501 | |
13 | #ifdef _LIBCPP_VERSION | |
14 | #include <boost/version.hpp> | |
15 | #if BOOST_VERSION < 105600 | |
16 | #define ICL_USE_BOOST_MOVE_IMPLEMENTATION | |
17 | #endif | |
18 | #endif | |
19 | #include <boost/icl/interval_map.hpp> | |
20 | #include <boost/algorithm/string/join.hpp> | |
31f18b77 | 21 | #include "common/SubProcess.h" |
7c673cae FG |
22 | |
23 | void CrushTester::set_device_weight(int dev, float f) | |
24 | { | |
25 | int w = (int)(f * 0x10000); | |
26 | if (w < 0) | |
27 | w = 0; | |
28 | if (w > 0x10000) | |
29 | w = 0x10000; | |
30 | device_weight[dev] = w; | |
31 | } | |
32 | ||
33 | int CrushTester::get_maximum_affected_by_rule(int ruleno) | |
34 | { | |
35 | // get the number of steps in RULENO | |
36 | int rule_size = crush.get_rule_len(ruleno); | |
37 | vector<int> affected_types; | |
38 | map<int,int> replications_by_type; | |
39 | ||
40 | for (int i = 0; i < rule_size; i++){ | |
41 | // get what operation is done by the current step | |
42 | int rule_operation = crush.get_rule_op(ruleno, i); | |
43 | ||
44 | // if the operation specifies choosing a device type, store it | |
45 | if (rule_operation >= 2 && rule_operation != 4){ | |
46 | int desired_replication = crush.get_rule_arg1(ruleno,i); | |
47 | int affected_type = crush.get_rule_arg2(ruleno,i); | |
48 | affected_types.push_back(affected_type); | |
49 | replications_by_type[affected_type] = desired_replication; | |
50 | } | |
51 | } | |
52 | ||
53 | /* | |
54 | * now for each of the affected bucket types, see what is the | |
55 | * maximum we are (a) requesting or (b) have | |
56 | */ | |
57 | ||
58 | map<int,int> max_devices_of_type; | |
59 | ||
60 | // loop through the vector of affected types | |
61 | for (vector<int>::iterator it = affected_types.begin(); it != affected_types.end(); ++it){ | |
62 | // loop through the number of buckets looking for affected types | |
63 | for (map<int,string>::iterator p = crush.name_map.begin(); p != crush.name_map.end(); ++p){ | |
64 | int bucket_type = crush.get_bucket_type(p->first); | |
65 | if ( bucket_type == *it) | |
66 | max_devices_of_type[*it]++; | |
67 | } | |
68 | } | |
69 | ||
70 | for(std::vector<int>::iterator it = affected_types.begin(); it != affected_types.end(); ++it){ | |
71 | if ( replications_by_type[*it] > 0 && replications_by_type[*it] < max_devices_of_type[*it] ) | |
72 | max_devices_of_type[*it] = replications_by_type[*it]; | |
73 | } | |
74 | ||
75 | /* | |
76 | * get the smallest number of buckets available of any type as this is our upper bound on | |
77 | * the number of replicas we can place | |
78 | */ | |
79 | int max_affected = max( crush.get_max_buckets(), crush.get_max_devices() ); | |
80 | ||
81 | for(std::vector<int>::iterator it = affected_types.begin(); it != affected_types.end(); ++it){ | |
82 | if (max_devices_of_type[*it] > 0 && max_devices_of_type[*it] < max_affected ) | |
83 | max_affected = max_devices_of_type[*it]; | |
84 | } | |
85 | ||
86 | return max_affected; | |
87 | } | |
88 | ||
89 | ||
90 | map<int,int> CrushTester::get_collapsed_mapping() | |
91 | { | |
92 | int num_to_check = crush.get_max_devices(); | |
93 | int next_id = 0; | |
94 | map<int, int> collapse_mask; | |
95 | ||
96 | for (int i = 0; i < num_to_check; i++){ | |
97 | if (crush.check_item_present(i)){ | |
98 | collapse_mask[i] = next_id; | |
99 | next_id++; | |
100 | } | |
101 | } | |
102 | ||
103 | return collapse_mask; | |
104 | } | |
105 | ||
106 | void CrushTester::adjust_weights(vector<__u32>& weight) | |
107 | { | |
108 | ||
109 | if (mark_down_device_ratio > 0) { | |
110 | // active buckets | |
111 | vector<int> bucket_ids; | |
112 | for (int i = 0; i < crush.get_max_buckets(); i++) { | |
113 | int id = -1 - i; | |
114 | if (crush.get_bucket_weight(id) > 0) { | |
115 | bucket_ids.push_back(id); | |
116 | } | |
117 | } | |
118 | ||
119 | // get buckets that are one level above a device | |
120 | vector<int> buckets_above_devices; | |
121 | for (unsigned i = 0; i < bucket_ids.size(); i++) { | |
122 | // grab the first child object of a bucket and check if it's ID is less than 0 | |
123 | int id = bucket_ids[i]; | |
124 | if (crush.get_bucket_size(id) == 0) | |
125 | continue; | |
126 | int first_child = crush.get_bucket_item(id, 0); // returns the ID of the bucket or device | |
127 | if (first_child >= 0) { | |
128 | buckets_above_devices.push_back(id); | |
129 | } | |
130 | } | |
131 | ||
132 | // permute bucket list | |
133 | for (unsigned i = 0; i < buckets_above_devices.size(); i++) { | |
134 | unsigned j = lrand48() % (buckets_above_devices.size() - 1); | |
135 | std::swap(buckets_above_devices[i], buckets_above_devices[j]); | |
136 | } | |
137 | ||
138 | // calculate how many buckets and devices we need to reap... | |
139 | int num_buckets_to_visit = (int) (mark_down_bucket_ratio * buckets_above_devices.size()); | |
140 | ||
141 | for (int i = 0; i < num_buckets_to_visit; i++) { | |
142 | int id = buckets_above_devices[i]; | |
143 | int size = crush.get_bucket_size(id); | |
144 | vector<int> items; | |
145 | for (int o = 0; o < size; o++) | |
146 | items.push_back(crush.get_bucket_item(id, o)); | |
147 | ||
148 | // permute items | |
149 | for (int o = 0; o < size; o++) { | |
150 | int j = lrand48() % (crush.get_bucket_size(id) - 1); | |
151 | std::swap(items[o], items[j]); | |
152 | } | |
153 | ||
154 | int local_devices_to_visit = (int) (mark_down_device_ratio*size); | |
155 | for (int o = 0; o < local_devices_to_visit; o++){ | |
156 | int item = crush.get_bucket_item(id, o); | |
157 | weight[item] = 0; | |
158 | } | |
159 | } | |
160 | } | |
161 | } | |
162 | ||
163 | bool CrushTester::check_valid_placement(int ruleno, vector<int> in, const vector<__u32>& weight) | |
164 | { | |
165 | ||
166 | bool valid_placement = true; | |
167 | vector<int> included_devices; | |
168 | map<string,string> seen_devices; | |
169 | ||
170 | // first do the easy check that all devices are "up" | |
171 | for (vector<int>::iterator it = in.begin(); it != in.end(); ++it) { | |
172 | if (weight[(*it)] == 0) { | |
173 | valid_placement = false; | |
174 | break; | |
175 | } else if (weight[(*it)] > 0) { | |
176 | included_devices.push_back( (*it) ); | |
177 | } | |
178 | } | |
179 | ||
180 | /* | |
181 | * now do the harder test of checking that the CRUSH rule r is not violated | |
182 | * we could test that none of the devices mentioned in out are unique, | |
183 | * but this is a special case of this test | |
184 | */ | |
185 | ||
186 | // get the number of steps in RULENO | |
187 | int rule_size = crush.get_rule_len(ruleno); | |
188 | vector<string> affected_types; | |
189 | ||
190 | // get the smallest type id, and name | |
191 | int min_map_type = crush.get_num_type_names(); | |
192 | for (map<int,string>::iterator it = crush.type_map.begin(); it != crush.type_map.end(); ++it ) { | |
193 | if ( (*it).first < min_map_type ) { | |
194 | min_map_type = (*it).first; | |
195 | } | |
196 | } | |
197 | ||
198 | string min_map_type_name = crush.type_map[min_map_type]; | |
199 | ||
200 | // get the types of devices affected by RULENO | |
201 | for (int i = 0; i < rule_size; i++) { | |
202 | // get what operation is done by the current step | |
203 | int rule_operation = crush.get_rule_op(ruleno, i); | |
204 | ||
205 | // if the operation specifies choosing a device type, store it | |
206 | if (rule_operation >= 2 && rule_operation != 4) { | |
207 | int affected_type = crush.get_rule_arg2(ruleno,i); | |
208 | affected_types.push_back( crush.get_type_name(affected_type)); | |
209 | } | |
210 | } | |
211 | ||
212 | // find in if we are only dealing with osd's | |
213 | bool only_osd_affected = false; | |
214 | if (affected_types.size() == 1) { | |
215 | if ((affected_types.back() == min_map_type_name) && (min_map_type_name == "osd")) { | |
216 | only_osd_affected = true; | |
217 | } | |
218 | } | |
219 | ||
220 | // check that we don't have any duplicate id's | |
221 | for (vector<int>::iterator it = included_devices.begin(); it != included_devices.end(); ++it) { | |
222 | int num_copies = std::count(included_devices.begin(), included_devices.end(), (*it) ); | |
223 | if (num_copies > 1) { | |
224 | valid_placement = false; | |
225 | } | |
226 | } | |
227 | ||
228 | // if we have more than just osd's affected we need to do a lot more work | |
229 | if (!only_osd_affected) { | |
230 | // loop through the devices that are "in/up" | |
231 | for (vector<int>::iterator it = included_devices.begin(); it != included_devices.end(); ++it) { | |
232 | if (valid_placement == false) | |
233 | break; | |
234 | ||
235 | // create a temporary map of the form (device type, device name in map) | |
236 | map<string,string> device_location_hierarchy = crush.get_full_location(*it); | |
237 | ||
238 | // loop over the types affected by RULENO looking for duplicate bucket assignments | |
239 | for (vector<string>::iterator t = affected_types.begin(); t != affected_types.end(); ++t) { | |
240 | if (seen_devices.count( device_location_hierarchy[*t])) { | |
241 | valid_placement = false; | |
242 | break; | |
243 | } else { | |
244 | // store the devices we have seen in the form of (device name, device type) | |
245 | seen_devices[ device_location_hierarchy[*t] ] = *t; | |
246 | } | |
247 | } | |
248 | } | |
249 | } | |
250 | ||
251 | return valid_placement; | |
252 | } | |
253 | ||
254 | int CrushTester::random_placement(int ruleno, vector<int>& out, int maxout, vector<__u32>& weight) | |
255 | { | |
256 | // get the total weight of the system | |
257 | int total_weight = 0; | |
258 | for (unsigned i = 0; i < weight.size(); i++) | |
259 | total_weight += weight[i]; | |
260 | ||
261 | if (total_weight == 0 || | |
262 | crush.get_max_devices() == 0) | |
263 | return -EINVAL; | |
264 | ||
265 | // determine the real maximum number of devices to return | |
266 | int devices_requested = min(maxout, get_maximum_affected_by_rule(ruleno)); | |
267 | bool accept_placement = false; | |
268 | ||
269 | vector<int> trial_placement(devices_requested); | |
270 | int attempted_tries = 0; | |
271 | int max_tries = 100; | |
272 | do { | |
273 | // create a vector to hold our trial mappings | |
274 | int temp_array[devices_requested]; | |
275 | for (int i = 0; i < devices_requested; i++){ | |
276 | temp_array[i] = lrand48() % (crush.get_max_devices()); | |
277 | } | |
278 | ||
279 | trial_placement.assign(temp_array, temp_array + devices_requested); | |
280 | accept_placement = check_valid_placement(ruleno, trial_placement, weight); | |
281 | attempted_tries++; | |
282 | } while (accept_placement == false && attempted_tries < max_tries); | |
283 | ||
284 | // save our random placement to the out vector | |
285 | if (accept_placement) | |
286 | out.assign(trial_placement.begin(), trial_placement.end()); | |
287 | ||
288 | // or don't.... | |
289 | else if (attempted_tries == max_tries) | |
290 | return -EINVAL; | |
291 | ||
292 | return 0; | |
293 | } | |
294 | ||
295 | void CrushTester::write_integer_indexed_vector_data_string(vector<string> &dst, int index, vector<int> vector_data) | |
296 | { | |
297 | stringstream data_buffer (stringstream::in | stringstream::out); | |
298 | unsigned input_size = vector_data.size(); | |
299 | ||
300 | // pass the indexing variable to the data buffer | |
301 | data_buffer << index; | |
302 | ||
303 | // pass the rest of the input data to the buffer | |
304 | for (unsigned i = 0; i < input_size; i++) { | |
305 | data_buffer << ',' << vector_data[i]; | |
306 | } | |
307 | ||
308 | data_buffer << std::endl; | |
309 | ||
310 | // write the data buffer to the destination | |
311 | dst.push_back( data_buffer.str() ); | |
312 | } | |
313 | ||
314 | void CrushTester::write_integer_indexed_vector_data_string(vector<string> &dst, int index, vector<float> vector_data) | |
315 | { | |
316 | stringstream data_buffer (stringstream::in | stringstream::out); | |
317 | unsigned input_size = vector_data.size(); | |
318 | ||
319 | // pass the indexing variable to the data buffer | |
320 | data_buffer << index; | |
321 | ||
322 | // pass the rest of the input data to the buffer | |
323 | for (unsigned i = 0; i < input_size; i++) { | |
324 | data_buffer << ',' << vector_data[i]; | |
325 | } | |
326 | ||
327 | data_buffer << std::endl; | |
328 | ||
329 | // write the data buffer to the destination | |
330 | dst.push_back( data_buffer.str() ); | |
331 | } | |
332 | ||
333 | void CrushTester::write_integer_indexed_scalar_data_string(vector<string> &dst, int index, int scalar_data) | |
334 | { | |
335 | stringstream data_buffer (stringstream::in | stringstream::out); | |
336 | ||
337 | // pass the indexing variable to the data buffer | |
338 | data_buffer << index; | |
339 | ||
340 | // pass the input data to the buffer | |
341 | data_buffer << ',' << scalar_data; | |
342 | data_buffer << std::endl; | |
343 | ||
344 | // write the data buffer to the destination | |
345 | dst.push_back( data_buffer.str() ); | |
346 | } | |
347 | void CrushTester::write_integer_indexed_scalar_data_string(vector<string> &dst, int index, float scalar_data) | |
348 | { | |
349 | stringstream data_buffer (stringstream::in | stringstream::out); | |
350 | ||
351 | // pass the indexing variable to the data buffer | |
352 | data_buffer << index; | |
353 | ||
354 | // pass the input data to the buffer | |
355 | data_buffer << ',' << scalar_data; | |
356 | data_buffer << std::endl; | |
357 | ||
358 | // write the data buffer to the destination | |
359 | dst.push_back( data_buffer.str() ); | |
360 | } | |
361 | ||
362 | int CrushTester::test_with_crushtool(const char *crushtool_cmd, | |
363 | int max_id, int timeout, | |
364 | int ruleset) | |
365 | { | |
366 | SubProcessTimed crushtool(crushtool_cmd, SubProcess::PIPE, SubProcess::CLOSE, SubProcess::PIPE, timeout); | |
367 | string opt_max_id = boost::lexical_cast<string>(max_id); | |
368 | crushtool.add_cmd_args( | |
369 | "-i", "-", | |
370 | "--test", "--check", opt_max_id.c_str(), | |
371 | "--min-x", "1", | |
372 | "--max-x", "50", | |
373 | NULL); | |
374 | if (ruleset >= 0) { | |
375 | crushtool.add_cmd_args( | |
376 | "--ruleset", | |
377 | stringify(ruleset).c_str(), | |
378 | NULL); | |
379 | } | |
380 | int ret = crushtool.spawn(); | |
381 | if (ret != 0) { | |
382 | err << "failed run crushtool: " << crushtool.err(); | |
383 | return ret; | |
384 | } | |
385 | ||
386 | bufferlist bl; | |
387 | ::encode(crush, bl, CEPH_FEATURES_SUPPORTED_DEFAULT); | |
388 | bl.write_fd(crushtool.get_stdin()); | |
389 | crushtool.close_stdin(); | |
390 | bl.clear(); | |
391 | ret = bl.read_fd(crushtool.get_stderr(), 100 * 1024); | |
392 | if (ret < 0) { | |
393 | err << "failed read from crushtool: " << cpp_strerror(-ret); | |
394 | return ret; | |
395 | } | |
396 | bl.write_stream(err); | |
397 | if (crushtool.join() != 0) { | |
398 | err << crushtool.err(); | |
399 | return -EINVAL; | |
400 | } | |
401 | ||
402 | return 0; | |
403 | } | |
404 | ||
405 | namespace { | |
406 | class BadCrushMap : public std::runtime_error { | |
407 | public: | |
408 | int item; | |
409 | BadCrushMap(const char* msg, int id) | |
410 | : std::runtime_error(msg), item(id) {} | |
411 | }; | |
412 | // throws if any node in the crush fail to print | |
413 | class CrushWalker : public CrushTreeDumper::Dumper<void> { | |
414 | typedef void DumbFormatter; | |
415 | typedef CrushTreeDumper::Dumper<DumbFormatter> Parent; | |
416 | int max_id; | |
417 | public: | |
418 | CrushWalker(const CrushWrapper *crush, unsigned max_id) | |
419 | : Parent(crush), max_id(max_id) {} | |
420 | void dump_item(const CrushTreeDumper::Item &qi, DumbFormatter *) override { | |
421 | int type = -1; | |
422 | if (qi.is_bucket()) { | |
423 | if (!crush->get_item_name(qi.id)) { | |
424 | throw BadCrushMap("unknown item name", qi.id); | |
425 | } | |
426 | type = crush->get_bucket_type(qi.id); | |
427 | } else { | |
428 | if (max_id > 0 && qi.id >= max_id) { | |
429 | throw BadCrushMap("item id too large", qi.id); | |
430 | } | |
431 | type = 0; | |
432 | } | |
433 | if (!crush->get_type_name(type)) { | |
434 | throw BadCrushMap("unknown type name", qi.id); | |
435 | } | |
436 | } | |
437 | }; | |
438 | } | |
439 | ||
440 | bool CrushTester::check_name_maps(unsigned max_id) const | |
441 | { | |
442 | CrushWalker crush_walker(&crush, max_id); | |
443 | try { | |
444 | // walk through the crush, to see if its self-contained | |
445 | crush_walker.dump(NULL); | |
446 | // and see if the maps is also able to handle straying OSDs, whose id >= 0. | |
447 | // "ceph osd tree" will try to print them, even they are not listed in the | |
448 | // crush map. | |
449 | crush_walker.dump_item(CrushTreeDumper::Item(0, 0, 0), NULL); | |
450 | } catch (const BadCrushMap& e) { | |
451 | err << e.what() << ": item#" << e.item << std::endl; | |
452 | return false; | |
453 | } | |
454 | return true; | |
455 | } | |
456 | ||
457 | static string get_rule_name(CrushWrapper& crush, int rule) | |
458 | { | |
459 | if (crush.get_rule_name(rule)) | |
460 | return crush.get_rule_name(rule); | |
461 | else | |
462 | return string("rule") + std::to_string(rule); | |
463 | } | |
464 | ||
465 | void CrushTester::check_overlapped_rules() const | |
466 | { | |
467 | namespace icl = boost::icl; | |
468 | typedef std::set<string> RuleNames; | |
469 | typedef icl::interval_map<int, RuleNames> Rules; | |
470 | // <ruleset, type> => interval_map<size, {names}> | |
471 | typedef std::map<std::pair<int, int>, Rules> RuleSets; | |
472 | using interval = icl::interval<int>; | |
473 | ||
474 | // mimic the logic of crush_find_rule(), but it only return the first matched | |
475 | // one, but I am collecting all of them by the overlapped sizes. | |
476 | RuleSets rulesets; | |
477 | for (int rule = 0; rule < crush.get_max_rules(); rule++) { | |
478 | if (!crush.rule_exists(rule)) { | |
479 | continue; | |
480 | } | |
481 | Rules& rules = rulesets[{crush.get_rule_mask_ruleset(rule), | |
482 | crush.get_rule_mask_type(rule)}]; | |
483 | rules += make_pair(interval::closed(crush.get_rule_mask_min_size(rule), | |
484 | crush.get_rule_mask_max_size(rule)), | |
485 | RuleNames{get_rule_name(crush, rule)}); | |
486 | } | |
487 | for (auto i : rulesets) { | |
488 | auto ruleset_type = i.first; | |
489 | const Rules& rules = i.second; | |
490 | for (auto r : rules) { | |
491 | const RuleNames& names = r.second; | |
492 | // if there are more than one rules covering the same size range, | |
493 | // print them out. | |
494 | if (names.size() > 1) { | |
495 | err << "overlapped rules in ruleset " << ruleset_type.first << ": " | |
496 | << boost::join(names, ", ") << "\n"; | |
497 | } | |
498 | } | |
499 | } | |
500 | } | |
501 | ||
502 | int CrushTester::test() | |
503 | { | |
504 | if (min_rule < 0 || max_rule < 0) { | |
505 | min_rule = 0; | |
506 | max_rule = crush.get_max_rules() - 1; | |
507 | } | |
508 | if (min_x < 0 || max_x < 0) { | |
509 | min_x = 0; | |
510 | max_x = 1023; | |
511 | } | |
512 | ||
513 | // initial osd weights | |
514 | vector<__u32> weight; | |
515 | ||
516 | /* | |
517 | * note device weight is set by crushtool | |
518 | * (likely due to a given a command line option) | |
519 | */ | |
520 | for (int o = 0; o < crush.get_max_devices(); o++) { | |
521 | if (device_weight.count(o)) { | |
522 | weight.push_back(device_weight[o]); | |
523 | } else if (crush.check_item_present(o)) { | |
524 | weight.push_back(0x10000); | |
525 | } else { | |
526 | weight.push_back(0); | |
527 | } | |
528 | } | |
529 | ||
530 | if (output_utilization_all) | |
531 | err << "devices weights (hex): " << hex << weight << dec << std::endl; | |
532 | ||
533 | // make adjustments | |
534 | adjust_weights(weight); | |
535 | ||
536 | ||
537 | int num_devices_active = 0; | |
538 | for (vector<__u32>::iterator p = weight.begin(); p != weight.end(); ++p) | |
539 | if (*p > 0) | |
540 | num_devices_active++; | |
541 | ||
542 | if (output_choose_tries) | |
543 | crush.start_choose_profile(); | |
544 | ||
545 | for (int r = min_rule; r < crush.get_max_rules() && r <= max_rule; r++) { | |
546 | if (!crush.rule_exists(r)) { | |
547 | if (output_statistics) | |
548 | err << "rule " << r << " dne" << std::endl; | |
549 | continue; | |
550 | } | |
551 | if (ruleset >= 0 && | |
552 | crush.get_rule_mask_ruleset(r) != ruleset) { | |
553 | continue; | |
554 | } | |
555 | int minr = min_rep, maxr = max_rep; | |
556 | if (min_rep < 0 || max_rep < 0) { | |
557 | minr = crush.get_rule_mask_min_size(r); | |
558 | maxr = crush.get_rule_mask_max_size(r); | |
559 | } | |
560 | ||
561 | if (output_statistics) | |
562 | err << "rule " << r << " (" << crush.get_rule_name(r) | |
563 | << "), x = " << min_x << ".." << max_x | |
564 | << ", numrep = " << minr << ".." << maxr | |
565 | << std::endl; | |
566 | ||
567 | for (int nr = minr; nr <= maxr; nr++) { | |
568 | vector<int> per(crush.get_max_devices()); | |
569 | map<int,int> sizes; | |
570 | ||
571 | int num_objects = ((max_x - min_x) + 1); | |
572 | float num_devices = (float) per.size(); // get the total number of devices, better to cast as a float here | |
573 | ||
574 | // create a structure to hold data for post-processing | |
575 | tester_data_set tester_data; | |
576 | vector<float> vector_data_buffer_f; | |
577 | ||
578 | // create a map to hold batch-level placement information | |
579 | map<int, vector<int> > batch_per; | |
580 | int objects_per_batch = num_objects / num_batches; | |
581 | int batch_min = min_x; | |
582 | int batch_max = min_x + objects_per_batch - 1; | |
583 | ||
584 | // get the total weight of the system | |
585 | int total_weight = 0; | |
586 | for (unsigned i = 0; i < per.size(); i++) | |
587 | total_weight += weight[i]; | |
588 | ||
589 | if (total_weight == 0) | |
590 | continue; | |
591 | ||
592 | // compute the expected number of objects stored per device in the absence of weighting | |
593 | float expected_objects = min(nr, get_maximum_affected_by_rule(r)) * num_objects; | |
594 | ||
595 | // compute each device's proportional weight | |
596 | vector<float> proportional_weights( per.size() ); | |
597 | ||
598 | for (unsigned i = 0; i < per.size(); i++) | |
599 | proportional_weights[i] = (float) weight[i] / (float) total_weight; | |
600 | ||
601 | if (output_data_file) { | |
602 | // stage the absolute weight information for post-processing | |
603 | for (unsigned i = 0; i < per.size(); i++) { | |
604 | tester_data.absolute_weights[i] = (float) weight[i] / (float)0x10000; | |
605 | } | |
606 | ||
607 | // stage the proportional weight information for post-processing | |
608 | for (unsigned i = 0; i < per.size(); i++) { | |
609 | if (proportional_weights[i] > 0 ) | |
610 | tester_data.proportional_weights[i] = proportional_weights[i]; | |
611 | ||
612 | tester_data.proportional_weights_all[i] = proportional_weights[i]; | |
613 | } | |
614 | ||
615 | } | |
616 | // compute the expected number of objects stored per device when a device's weight is considered | |
617 | vector<float> num_objects_expected(num_devices); | |
618 | ||
619 | for (unsigned i = 0; i < num_devices; i++) | |
620 | num_objects_expected[i] = (proportional_weights[i]*expected_objects); | |
621 | ||
622 | for (int current_batch = 0; current_batch < num_batches; current_batch++) { | |
623 | if (current_batch == (num_batches - 1)) { | |
624 | batch_max = max_x; | |
625 | objects_per_batch = (batch_max - batch_min + 1); | |
626 | } | |
627 | ||
628 | float batch_expected_objects = min(nr, get_maximum_affected_by_rule(r)) * objects_per_batch; | |
629 | vector<float> batch_num_objects_expected( per.size() ); | |
630 | ||
631 | for (unsigned i = 0; i < per.size() ; i++) | |
632 | batch_num_objects_expected[i] = (proportional_weights[i]*batch_expected_objects); | |
633 | ||
634 | // create a vector to hold placement results temporarily | |
635 | vector<int> temporary_per ( per.size() ); | |
636 | ||
637 | for (int x = batch_min; x <= batch_max; x++) { | |
638 | // create a vector to hold the results of a CRUSH placement or RNG simulation | |
639 | vector<int> out; | |
640 | ||
641 | if (use_crush) { | |
642 | if (output_mappings) | |
643 | err << "CRUSH"; // prepend CRUSH to placement output | |
644 | uint32_t real_x = x; | |
645 | if (pool_id != -1) { | |
646 | real_x = crush_hash32_2(CRUSH_HASH_RJENKINS1, x, (uint32_t)pool_id); | |
647 | } | |
648 | crush.do_rule(r, real_x, out, nr, weight, 0); | |
649 | } else { | |
650 | if (output_mappings) | |
651 | err << "RNG"; // prepend RNG to placement output to denote simulation | |
652 | // test our new monte carlo placement generator | |
653 | random_placement(r, out, nr, weight); | |
654 | } | |
655 | ||
656 | if (output_mappings) | |
657 | err << " rule " << r << " x " << x << " " << out << std::endl; | |
658 | ||
659 | if (output_data_file) | |
660 | write_integer_indexed_vector_data_string(tester_data.placement_information, x, out); | |
661 | ||
662 | bool has_item_none = false; | |
663 | for (unsigned i = 0; i < out.size(); i++) { | |
664 | if (out[i] != CRUSH_ITEM_NONE) { | |
665 | per[out[i]]++; | |
666 | temporary_per[out[i]]++; | |
667 | } else { | |
668 | has_item_none = true; | |
669 | } | |
670 | } | |
671 | ||
672 | batch_per[current_batch] = temporary_per; | |
673 | sizes[out.size()]++; | |
674 | if (output_bad_mappings && | |
675 | (out.size() != (unsigned)nr || | |
676 | has_item_none)) { | |
677 | err << "bad mapping rule " << r << " x " << x << " num_rep " << nr << " result " << out << std::endl; | |
678 | } | |
679 | } | |
680 | ||
681 | batch_min = batch_max + 1; | |
682 | batch_max = batch_min + objects_per_batch - 1; | |
683 | } | |
684 | ||
685 | for (unsigned i = 0; i < per.size(); i++) | |
686 | if (output_utilization && !output_statistics) | |
687 | err << " device " << i | |
688 | << ":\t" << per[i] << std::endl; | |
689 | ||
690 | for (map<int,int>::iterator p = sizes.begin(); p != sizes.end(); ++p) | |
691 | if (output_statistics) | |
692 | err << "rule " << r << " (" << crush.get_rule_name(r) << ") num_rep " << nr | |
693 | << " result size == " << p->first << ":\t" | |
694 | << p->second << "/" << (max_x-min_x+1) << std::endl; | |
695 | ||
696 | if (output_statistics) | |
697 | for (unsigned i = 0; i < per.size(); i++) { | |
698 | if (output_utilization) { | |
699 | if (num_objects_expected[i] > 0 && per[i] > 0) { | |
700 | err << " device " << i << ":\t" | |
701 | << "\t" << " stored " << ": " << per[i] | |
702 | << "\t" << " expected " << ": " << num_objects_expected[i] | |
703 | << std::endl; | |
704 | } | |
705 | } else if (output_utilization_all) { | |
706 | err << " device " << i << ":\t" | |
707 | << "\t" << " stored " << ": " << per[i] | |
708 | << "\t" << " expected " << ": " << num_objects_expected[i] | |
709 | << std::endl; | |
710 | } | |
711 | } | |
712 | ||
713 | if (output_data_file) | |
714 | for (unsigned i = 0; i < per.size(); i++) { | |
715 | vector_data_buffer_f.clear(); | |
716 | vector_data_buffer_f.push_back( (float) per[i]); | |
717 | vector_data_buffer_f.push_back( (float) num_objects_expected[i]); | |
718 | ||
719 | write_integer_indexed_vector_data_string(tester_data.device_utilization_all, i, vector_data_buffer_f); | |
720 | ||
721 | if (num_objects_expected[i] > 0 && per[i] > 0) | |
722 | write_integer_indexed_vector_data_string(tester_data.device_utilization, i, vector_data_buffer_f); | |
723 | } | |
724 | ||
725 | if (output_data_file && num_batches > 1) { | |
726 | // stage batch utilization information for post-processing | |
727 | for (int i = 0; i < num_batches; i++) { | |
728 | write_integer_indexed_vector_data_string(tester_data.batch_device_utilization_all, i, batch_per[i]); | |
729 | write_integer_indexed_vector_data_string(tester_data.batch_device_expected_utilization_all, i, batch_per[i]); | |
730 | } | |
731 | } | |
732 | ||
733 | string rule_tag = crush.get_rule_name(r); | |
734 | ||
735 | if (output_csv) | |
736 | write_data_set_to_csv(output_data_file_name+rule_tag,tester_data); | |
737 | } | |
738 | } | |
739 | ||
740 | if (output_choose_tries) { | |
741 | __u32 *v = 0; | |
742 | int n = crush.get_choose_profile(&v); | |
743 | for (int i=0; i<n; i++) { | |
744 | cout.setf(std::ios::right); | |
745 | cout << std::setw(2) | |
746 | << i << ": " << std::setw(9) << v[i]; | |
747 | cout.unsetf(std::ios::right); | |
748 | cout << std::endl; | |
749 | } | |
750 | ||
751 | crush.stop_choose_profile(); | |
752 | } | |
753 | ||
754 | return 0; | |
755 | } |