1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "include/stringify.h"
5 #include "CrushTester.h"
6 #include "CrushTreeDumper.h"
7 #include "include/ceph_features.h"
11 #include <boost/lexical_cast.hpp>
12 // to workaround https://svn.boost.org/trac/boost/ticket/9501
13 #ifdef _LIBCPP_VERSION
14 #include <boost/version.hpp>
15 #if BOOST_VERSION < 105600
16 #define ICL_USE_BOOST_MOVE_IMPLEMENTATION
19 #include <boost/icl/interval_map.hpp>
20 #include <boost/algorithm/string/join.hpp>
21 #include "common/SubProcess.h"
23 void CrushTester::set_device_weight(int dev
, float f
)
25 int w
= (int)(f
* 0x10000);
30 device_weight
[dev
] = w
;
33 int CrushTester::get_maximum_affected_by_rule(int ruleno
)
35 // get the number of steps in RULENO
36 int rule_size
= crush
.get_rule_len(ruleno
);
37 vector
<int> affected_types
;
38 map
<int,int> replications_by_type
;
40 for (int i
= 0; i
< rule_size
; i
++){
41 // get what operation is done by the current step
42 int rule_operation
= crush
.get_rule_op(ruleno
, i
);
44 // if the operation specifies choosing a device type, store it
45 if (rule_operation
>= 2 && rule_operation
!= 4){
46 int desired_replication
= crush
.get_rule_arg1(ruleno
,i
);
47 int affected_type
= crush
.get_rule_arg2(ruleno
,i
);
48 affected_types
.push_back(affected_type
);
49 replications_by_type
[affected_type
] = desired_replication
;
54 * now for each of the affected bucket types, see what is the
55 * maximum we are (a) requesting or (b) have
58 map
<int,int> max_devices_of_type
;
60 // loop through the vector of affected types
61 for (vector
<int>::iterator it
= affected_types
.begin(); it
!= affected_types
.end(); ++it
){
62 // loop through the number of buckets looking for affected types
63 for (map
<int,string
>::iterator p
= crush
.name_map
.begin(); p
!= crush
.name_map
.end(); ++p
){
64 int bucket_type
= crush
.get_bucket_type(p
->first
);
65 if ( bucket_type
== *it
)
66 max_devices_of_type
[*it
]++;
70 for(std::vector
<int>::iterator it
= affected_types
.begin(); it
!= affected_types
.end(); ++it
){
71 if ( replications_by_type
[*it
] > 0 && replications_by_type
[*it
] < max_devices_of_type
[*it
] )
72 max_devices_of_type
[*it
] = replications_by_type
[*it
];
76 * get the smallest number of buckets available of any type as this is our upper bound on
77 * the number of replicas we can place
79 int max_affected
= max( crush
.get_max_buckets(), crush
.get_max_devices() );
81 for(std::vector
<int>::iterator it
= affected_types
.begin(); it
!= affected_types
.end(); ++it
){
82 if (max_devices_of_type
[*it
] > 0 && max_devices_of_type
[*it
] < max_affected
)
83 max_affected
= max_devices_of_type
[*it
];
90 map
<int,int> CrushTester::get_collapsed_mapping()
92 int num_to_check
= crush
.get_max_devices();
94 map
<int, int> collapse_mask
;
96 for (int i
= 0; i
< num_to_check
; i
++){
97 if (crush
.check_item_present(i
)){
98 collapse_mask
[i
] = next_id
;
103 return collapse_mask
;
106 void CrushTester::adjust_weights(vector
<__u32
>& weight
)
109 if (mark_down_device_ratio
> 0) {
111 vector
<int> bucket_ids
;
112 for (int i
= 0; i
< crush
.get_max_buckets(); i
++) {
114 if (crush
.get_bucket_weight(id
) > 0) {
115 bucket_ids
.push_back(id
);
119 // get buckets that are one level above a device
120 vector
<int> buckets_above_devices
;
121 for (unsigned i
= 0; i
< bucket_ids
.size(); i
++) {
122 // grab the first child object of a bucket and check if it's ID is less than 0
123 int id
= bucket_ids
[i
];
124 if (crush
.get_bucket_size(id
) == 0)
126 int first_child
= crush
.get_bucket_item(id
, 0); // returns the ID of the bucket or device
127 if (first_child
>= 0) {
128 buckets_above_devices
.push_back(id
);
132 // permute bucket list
133 for (unsigned i
= 0; i
< buckets_above_devices
.size(); i
++) {
134 unsigned j
= lrand48() % (buckets_above_devices
.size() - 1);
135 std::swap(buckets_above_devices
[i
], buckets_above_devices
[j
]);
138 // calculate how many buckets and devices we need to reap...
139 int num_buckets_to_visit
= (int) (mark_down_bucket_ratio
* buckets_above_devices
.size());
141 for (int i
= 0; i
< num_buckets_to_visit
; i
++) {
142 int id
= buckets_above_devices
[i
];
143 int size
= crush
.get_bucket_size(id
);
145 for (int o
= 0; o
< size
; o
++)
146 items
.push_back(crush
.get_bucket_item(id
, o
));
149 for (int o
= 0; o
< size
; o
++) {
150 int j
= lrand48() % (crush
.get_bucket_size(id
) - 1);
151 std::swap(items
[o
], items
[j
]);
154 int local_devices_to_visit
= (int) (mark_down_device_ratio
*size
);
155 for (int o
= 0; o
< local_devices_to_visit
; o
++){
156 int item
= crush
.get_bucket_item(id
, o
);
163 bool CrushTester::check_valid_placement(int ruleno
, vector
<int> in
, const vector
<__u32
>& weight
)
166 bool valid_placement
= true;
167 vector
<int> included_devices
;
168 map
<string
,string
> seen_devices
;
170 // first do the easy check that all devices are "up"
171 for (vector
<int>::iterator it
= in
.begin(); it
!= in
.end(); ++it
) {
172 if (weight
[(*it
)] == 0) {
173 valid_placement
= false;
175 } else if (weight
[(*it
)] > 0) {
176 included_devices
.push_back( (*it
) );
181 * now do the harder test of checking that the CRUSH rule r is not violated
182 * we could test that none of the devices mentioned in out are unique,
183 * but this is a special case of this test
186 // get the number of steps in RULENO
187 int rule_size
= crush
.get_rule_len(ruleno
);
188 vector
<string
> affected_types
;
190 // get the smallest type id, and name
191 int min_map_type
= crush
.get_num_type_names();
192 for (map
<int,string
>::iterator it
= crush
.type_map
.begin(); it
!= crush
.type_map
.end(); ++it
) {
193 if ( (*it
).first
< min_map_type
) {
194 min_map_type
= (*it
).first
;
198 string min_map_type_name
= crush
.type_map
[min_map_type
];
200 // get the types of devices affected by RULENO
201 for (int i
= 0; i
< rule_size
; i
++) {
202 // get what operation is done by the current step
203 int rule_operation
= crush
.get_rule_op(ruleno
, i
);
205 // if the operation specifies choosing a device type, store it
206 if (rule_operation
>= 2 && rule_operation
!= 4) {
207 int affected_type
= crush
.get_rule_arg2(ruleno
,i
);
208 affected_types
.push_back( crush
.get_type_name(affected_type
));
212 // find in if we are only dealing with osd's
213 bool only_osd_affected
= false;
214 if (affected_types
.size() == 1) {
215 if ((affected_types
.back() == min_map_type_name
) && (min_map_type_name
== "osd")) {
216 only_osd_affected
= true;
220 // check that we don't have any duplicate id's
221 for (vector
<int>::iterator it
= included_devices
.begin(); it
!= included_devices
.end(); ++it
) {
222 int num_copies
= std::count(included_devices
.begin(), included_devices
.end(), (*it
) );
223 if (num_copies
> 1) {
224 valid_placement
= false;
228 // if we have more than just osd's affected we need to do a lot more work
229 if (!only_osd_affected
) {
230 // loop through the devices that are "in/up"
231 for (vector
<int>::iterator it
= included_devices
.begin(); it
!= included_devices
.end(); ++it
) {
232 if (valid_placement
== false)
235 // create a temporary map of the form (device type, device name in map)
236 map
<string
,string
> device_location_hierarchy
= crush
.get_full_location(*it
);
238 // loop over the types affected by RULENO looking for duplicate bucket assignments
239 for (vector
<string
>::iterator t
= affected_types
.begin(); t
!= affected_types
.end(); ++t
) {
240 if (seen_devices
.count( device_location_hierarchy
[*t
])) {
241 valid_placement
= false;
244 // store the devices we have seen in the form of (device name, device type)
245 seen_devices
[ device_location_hierarchy
[*t
] ] = *t
;
251 return valid_placement
;
254 int CrushTester::random_placement(int ruleno
, vector
<int>& out
, int maxout
, vector
<__u32
>& weight
)
256 // get the total weight of the system
257 int total_weight
= 0;
258 for (unsigned i
= 0; i
< weight
.size(); i
++)
259 total_weight
+= weight
[i
];
261 if (total_weight
== 0 ||
262 crush
.get_max_devices() == 0)
265 // determine the real maximum number of devices to return
266 int devices_requested
= min(maxout
, get_maximum_affected_by_rule(ruleno
));
267 bool accept_placement
= false;
269 vector
<int> trial_placement(devices_requested
);
270 int attempted_tries
= 0;
273 // create a vector to hold our trial mappings
274 int temp_array
[devices_requested
];
275 for (int i
= 0; i
< devices_requested
; i
++){
276 temp_array
[i
] = lrand48() % (crush
.get_max_devices());
279 trial_placement
.assign(temp_array
, temp_array
+ devices_requested
);
280 accept_placement
= check_valid_placement(ruleno
, trial_placement
, weight
);
282 } while (accept_placement
== false && attempted_tries
< max_tries
);
284 // save our random placement to the out vector
285 if (accept_placement
)
286 out
.assign(trial_placement
.begin(), trial_placement
.end());
289 else if (attempted_tries
== max_tries
)
295 void CrushTester::write_integer_indexed_vector_data_string(vector
<string
> &dst
, int index
, vector
<int> vector_data
)
297 stringstream
data_buffer (stringstream::in
| stringstream::out
);
298 unsigned input_size
= vector_data
.size();
300 // pass the indexing variable to the data buffer
301 data_buffer
<< index
;
303 // pass the rest of the input data to the buffer
304 for (unsigned i
= 0; i
< input_size
; i
++) {
305 data_buffer
<< ',' << vector_data
[i
];
308 data_buffer
<< std::endl
;
310 // write the data buffer to the destination
311 dst
.push_back( data_buffer
.str() );
314 void CrushTester::write_integer_indexed_vector_data_string(vector
<string
> &dst
, int index
, vector
<float> vector_data
)
316 stringstream
data_buffer (stringstream::in
| stringstream::out
);
317 unsigned input_size
= vector_data
.size();
319 // pass the indexing variable to the data buffer
320 data_buffer
<< index
;
322 // pass the rest of the input data to the buffer
323 for (unsigned i
= 0; i
< input_size
; i
++) {
324 data_buffer
<< ',' << vector_data
[i
];
327 data_buffer
<< std::endl
;
329 // write the data buffer to the destination
330 dst
.push_back( data_buffer
.str() );
333 void CrushTester::write_integer_indexed_scalar_data_string(vector
<string
> &dst
, int index
, int scalar_data
)
335 stringstream
data_buffer (stringstream::in
| stringstream::out
);
337 // pass the indexing variable to the data buffer
338 data_buffer
<< index
;
340 // pass the input data to the buffer
341 data_buffer
<< ',' << scalar_data
;
342 data_buffer
<< std::endl
;
344 // write the data buffer to the destination
345 dst
.push_back( data_buffer
.str() );
347 void CrushTester::write_integer_indexed_scalar_data_string(vector
<string
> &dst
, int index
, float scalar_data
)
349 stringstream
data_buffer (stringstream::in
| stringstream::out
);
351 // pass the indexing variable to the data buffer
352 data_buffer
<< index
;
354 // pass the input data to the buffer
355 data_buffer
<< ',' << scalar_data
;
356 data_buffer
<< std::endl
;
358 // write the data buffer to the destination
359 dst
.push_back( data_buffer
.str() );
362 int CrushTester::test_with_crushtool(const char *crushtool_cmd
,
363 int max_id
, int timeout
,
366 SubProcessTimed
crushtool(crushtool_cmd
, SubProcess::PIPE
, SubProcess::CLOSE
, SubProcess::PIPE
, timeout
);
367 string opt_max_id
= boost::lexical_cast
<string
>(max_id
);
368 crushtool
.add_cmd_args(
370 "--test", "--check", opt_max_id
.c_str(),
375 crushtool
.add_cmd_args(
377 stringify(ruleset
).c_str(),
380 int ret
= crushtool
.spawn();
382 err
<< "failed run crushtool: " << crushtool
.err();
387 ::encode(crush
, bl
, CEPH_FEATURES_SUPPORTED_DEFAULT
);
388 bl
.write_fd(crushtool
.get_stdin());
389 crushtool
.close_stdin();
391 ret
= bl
.read_fd(crushtool
.get_stderr(), 100 * 1024);
393 err
<< "failed read from crushtool: " << cpp_strerror(-ret
);
396 bl
.write_stream(err
);
397 if (crushtool
.join() != 0) {
398 err
<< crushtool
.err();
406 class BadCrushMap
: public std::runtime_error
{
409 BadCrushMap(const char* msg
, int id
)
410 : std::runtime_error(msg
), item(id
) {}
412 // throws if any node in the crush fail to print
413 class CrushWalker
: public CrushTreeDumper::Dumper
<void> {
414 typedef void DumbFormatter
;
415 typedef CrushTreeDumper::Dumper
<DumbFormatter
> Parent
;
418 CrushWalker(const CrushWrapper
*crush
, unsigned max_id
)
419 : Parent(crush
), max_id(max_id
) {}
420 void dump_item(const CrushTreeDumper::Item
&qi
, DumbFormatter
*) override
{
422 if (qi
.is_bucket()) {
423 if (!crush
->get_item_name(qi
.id
)) {
424 throw BadCrushMap("unknown item name", qi
.id
);
426 type
= crush
->get_bucket_type(qi
.id
);
428 if (max_id
> 0 && qi
.id
>= max_id
) {
429 throw BadCrushMap("item id too large", qi
.id
);
433 if (!crush
->get_type_name(type
)) {
434 throw BadCrushMap("unknown type name", qi
.id
);
440 bool CrushTester::check_name_maps(unsigned max_id
) const
442 CrushWalker
crush_walker(&crush
, max_id
);
444 // walk through the crush, to see if its self-contained
445 crush_walker
.dump(NULL
);
446 // and see if the maps is also able to handle straying OSDs, whose id >= 0.
447 // "ceph osd tree" will try to print them, even they are not listed in the
449 crush_walker
.dump_item(CrushTreeDumper::Item(0, 0, 0), NULL
);
450 } catch (const BadCrushMap
& e
) {
451 err
<< e
.what() << ": item#" << e
.item
<< std::endl
;
457 static string
get_rule_name(CrushWrapper
& crush
, int rule
)
459 if (crush
.get_rule_name(rule
))
460 return crush
.get_rule_name(rule
);
462 return string("rule") + std::to_string(rule
);
465 void CrushTester::check_overlapped_rules() const
467 namespace icl
= boost::icl
;
468 typedef std::set
<string
> RuleNames
;
469 typedef icl::interval_map
<int, RuleNames
> Rules
;
470 // <ruleset, type> => interval_map<size, {names}>
471 typedef std::map
<std::pair
<int, int>, Rules
> RuleSets
;
472 using interval
= icl::interval
<int>;
474 // mimic the logic of crush_find_rule(), but it only return the first matched
475 // one, but I am collecting all of them by the overlapped sizes.
477 for (int rule
= 0; rule
< crush
.get_max_rules(); rule
++) {
478 if (!crush
.rule_exists(rule
)) {
481 Rules
& rules
= rulesets
[{crush
.get_rule_mask_ruleset(rule
),
482 crush
.get_rule_mask_type(rule
)}];
483 rules
+= make_pair(interval::closed(crush
.get_rule_mask_min_size(rule
),
484 crush
.get_rule_mask_max_size(rule
)),
485 RuleNames
{get_rule_name(crush
, rule
)});
487 for (auto i
: rulesets
) {
488 auto ruleset_type
= i
.first
;
489 const Rules
& rules
= i
.second
;
490 for (auto r
: rules
) {
491 const RuleNames
& names
= r
.second
;
492 // if there are more than one rules covering the same size range,
494 if (names
.size() > 1) {
495 err
<< "overlapped rules in ruleset " << ruleset_type
.first
<< ": "
496 << boost::join(names
, ", ") << "\n";
502 int CrushTester::test()
504 if (min_rule
< 0 || max_rule
< 0) {
506 max_rule
= crush
.get_max_rules() - 1;
508 if (min_x
< 0 || max_x
< 0) {
513 // initial osd weights
514 vector
<__u32
> weight
;
517 * note device weight is set by crushtool
518 * (likely due to a given a command line option)
520 for (int o
= 0; o
< crush
.get_max_devices(); o
++) {
521 if (device_weight
.count(o
)) {
522 weight
.push_back(device_weight
[o
]);
523 } else if (crush
.check_item_present(o
)) {
524 weight
.push_back(0x10000);
530 if (output_utilization_all
)
531 err
<< "devices weights (hex): " << hex
<< weight
<< dec
<< std::endl
;
534 adjust_weights(weight
);
537 int num_devices_active
= 0;
538 for (vector
<__u32
>::iterator p
= weight
.begin(); p
!= weight
.end(); ++p
)
540 num_devices_active
++;
542 if (output_choose_tries
)
543 crush
.start_choose_profile();
545 for (int r
= min_rule
; r
< crush
.get_max_rules() && r
<= max_rule
; r
++) {
546 if (!crush
.rule_exists(r
)) {
547 if (output_statistics
)
548 err
<< "rule " << r
<< " dne" << std::endl
;
552 crush
.get_rule_mask_ruleset(r
) != ruleset
) {
555 int minr
= min_rep
, maxr
= max_rep
;
556 if (min_rep
< 0 || max_rep
< 0) {
557 minr
= crush
.get_rule_mask_min_size(r
);
558 maxr
= crush
.get_rule_mask_max_size(r
);
561 if (output_statistics
)
562 err
<< "rule " << r
<< " (" << crush
.get_rule_name(r
)
563 << "), x = " << min_x
<< ".." << max_x
564 << ", numrep = " << minr
<< ".." << maxr
567 for (int nr
= minr
; nr
<= maxr
; nr
++) {
568 vector
<int> per(crush
.get_max_devices());
571 int num_objects
= ((max_x
- min_x
) + 1);
572 float num_devices
= (float) per
.size(); // get the total number of devices, better to cast as a float here
574 // create a structure to hold data for post-processing
575 tester_data_set tester_data
;
576 vector
<float> vector_data_buffer_f
;
578 // create a map to hold batch-level placement information
579 map
<int, vector
<int> > batch_per
;
580 int objects_per_batch
= num_objects
/ num_batches
;
581 int batch_min
= min_x
;
582 int batch_max
= min_x
+ objects_per_batch
- 1;
584 // get the total weight of the system
585 int total_weight
= 0;
586 for (unsigned i
= 0; i
< per
.size(); i
++)
587 total_weight
+= weight
[i
];
589 if (total_weight
== 0)
592 // compute the expected number of objects stored per device in the absence of weighting
593 float expected_objects
= min(nr
, get_maximum_affected_by_rule(r
)) * num_objects
;
595 // compute each device's proportional weight
596 vector
<float> proportional_weights( per
.size() );
598 for (unsigned i
= 0; i
< per
.size(); i
++)
599 proportional_weights
[i
] = (float) weight
[i
] / (float) total_weight
;
601 if (output_data_file
) {
602 // stage the absolute weight information for post-processing
603 for (unsigned i
= 0; i
< per
.size(); i
++) {
604 tester_data
.absolute_weights
[i
] = (float) weight
[i
] / (float)0x10000;
607 // stage the proportional weight information for post-processing
608 for (unsigned i
= 0; i
< per
.size(); i
++) {
609 if (proportional_weights
[i
] > 0 )
610 tester_data
.proportional_weights
[i
] = proportional_weights
[i
];
612 tester_data
.proportional_weights_all
[i
] = proportional_weights
[i
];
616 // compute the expected number of objects stored per device when a device's weight is considered
617 vector
<float> num_objects_expected(num_devices
);
619 for (unsigned i
= 0; i
< num_devices
; i
++)
620 num_objects_expected
[i
] = (proportional_weights
[i
]*expected_objects
);
622 for (int current_batch
= 0; current_batch
< num_batches
; current_batch
++) {
623 if (current_batch
== (num_batches
- 1)) {
625 objects_per_batch
= (batch_max
- batch_min
+ 1);
628 float batch_expected_objects
= min(nr
, get_maximum_affected_by_rule(r
)) * objects_per_batch
;
629 vector
<float> batch_num_objects_expected( per
.size() );
631 for (unsigned i
= 0; i
< per
.size() ; i
++)
632 batch_num_objects_expected
[i
] = (proportional_weights
[i
]*batch_expected_objects
);
634 // create a vector to hold placement results temporarily
635 vector
<int> temporary_per ( per
.size() );
637 for (int x
= batch_min
; x
<= batch_max
; x
++) {
638 // create a vector to hold the results of a CRUSH placement or RNG simulation
643 err
<< "CRUSH"; // prepend CRUSH to placement output
646 real_x
= crush_hash32_2(CRUSH_HASH_RJENKINS1
, x
, (uint32_t)pool_id
);
648 crush
.do_rule(r
, real_x
, out
, nr
, weight
, 0);
651 err
<< "RNG"; // prepend RNG to placement output to denote simulation
652 // test our new monte carlo placement generator
653 random_placement(r
, out
, nr
, weight
);
657 err
<< " rule " << r
<< " x " << x
<< " " << out
<< std::endl
;
659 if (output_data_file
)
660 write_integer_indexed_vector_data_string(tester_data
.placement_information
, x
, out
);
662 bool has_item_none
= false;
663 for (unsigned i
= 0; i
< out
.size(); i
++) {
664 if (out
[i
] != CRUSH_ITEM_NONE
) {
666 temporary_per
[out
[i
]]++;
668 has_item_none
= true;
672 batch_per
[current_batch
] = temporary_per
;
674 if (output_bad_mappings
&&
675 (out
.size() != (unsigned)nr
||
677 err
<< "bad mapping rule " << r
<< " x " << x
<< " num_rep " << nr
<< " result " << out
<< std::endl
;
681 batch_min
= batch_max
+ 1;
682 batch_max
= batch_min
+ objects_per_batch
- 1;
685 for (unsigned i
= 0; i
< per
.size(); i
++)
686 if (output_utilization
&& !output_statistics
)
687 err
<< " device " << i
688 << ":\t" << per
[i
] << std::endl
;
690 for (map
<int,int>::iterator p
= sizes
.begin(); p
!= sizes
.end(); ++p
)
691 if (output_statistics
)
692 err
<< "rule " << r
<< " (" << crush
.get_rule_name(r
) << ") num_rep " << nr
693 << " result size == " << p
->first
<< ":\t"
694 << p
->second
<< "/" << (max_x
-min_x
+1) << std::endl
;
696 if (output_statistics
)
697 for (unsigned i
= 0; i
< per
.size(); i
++) {
698 if (output_utilization
) {
699 if (num_objects_expected
[i
] > 0 && per
[i
] > 0) {
700 err
<< " device " << i
<< ":\t"
701 << "\t" << " stored " << ": " << per
[i
]
702 << "\t" << " expected " << ": " << num_objects_expected
[i
]
705 } else if (output_utilization_all
) {
706 err
<< " device " << i
<< ":\t"
707 << "\t" << " stored " << ": " << per
[i
]
708 << "\t" << " expected " << ": " << num_objects_expected
[i
]
713 if (output_data_file
)
714 for (unsigned i
= 0; i
< per
.size(); i
++) {
715 vector_data_buffer_f
.clear();
716 vector_data_buffer_f
.push_back( (float) per
[i
]);
717 vector_data_buffer_f
.push_back( (float) num_objects_expected
[i
]);
719 write_integer_indexed_vector_data_string(tester_data
.device_utilization_all
, i
, vector_data_buffer_f
);
721 if (num_objects_expected
[i
] > 0 && per
[i
] > 0)
722 write_integer_indexed_vector_data_string(tester_data
.device_utilization
, i
, vector_data_buffer_f
);
725 if (output_data_file
&& num_batches
> 1) {
726 // stage batch utilization information for post-processing
727 for (int i
= 0; i
< num_batches
; i
++) {
728 write_integer_indexed_vector_data_string(tester_data
.batch_device_utilization_all
, i
, batch_per
[i
]);
729 write_integer_indexed_vector_data_string(tester_data
.batch_device_expected_utilization_all
, i
, batch_per
[i
]);
733 string rule_tag
= crush
.get_rule_name(r
);
736 write_data_set_to_csv(output_data_file_name
+rule_tag
,tester_data
);
740 if (output_choose_tries
) {
742 int n
= crush
.get_choose_profile(&v
);
743 for (int i
=0; i
<n
; i
++) {
744 cout
.setf(std::ios::right
);
746 << i
<< ": " << std::setw(9) << v
[i
];
747 cout
.unsetf(std::ios::right
);
751 crush
.stop_choose_profile();