2 #include "CrushCompiler.h"
12 #include "common/errno.h"
13 #include <boost/algorithm/string.hpp>
17 static void print_type_name(ostream
& out
, int t
, CrushWrapper
&crush
)
19 const char *name
= crush
.get_type_name(t
);
28 static void print_item_name(ostream
& out
, int t
, CrushWrapper
&crush
)
30 const char *name
= crush
.get_item_name(t
);
36 out
<< "bucket" << (-1-t
);
39 static void print_bucket_class_ids(ostream
& out
, int t
, CrushWrapper
&crush
)
41 if (crush
.class_bucket
.count(t
) == 0)
43 auto &class_to_id
= crush
.class_bucket
[t
];
44 for (auto &i
: class_to_id
) {
47 const char* class_name
= crush
.get_class_name(c
);
49 out
<< "\tid " << cid
<< " class " << class_name
<< "\t\t# do not change unnecessarily\n";
53 static void print_item_class(ostream
& out
, int t
, CrushWrapper
&crush
)
55 const char *c
= crush
.get_item_class(t
);
57 out
<< " class " << c
;
60 static void print_class(ostream
& out
, int t
, CrushWrapper
&crush
)
62 const char *c
= crush
.get_class_name(t
);
64 out
<< " class " << c
;
66 out
<< " # unexpected class " << t
;
69 static void print_rule_name(ostream
& out
, int t
, CrushWrapper
&crush
)
71 const char *name
= crush
.get_rule_name(t
);
78 static void print_fixedpoint(ostream
& out
, int i
)
81 snprintf(s
, sizeof(s
), "%.3f", (float)i
/ (float)0x10000);
85 int CrushCompiler::decompile_bucket_impl(int i
, ostream
&out
)
87 const char *name
= crush
.get_item_name(i
);
88 if (name
&& !crush
.is_valid_crush_name(name
))
90 int type
= crush
.get_bucket_type(i
);
91 print_type_name(out
, type
, crush
);
93 print_item_name(out
, i
, crush
);
95 out
<< "\tid " << i
<< "\t\t# do not change unnecessarily\n";
96 print_bucket_class_ids(out
, i
, crush
);
99 print_fixedpoint(out
, crush
.get_bucket_weight(i
));
102 int n
= crush
.get_bucket_size(i
);
104 int alg
= crush
.get_bucket_alg(i
);
105 out
<< "\talg " << crush_bucket_alg_name(alg
);
107 // notate based on alg type
110 case CRUSH_BUCKET_UNIFORM
:
111 out
<< "\t# do not change bucket size (" << n
<< ") unnecessarily";
114 case CRUSH_BUCKET_LIST
:
115 out
<< "\t# add new items at the end; do not change order unnecessarily";
117 case CRUSH_BUCKET_TREE
:
118 out
<< "\t# do not change pos for existing items unnecessarily";
124 int hash
= crush
.get_bucket_hash(i
);
125 out
<< "\thash " << hash
<< "\t# " << crush_hash_name(hash
) << "\n";
127 for (int j
=0; j
<n
; j
++) {
128 int item
= crush
.get_bucket_item(i
, j
);
129 int w
= crush
.get_bucket_item_weight(i
, j
);
131 print_item_name(out
, item
, crush
);
133 print_fixedpoint(out
, w
);
143 /* Basically, we just descend recursively into all of the buckets,
144 * executing a depth-first traversal of the graph. Since the buckets form a
145 * directed acyclic graph, this should work just fine. The graph isn't
146 * necessarily a tree, so we have to keep track of what buckets we already
147 * outputted. We don't want to output anything twice. We also keep track of
148 * what buckets are in progress so that we can detect cycles. These can
149 * arise through user error.
151 int CrushCompiler::decompile_bucket(int cur
,
152 std::map
<int, dcb_state_t
>& dcb_states
,
155 if ((cur
== 0) || (!crush
.bucket_exists(cur
)))
158 std::map
<int, dcb_state_t
>::iterator c
= dcb_states
.find(cur
);
159 if (c
== dcb_states
.end()) {
160 // Mark this bucket as "in progress."
161 std::map
<int, dcb_state_t
>::value_type
val(cur
, DCB_STATE_IN_PROGRESS
);
162 std::pair
<std::map
<int, dcb_state_t
>::iterator
, bool> rval
163 (dcb_states
.insert(val
));
167 else if (c
->second
== DCB_STATE_DONE
) {
168 // We already did this bucket.
171 else if (c
->second
== DCB_STATE_IN_PROGRESS
) {
172 err
<< "decompile_crush_bucket: logic error: tried to decompile "
173 "a bucket that is already being decompiled" << std::endl
;
177 err
<< "decompile_crush_bucket: logic error: illegal bucket state! "
178 << c
->second
<< std::endl
;
182 int bsize
= crush
.get_bucket_size(cur
);
183 for (int i
= 0; i
< bsize
; ++i
) {
184 int item
= crush
.get_bucket_item(cur
, i
);
185 std::map
<int, dcb_state_t
>::iterator d
= dcb_states
.find(item
);
186 if (d
== dcb_states
.end()) {
187 int ret
= decompile_bucket(item
, dcb_states
, out
);
191 else if (d
->second
== DCB_STATE_IN_PROGRESS
) {
192 err
<< "decompile_crush_bucket: error: while trying to output bucket "
193 << cur
<< ", we found out that it contains one of the buckets that "
194 << "contain it. This is not allowed. The buckets must form a "
195 << "directed acyclic graph." << std::endl
;
198 else if (d
->second
!= DCB_STATE_DONE
) {
199 err
<< "decompile_crush_bucket: logic error: illegal bucket state "
200 << d
->second
<< std::endl
;
204 decompile_bucket_impl(cur
, out
);
205 c
->second
= DCB_STATE_DONE
;
209 int CrushCompiler::decompile_weight_set_weights(crush_weight_set weight_set
,
213 for (__u32 i
= 0; i
< weight_set
.size
; i
++) {
214 print_fixedpoint(out
, weight_set
.weights
[i
]);
221 int CrushCompiler::decompile_weight_set(crush_weight_set
*weight_set
,
225 out
<< " weight_set [\n";
226 for (__u32 i
= 0; i
< size
; i
++) {
227 int r
= decompile_weight_set_weights(weight_set
[i
], out
);
235 int CrushCompiler::decompile_ids(__s32
*ids
,
240 for (__u32 i
= 0; i
< size
; i
++)
241 out
<< ids
[i
] << " ";
246 int CrushCompiler::decompile_choose_arg(crush_choose_arg
*arg
,
252 out
<< " bucket_id " << bucket_id
<< "\n";
253 if (arg
->weight_set_size
> 0) {
254 r
= decompile_weight_set(arg
->weight_set
, arg
->weight_set_size
, out
);
258 if (arg
->ids_size
> 0) {
259 r
= decompile_ids(arg
->ids
, arg
->ids_size
, out
);
267 int CrushCompiler::decompile_choose_arg_map(crush_choose_arg_map arg_map
,
270 for (__u32 i
= 0; i
< arg_map
.size
; i
++) {
271 if ((arg_map
.args
[i
].ids_size
== 0) &&
272 (arg_map
.args
[i
].weight_set_size
== 0))
274 int r
= decompile_choose_arg(&arg_map
.args
[i
], -1-i
, out
);
281 int CrushCompiler::decompile_choose_args(const std::pair
<const long unsigned int, crush_choose_arg_map
> &i
,
284 out
<< "choose_args " << i
.first
<< " {\n";
285 int r
= decompile_choose_arg_map(i
.second
, out
);
292 int CrushCompiler::decompile(ostream
&out
)
294 crush
.cleanup_classes();
296 out
<< "# begin crush map\n";
298 // only dump tunables if they differ from the defaults
299 if (crush
.get_choose_local_tries() != 2)
300 out
<< "tunable choose_local_tries " << crush
.get_choose_local_tries() << "\n";
301 if (crush
.get_choose_local_fallback_tries() != 5)
302 out
<< "tunable choose_local_fallback_tries " << crush
.get_choose_local_fallback_tries() << "\n";
303 if (crush
.get_choose_total_tries() != 19)
304 out
<< "tunable choose_total_tries " << crush
.get_choose_total_tries() << "\n";
305 if (crush
.get_chooseleaf_descend_once() != 0)
306 out
<< "tunable chooseleaf_descend_once " << crush
.get_chooseleaf_descend_once() << "\n";
307 if (crush
.get_chooseleaf_vary_r() != 0)
308 out
<< "tunable chooseleaf_vary_r " << crush
.get_chooseleaf_vary_r() << "\n";
309 if (crush
.get_chooseleaf_stable() != 0)
310 out
<< "tunable chooseleaf_stable " << crush
.get_chooseleaf_stable() << "\n";
311 if (crush
.get_straw_calc_version() != 0)
312 out
<< "tunable straw_calc_version " << crush
.get_straw_calc_version() << "\n";
313 if (crush
.get_allowed_bucket_algs() != CRUSH_LEGACY_ALLOWED_BUCKET_ALGS
)
314 out
<< "tunable allowed_bucket_algs " << crush
.get_allowed_bucket_algs()
317 out
<< "\n# devices\n";
318 for (int i
=0; i
<crush
.get_max_devices(); i
++) {
319 out
<< "device " << i
<< " ";
320 print_item_name(out
, i
, crush
);
321 print_item_class(out
, i
, crush
);
325 out
<< "\n# types\n";
326 int n
= crush
.get_num_type_names();
327 for (int i
=0; n
; i
++) {
328 const char *name
= crush
.get_type_name(i
);
330 if (i
== 0) out
<< "type 0 osd\n";
334 out
<< "type " << i
<< " " << name
<< "\n";
337 out
<< "\n# buckets\n";
338 std::map
<int, dcb_state_t
> dcb_states
;
339 for (int bucket
= -1; bucket
> -1-crush
.get_max_buckets(); --bucket
) {
340 int ret
= decompile_bucket(bucket
, dcb_states
, out
);
345 out
<< "\n# rules\n";
346 for (int i
=0; i
<crush
.get_max_rules(); i
++) {
347 if (!crush
.rule_exists(i
))
350 if (crush
.get_rule_name(i
))
351 print_rule_name(out
, i
, crush
);
353 out
<< "\truleset " << crush
.get_rule_mask_ruleset(i
) << "\n";
355 switch (crush
.get_rule_mask_type(i
)) {
356 case CEPH_PG_TYPE_REPLICATED
:
357 out
<< "\ttype replicated\n";
359 case CEPH_PG_TYPE_ERASURE
:
360 out
<< "\ttype erasure\n";
363 out
<< "\ttype " << crush
.get_rule_mask_type(i
) << "\n";
366 out
<< "\tmin_size " << crush
.get_rule_mask_min_size(i
) << "\n";
367 out
<< "\tmax_size " << crush
.get_rule_mask_max_size(i
) << "\n";
369 for (int j
=0; j
<crush
.get_rule_len(i
); j
++) {
370 switch (crush
.get_rule_op(i
, j
)) {
371 case CRUSH_RULE_NOOP
:
372 out
<< "\tstep noop\n";
374 case CRUSH_RULE_TAKE
:
375 out
<< "\tstep take ";
377 int step_item
= crush
.get_rule_arg1(i
, j
);
380 int res
= crush
.split_id_class(step_item
, &original_item
, &c
);
384 step_item
= original_item
;
385 print_item_name(out
, step_item
, crush
);
387 print_class(out
, c
, crush
);
391 case CRUSH_RULE_EMIT
:
392 out
<< "\tstep emit\n";
394 case CRUSH_RULE_SET_CHOOSE_TRIES
:
395 out
<< "\tstep set_choose_tries " << crush
.get_rule_arg1(i
, j
)
398 case CRUSH_RULE_SET_CHOOSE_LOCAL_TRIES
:
399 out
<< "\tstep set_choose_local_tries " << crush
.get_rule_arg1(i
, j
)
402 case CRUSH_RULE_SET_CHOOSE_LOCAL_FALLBACK_TRIES
:
403 out
<< "\tstep set_choose_local_fallback_tries " << crush
.get_rule_arg1(i
, j
)
406 case CRUSH_RULE_SET_CHOOSELEAF_TRIES
:
407 out
<< "\tstep set_chooseleaf_tries " << crush
.get_rule_arg1(i
, j
)
410 case CRUSH_RULE_SET_CHOOSELEAF_VARY_R
:
411 out
<< "\tstep set_chooseleaf_vary_r " << crush
.get_rule_arg1(i
, j
)
414 case CRUSH_RULE_SET_CHOOSELEAF_STABLE
:
415 out
<< "\tstep set_chooseleaf_stable " << crush
.get_rule_arg1(i
, j
)
418 case CRUSH_RULE_CHOOSE_FIRSTN
:
419 out
<< "\tstep choose firstn "
420 << crush
.get_rule_arg1(i
, j
)
422 print_type_name(out
, crush
.get_rule_arg2(i
, j
), crush
);
425 case CRUSH_RULE_CHOOSE_INDEP
:
426 out
<< "\tstep choose indep "
427 << crush
.get_rule_arg1(i
, j
)
429 print_type_name(out
, crush
.get_rule_arg2(i
, j
), crush
);
432 case CRUSH_RULE_CHOOSELEAF_FIRSTN
:
433 out
<< "\tstep chooseleaf firstn "
434 << crush
.get_rule_arg1(i
, j
)
436 print_type_name(out
, crush
.get_rule_arg2(i
, j
), crush
);
439 case CRUSH_RULE_CHOOSELEAF_INDEP
:
440 out
<< "\tstep chooseleaf indep "
441 << crush
.get_rule_arg1(i
, j
)
443 print_type_name(out
, crush
.get_rule_arg2(i
, j
), crush
);
450 if (crush
.choose_args
.size() > 0) {
451 out
<< "\n# choose_args\n";
452 for (auto i
: crush
.choose_args
) {
453 int ret
= decompile_choose_args(i
, out
);
458 out
<< "\n# end crush map" << std::endl
;
463 // ================================================================
465 string
CrushCompiler::string_node(node_t
&node
)
467 return boost::trim_copy(string(node
.value
.begin(), node
.value
.end()));
470 int CrushCompiler::int_node(node_t
&node
)
472 string str
= string_node(node
);
473 return strtol(str
.c_str(), 0, 10);
476 float CrushCompiler::float_node(node_t
&node
)
478 string s
= string_node(node
);
479 return strtof(s
.c_str(), 0);
482 int CrushCompiler::parse_device(iter_t
const& i
)
484 int id
= int_node(i
->children
[1]);
486 string name
= string_node(i
->children
[2]);
487 crush
.set_item_name(id
, name
.c_str());
488 if (item_id
.count(name
)) {
489 err
<< "item " << name
<< " defined twice" << std::endl
;
495 if (verbose
) err
<< "device " << id
<< " '" << name
<< "'";
497 if (i
->children
.size() > 3) {
498 string c
= string_node(i
->children
[4]);
499 crush
.set_item_class(id
, c
);
500 if (verbose
) err
<< " class" << " '" << c
<< "'" << std::endl
;
502 if (verbose
) err
<< std::endl
;
507 int CrushCompiler::parse_tunable(iter_t
const& i
)
509 string name
= string_node(i
->children
[1]);
510 int val
= int_node(i
->children
[2]);
512 if (name
== "choose_local_tries")
513 crush
.set_choose_local_tries(val
);
514 else if (name
== "choose_local_fallback_tries")
515 crush
.set_choose_local_fallback_tries(val
);
516 else if (name
== "choose_total_tries")
517 crush
.set_choose_total_tries(val
);
518 else if (name
== "chooseleaf_descend_once")
519 crush
.set_chooseleaf_descend_once(val
);
520 else if (name
== "chooseleaf_vary_r")
521 crush
.set_chooseleaf_vary_r(val
);
522 else if (name
== "chooseleaf_stable")
523 crush
.set_chooseleaf_stable(val
);
524 else if (name
== "straw_calc_version")
525 crush
.set_straw_calc_version(val
);
526 else if (name
== "allowed_bucket_algs")
527 crush
.set_allowed_bucket_algs(val
);
529 err
<< "tunable " << name
<< " not recognized" << std::endl
;
535 current crop of tunables are all now "safe". re-enable this when we
536 add new ones that are ... new.
538 if (!unsafe_tunables) {
539 err << "tunables are NOT FULLY IMPLEMENTED; enable with --enable-unsafe-tunables to enable this feature" << std::endl;
544 if (verbose
) err
<< "tunable " << name
<< " " << val
<< std::endl
;
548 int CrushCompiler::parse_bucket_type(iter_t
const& i
)
550 int id
= int_node(i
->children
[1]);
551 string name
= string_node(i
->children
[2]);
552 if (verbose
) err
<< "type " << id
<< " '" << name
<< "'" << std::endl
;
554 crush
.set_type_name(id
, name
.c_str());
558 int CrushCompiler::parse_bucket(iter_t
const& i
)
560 string tname
= string_node(i
->children
[0]);
561 if (!type_id
.count(tname
)) {
562 err
<< "bucket type '" << tname
<< "' is not defined" << std::endl
;
565 int type
= type_id
[tname
];
567 string name
= string_node(i
->children
[1]);
568 if (item_id
.count(name
)) {
569 err
<< "bucket or device '" << name
<< "' is already defined" << std::endl
;
573 int id
= 0; // none, yet!
578 map
<int32_t, int32_t> class_id
;
580 for (unsigned p
=3; p
<i
->children
.size()-1; p
++) {
581 iter_t sub
= i
->children
.begin() + p
;
582 string tag
= string_node(sub
->children
[0]);
583 //err << "tag " << tag << std::endl;
585 int maybe_id
= int_node(sub
->children
[1]);
586 if (verbose
) err
<< "bucket " << name
<< " id " << maybe_id
;
587 if (sub
->children
.size() > 2) {
588 string class_name
= string_node(sub
->children
[3]);
589 if (!crush
.class_exists(class_name
)) {
590 err
<< " unknown device class '" << class_name
<< "'" << std::endl
;
593 int cid
= crush
.get_class_id(class_name
);
594 if (class_id
.count(cid
) != 0) {
595 err
<< "duplicate device class " << class_name
<< " for bucket " << name
<< std::endl
;
598 class_id
[cid
] = maybe_id
;
599 if (verbose
) err
<< " class" << " '" << class_name
<< "'" << std::endl
;
602 if (verbose
) err
<< std::endl
;
604 } else if (tag
== "alg") {
605 string a
= string_node(sub
->children
[1]);
607 alg
= CRUSH_BUCKET_UNIFORM
;
608 else if (a
== "list")
609 alg
= CRUSH_BUCKET_LIST
;
610 else if (a
== "tree")
611 alg
= CRUSH_BUCKET_TREE
;
612 else if (a
== "straw")
613 alg
= CRUSH_BUCKET_STRAW
;
614 else if (a
== "straw2")
615 alg
= CRUSH_BUCKET_STRAW2
;
617 err
<< "unknown bucket alg '" << a
<< "'" << std::endl
<< std::endl
;
621 else if (tag
== "hash") {
622 string a
= string_node(sub
->children
[1]);
623 if (a
== "rjenkins1")
624 hash
= CRUSH_HASH_RJENKINS1
;
626 hash
= atoi(a
.c_str());
628 else if (tag
== "item") {
629 // first, just determine which item pos's are already used
631 for (unsigned q
= 2; q
< sub
->children
.size(); q
++) {
632 string tag
= string_node(sub
->children
[q
++]);
634 int pos
= int_node(sub
->children
[q
]);
635 if (used_items
.count(pos
)) {
636 err
<< "item '" << string_node(sub
->children
[1]) << "' in bucket '" << name
<< "' has explicit pos " << pos
<< ", which is occupied" << std::endl
;
639 used_items
.insert(pos
);
647 if (!used_items
.empty())
648 size
= MAX(size
, *used_items
.rbegin());
649 vector
<int> items(size
);
650 vector
<int> weights(size
);
653 unsigned bucketweight
= 0;
654 bool have_uniform_weight
= false;
655 unsigned uniform_weight
= 0;
656 for (unsigned p
=3; p
<i
->children
.size()-1; p
++) {
657 iter_t sub
= i
->children
.begin() + p
;
658 string tag
= string_node(sub
->children
[0]);
661 string iname
= string_node(sub
->children
[1]);
662 if (!item_id
.count(iname
)) {
663 err
<< "item '" << iname
<< "' in bucket '" << name
<< "' is not defined" << std::endl
;
666 int itemid
= item_id
[iname
];
668 unsigned weight
= 0x10000;
669 if (item_weight
.count(itemid
))
670 weight
= item_weight
[itemid
];
673 for (unsigned q
= 2; q
< sub
->children
.size(); q
++) {
674 string tag
= string_node(sub
->children
[q
++]);
675 if (tag
== "weight") {
676 weight
= float_node(sub
->children
[q
]) * (float)0x10000;
677 if (weight
> CRUSH_MAX_DEVICE_WEIGHT
&& itemid
>= 0) {
678 err
<< "device weight limited to " << CRUSH_MAX_DEVICE_WEIGHT
/ 0x10000 << std::endl
;
681 else if (weight
> CRUSH_MAX_BUCKET_WEIGHT
&& itemid
< 0) {
682 err
<< "bucket weight limited to " << CRUSH_MAX_BUCKET_WEIGHT
/ 0x10000
683 << " to prevent overflow" << std::endl
;
687 else if (tag
== "pos")
688 pos
= int_node(sub
->children
[q
]);
693 if (alg
== CRUSH_BUCKET_UNIFORM
) {
694 if (!have_uniform_weight
) {
695 have_uniform_weight
= true;
696 uniform_weight
= weight
;
698 if (uniform_weight
!= weight
) {
699 err
<< "item '" << iname
<< "' in uniform bucket '" << name
<< "' has weight " << weight
700 << " but previous item(s) have weight " << (float)uniform_weight
/(float)0x10000
701 << "; uniform bucket items must all have identical weights." << std::endl
;
708 err
<< "item '" << iname
<< "' in bucket '" << name
<< "' has pos " << pos
<< " >= size " << size
<< std::endl
;
712 while (used_items
.count(curpos
)) curpos
++;
715 //err << " item " << iname << " (" << itemid << ") pos " << pos << " weight " << weight << std::endl;
717 weights
[pos
] = weight
;
719 if (crush_addition_is_unsafe(bucketweight
, weight
)) {
720 err
<< "oh no! our bucket weights are overflowing all over the place, better lower the item weights" << std::endl
;
724 bucketweight
+= weight
;
729 for (id
=-1; id_item
.count(id
); id
--) ;
730 //err << "assigned id " << id << std::endl;
733 for (auto &i
: class_id
)
734 crush
.class_bucket
[id
][i
.first
] = i
.second
;
736 if (verbose
) err
<< "bucket " << name
<< " (" << id
<< ") " << size
<< " items and weight "
737 << (float)bucketweight
/ (float)0x10000 << std::endl
;
740 item_weight
[id
] = bucketweight
;
743 int r
= crush
.add_bucket(id
, alg
, hash
, type
, size
, &items
[0], &weights
[0], NULL
);
746 err
<< "Duplicate bucket id " << id
<< std::endl
;
748 err
<< "add_bucket failed " << cpp_strerror(r
) << std::endl
;
751 r
= crush
.set_item_name(id
, name
.c_str());
755 int CrushCompiler::parse_rule(iter_t
const& i
)
757 crush
.populate_classes();
759 int start
; // rule name is optional!
761 string rname
= string_node(i
->children
[1]);
763 if (rule_id
.count(rname
)) {
764 err
<< "rule name '" << rname
<< "' already defined\n" << std::endl
;
773 int ruleset
= int_node(i
->children
[start
]);
775 string tname
= string_node(i
->children
[start
+2]);
777 if (tname
== "replicated")
778 type
= CEPH_PG_TYPE_REPLICATED
;
779 else if (tname
== "erasure")
780 type
= CEPH_PG_TYPE_ERASURE
;
784 int minsize
= int_node(i
->children
[start
+4]);
785 int maxsize
= int_node(i
->children
[start
+6]);
787 int steps
= i
->children
.size() - start
- 8;
788 //err << "num steps " << steps << std::endl;
790 int ruleno
= crush
.add_rule(steps
, ruleset
, type
, minsize
, maxsize
, -1);
791 if (rname
.length()) {
792 crush
.set_rule_name(ruleno
, rname
.c_str());
793 rule_id
[rname
] = ruleno
;
797 for (iter_t p
= i
->children
.begin() + start
+ 7; step
< steps
; p
++) {
798 iter_t s
= p
->children
.begin() + 1;
799 int stepid
= s
->value
.id().to_long();
801 case crush_grammar::_step_take
:
803 string item
= string_node(s
->children
[1]);
804 if (!item_id
.count(item
)) {
805 err
<< "in rule '" << rname
<< "' item '" << item
<< "' not defined" << std::endl
;
808 int id
= item_id
[item
];
811 if (s
->children
.size() > 2) {
812 class_name
= string_node(s
->children
[3]);
813 c
= crush
.get_class_id(class_name
);
816 if (crush
.class_bucket
.count(id
) == 0) {
817 err
<< "in rule '" << rname
<< "' step take " << item
818 << " has no class information" << std::endl
;
821 if (crush
.class_bucket
[id
].count(c
) == 0) {
822 err
<< "in rule '" << rname
<< "' step take " << item
823 << " no matching bucket for class " << class_name
<< std::endl
;
826 id
= crush
.class_bucket
[id
][c
];
829 err
<< "rule " << rname
<< " take " << item
;
833 err
<< " remapped to " << crush
.get_item_name(id
) << std::endl
;
836 crush
.set_rule_step_take(ruleno
, step
++, id
);
840 case crush_grammar::_step_set_choose_tries
:
842 int val
= int_node(s
->children
[1]);
843 crush
.set_rule_step_set_choose_tries(ruleno
, step
++, val
);
847 case crush_grammar::_step_set_choose_local_tries
:
849 int val
= int_node(s
->children
[1]);
850 crush
.set_rule_step_set_choose_local_tries(ruleno
, step
++, val
);
854 case crush_grammar::_step_set_choose_local_fallback_tries
:
856 int val
= int_node(s
->children
[1]);
857 crush
.set_rule_step_set_choose_local_fallback_tries(ruleno
, step
++, val
);
861 case crush_grammar::_step_set_chooseleaf_tries
:
863 int val
= int_node(s
->children
[1]);
864 crush
.set_rule_step_set_chooseleaf_tries(ruleno
, step
++, val
);
868 case crush_grammar::_step_set_chooseleaf_vary_r
:
870 int val
= int_node(s
->children
[1]);
871 crush
.set_rule_step_set_chooseleaf_vary_r(ruleno
, step
++, val
);
875 case crush_grammar::_step_set_chooseleaf_stable
:
877 int val
= int_node(s
->children
[1]);
878 crush
.set_rule_step_set_chooseleaf_stable(ruleno
, step
++, val
);
882 case crush_grammar::_step_choose
:
883 case crush_grammar::_step_chooseleaf
:
885 string type
= string_node(s
->children
[4]);
886 if (!type_id
.count(type
)) {
887 err
<< "in rule '" << rname
<< "' type '" << type
<< "' not defined" << std::endl
;
890 string choose
= string_node(s
->children
[0]);
891 string mode
= string_node(s
->children
[1]);
892 if (choose
== "choose") {
893 if (mode
== "firstn")
894 crush
.set_rule_step_choose_firstn(ruleno
, step
++, int_node(s
->children
[2]), type_id
[type
]);
895 else if (mode
== "indep")
896 crush
.set_rule_step_choose_indep(ruleno
, step
++, int_node(s
->children
[2]), type_id
[type
]);
898 } else if (choose
== "chooseleaf") {
899 if (mode
== "firstn")
900 crush
.set_rule_step_choose_leaf_firstn(ruleno
, step
++, int_node(s
->children
[2]), type_id
[type
]);
901 else if (mode
== "indep")
902 crush
.set_rule_step_choose_leaf_indep(ruleno
, step
++, int_node(s
->children
[2]), type_id
[type
]);
908 case crush_grammar::_step_emit
:
909 crush
.set_rule_step_emit(ruleno
, step
++);
913 err
<< "bad crush step " << stepid
<< std::endl
;
917 assert(step
== steps
);
921 int CrushCompiler::parse_weight_set_weights(iter_t
const& i
, int bucket_id
, crush_weight_set
*weight_set
)
923 // -2 for the enclosing [ ]
924 __u32 size
= i
->children
.size() - 2;
925 __u32 bucket_size
= crush
.get_bucket_size(bucket_id
);
926 if (size
!= bucket_size
) {
927 err
<< bucket_id
<< " needs exactly " << bucket_size
928 << " weights but got " << size
<< std::endl
;
931 weight_set
->size
= size
;
932 weight_set
->weights
= (__u32
*)calloc(weight_set
->size
, sizeof(__u32
));
934 for (iter_t p
= i
->children
.begin() + 1; p
!= i
->children
.end(); p
++, pos
++)
936 weight_set
->weights
[pos
] = float_node(*p
) * (float)0x10000;
940 int CrushCompiler::parse_weight_set(iter_t
const& i
, int bucket_id
, crush_choose_arg
*arg
)
942 // -3 stands for the leading "weight_set" keyword and the enclosing [ ]
943 arg
->weight_set_size
= i
->children
.size() - 3;
944 arg
->weight_set
= (crush_weight_set
*)calloc(arg
->weight_set_size
, sizeof(crush_weight_set
));
946 for (iter_t p
= i
->children
.begin(); p
!= i
->children
.end(); p
++) {
948 switch((int)p
->value
.id().to_long()) {
949 case crush_grammar::_weight_set_weights
:
950 if (pos
< arg
->weight_set_size
) {
951 r
= parse_weight_set_weights(p
, bucket_id
, &arg
->weight_set
[pos
]);
954 err
<< "invalid weight_set syntax" << std::endl
;
964 int CrushCompiler::parse_choose_arg_ids(iter_t
const& i
, int bucket_id
, crush_choose_arg
*arg
)
966 // -3 for the leading "ids" keyword and the enclosing [ ]
967 __u32 size
= i
->children
.size() - 3;
968 __u32 bucket_size
= crush
.get_bucket_size(bucket_id
);
969 if (size
!= bucket_size
) {
970 err
<< bucket_id
<< " needs exactly " << bucket_size
971 << " ids but got " << size
<< std::endl
;
974 arg
->ids_size
= size
;
975 arg
->ids
= (__s32
*)calloc(arg
->ids_size
, sizeof(__s32
));
977 for (iter_t p
= i
->children
.begin() + 2; pos
< size
; p
++, pos
++)
978 arg
->ids
[pos
] = int_node(*p
);
982 int CrushCompiler::parse_choose_arg(iter_t
const& i
, crush_choose_arg
*args
)
984 int bucket_id
= int_node(i
->children
[2]);
985 if (-1-bucket_id
< 0 || -1-bucket_id
>= crush
.get_max_buckets()) {
986 err
<< bucket_id
<< " is out of range" << std::endl
;
989 if (!crush
.bucket_exists(bucket_id
)) {
990 err
<< bucket_id
<< " does not exist" << std::endl
;
993 crush_choose_arg
*arg
= &args
[-1-bucket_id
];
994 for (iter_t p
= i
->children
.begin(); p
!= i
->children
.end(); p
++) {
996 switch((int)p
->value
.id().to_long()) {
997 case crush_grammar::_weight_set
:
998 r
= parse_weight_set(p
, bucket_id
, arg
);
1000 case crush_grammar::_choose_arg_ids
:
1001 r
= parse_choose_arg_ids(p
, bucket_id
, arg
);
1010 int CrushCompiler::parse_choose_args(iter_t
const& i
)
1012 int choose_arg_index
= int_node(i
->children
[1]);
1013 if (crush
.choose_args
.find(choose_arg_index
) != crush
.choose_args
.end()) {
1014 err
<< choose_arg_index
<< " duplicated" << std::endl
;
1017 crush_choose_arg_map arg_map
;
1018 arg_map
.size
= crush
.get_max_buckets();
1019 arg_map
.args
= (crush_choose_arg
*)calloc(arg_map
.size
, sizeof(crush_choose_arg
));
1020 for (iter_t p
= i
->children
.begin() + 2; p
!= i
->children
.end(); p
++) {
1022 switch((int)p
->value
.id().to_long()) {
1023 case crush_grammar::_choose_arg
:
1024 r
= parse_choose_arg(p
, arg_map
.args
);
1028 crush
.destroy_choose_args(arg_map
);
1032 crush
.choose_args
[choose_arg_index
] = arg_map
;
1036 void CrushCompiler::find_used_bucket_ids(iter_t
const& i
)
1038 for (iter_t p
= i
->children
.begin(); p
!= i
->children
.end(); p
++) {
1039 if ((int)p
->value
.id().to_long() == crush_grammar::_bucket
) {
1040 iter_t firstline
= p
->children
.begin() + 3;
1041 string tag
= string_node(firstline
->children
[0]);
1043 int id
= int_node(firstline
->children
[1]);
1044 //err << "saw bucket id " << id << std::endl;
1045 id_item
[id
] = string();
1051 int CrushCompiler::parse_crush(iter_t
const& i
)
1053 find_used_bucket_ids(i
);
1055 for (iter_t p
= i
->children
.begin(); p
!= i
->children
.end(); p
++) {
1057 switch (p
->value
.id().to_long()) {
1058 case crush_grammar::_tunable
:
1059 r
= parse_tunable(p
);
1061 case crush_grammar::_device
:
1062 r
= parse_device(p
);
1064 case crush_grammar::_bucket_type
:
1065 r
= parse_bucket_type(p
);
1067 case crush_grammar::_bucket
:
1068 r
= parse_bucket(p
);
1070 case crush_grammar::_crushrule
:
1073 case crush_grammar::_choose_args
:
1074 r
= parse_choose_args(p
);
1084 //err << "max_devices " << crush.get_max_devices() << std::endl;
1085 crush
.cleanup_classes();
1091 // squash runs of whitespace to one space, excepting newlines
1092 string
CrushCompiler::consolidate_whitespace(string in
)
1097 for (unsigned p
=0; p
<in
.length(); p
++) {
1098 if (isspace(in
[p
]) && in
[p
] != '\n') {
1104 if (out
.length()) out
+= " ";
1111 err
<< " \"" << in
<< "\" -> \"" << out
<< "\"" << std::endl
;
1115 void CrushCompiler::dump(iter_t
const& i
, int ind
)
1118 for (int j
=0; j
<ind
; j
++)
1120 long id
= i
->value
.id().to_long();
1122 err
<< "'" << string(i
->value
.begin(), i
->value
.end())
1123 << "' " << i
->children
.size() << " children" << std::endl
;
1124 for (unsigned int j
= 0; j
< i
->children
.size(); j
++)
1125 dump(i
->children
.begin() + j
, ind
+1);
1129 * This function fix the problem like below
1130 * rack using_foo { item foo }
1133 * if an item being used by a bucket is defined after that bucket.
1134 * CRUSH compiler will create a map by which we can
1135 * not identify that item when selecting in that bucket.
1137 int CrushCompiler::adjust_bucket_item_place(iter_t
const &i
)
1139 map
<string
,set
<string
> > bucket_items
;
1140 map
<string
,iter_t
> bucket_itrer
;
1141 vector
<string
> buckets
;
1142 for (iter_t p
= i
->children
.begin(); p
!= i
->children
.end(); ++p
) {
1143 if ((int)p
->value
.id().to_long() == crush_grammar::_bucket
) {
1144 string name
= string_node(p
->children
[1]);
1145 buckets
.push_back(name
);
1146 bucket_itrer
[name
] = p
;
1147 //skip non-bucket-item children in the bucket's parse tree
1148 for (unsigned q
=3; q
< p
->children
.size()-1; ++q
) {
1149 iter_t sub
= p
->children
.begin() + q
;
1150 if ((int)sub
->value
.id().to_long()
1151 == crush_grammar::_bucket_item
) {
1152 string iname
= string_node(sub
->children
[1]);
1153 bucket_items
[name
].insert(iname
);
1160 for (unsigned i
=0; i
< buckets
.size(); ++i
) {
1161 for (unsigned j
=i
+1; j
< buckets
.size(); ++j
) {
1162 if (bucket_items
[buckets
[i
]].count(buckets
[j
])) {
1163 if (bucket_items
[buckets
[j
]].count(buckets
[i
])) {
1164 err
<< "bucket '" << buckets
[i
] << "' and bucket '"
1165 << buckets
[j
] << "' are included each other" << std::endl
;
1168 std::iter_swap(bucket_itrer
[buckets
[i
]], bucket_itrer
[buckets
[j
]]);
1177 int CrushCompiler::compile(istream
& in
, const char *infn
)
1182 // always start with legacy tunables, so that the compiled result of
1183 // a given crush file is fixed for all time.
1184 crush
.set_tunables_legacy();
1189 map
<int,int> line_pos
; // pos -> line
1190 map
<int,string
> line_val
;
1191 while (getline(in
, str
)) {
1193 int l
= str
.length();
1194 if (l
&& str
[l
- 1] == '\n')
1197 line_val
[line
] = str
;
1200 int n
= str
.find("#");
1202 str
.erase(n
, str
.length()-n
);
1204 if (verbose
>1) err
<< line
<< ": " << str
<< std::endl
;
1206 // work around spirit crankiness by removing extraneous
1207 // whitespace. there is probably a more elegant solution, but
1208 // this only broke with the latest spirit (with the switchover to
1209 // "classic"), i don't want to spend too much time figuring it
1211 string stripped
= consolidate_whitespace(str
);
1212 if (stripped
.length() && big
.length() && big
[big
.length()-1] != ' ') big
+= " ";
1214 line_pos
[big
.length()] = line
;
1219 if (verbose
> 2) err
<< "whole file is: \"" << big
<< "\"" << std::endl
;
1221 crush_grammar crushg
;
1222 const char *start
= big
.c_str();
1223 //tree_parse_info<const char *> info = ast_parse(start, crushg, space_p);
1224 tree_parse_info
<> info
= ast_parse(start
, crushg
, space_p
);
1228 int cpos
= info
.stop
- start
;
1229 //out << "cpos " << cpos << std::endl;
1230 //out << " linemap " << line_pos << std::endl;
1231 assert(!line_pos
.empty());
1232 map
<int,int>::iterator p
= line_pos
.upper_bound(cpos
);
1233 if (p
!= line_pos
.begin())
1235 int line
= p
->second
;
1236 int pos
= cpos
- p
->first
;
1237 err
<< infn
<< ":" << line
//<< ":" << (pos+1)
1238 << " error: parse error at '" << line_val
[line
].substr(pos
) << "'" << std::endl
;
1242 int r
= adjust_bucket_item_place(info
.trees
.begin());
1246 //out << "parsing succeeded\n";
1247 //dump(info.trees.begin());
1248 return parse_crush(info
.trees
.begin());