]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | ||
3 | #include <errno.h> | |
4 | #include "include/encoding.h" | |
5 | #include "ECUtil.h" | |
11fdf7f2 | 6 | |
31f18b77 | 7 | using namespace std; |
f67539c2 TL |
8 | using ceph::bufferlist; |
9 | using ceph::ErasureCodeInterfaceRef; | |
10 | using ceph::Formatter; | |
7c673cae FG |
11 | |
12 | int ECUtil::decode( | |
13 | const stripe_info_t &sinfo, | |
14 | ErasureCodeInterfaceRef &ec_impl, | |
15 | map<int, bufferlist> &to_decode, | |
16 | bufferlist *out) { | |
11fdf7f2 | 17 | ceph_assert(to_decode.size()); |
7c673cae FG |
18 | |
19 | uint64_t total_data_size = to_decode.begin()->second.length(); | |
11fdf7f2 | 20 | ceph_assert(total_data_size % sinfo.get_chunk_size() == 0); |
7c673cae | 21 | |
11fdf7f2 TL |
22 | ceph_assert(out); |
23 | ceph_assert(out->length() == 0); | |
7c673cae FG |
24 | |
25 | for (map<int, bufferlist>::iterator i = to_decode.begin(); | |
26 | i != to_decode.end(); | |
27 | ++i) { | |
11fdf7f2 | 28 | ceph_assert(i->second.length() == total_data_size); |
7c673cae FG |
29 | } |
30 | ||
31 | if (total_data_size == 0) | |
32 | return 0; | |
33 | ||
34 | for (uint64_t i = 0; i < total_data_size; i += sinfo.get_chunk_size()) { | |
35 | map<int, bufferlist> chunks; | |
36 | for (map<int, bufferlist>::iterator j = to_decode.begin(); | |
37 | j != to_decode.end(); | |
38 | ++j) { | |
39 | chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size()); | |
40 | } | |
41 | bufferlist bl; | |
42 | int r = ec_impl->decode_concat(chunks, &bl); | |
11fdf7f2 TL |
43 | ceph_assert(r == 0); |
44 | ceph_assert(bl.length() == sinfo.get_stripe_width()); | |
7c673cae FG |
45 | out->claim_append(bl); |
46 | } | |
47 | return 0; | |
48 | } | |
49 | ||
50 | int ECUtil::decode( | |
51 | const stripe_info_t &sinfo, | |
52 | ErasureCodeInterfaceRef &ec_impl, | |
53 | map<int, bufferlist> &to_decode, | |
54 | map<int, bufferlist*> &out) { | |
7c673cae | 55 | |
11fdf7f2 | 56 | ceph_assert(to_decode.size()); |
7c673cae | 57 | |
11fdf7f2 TL |
58 | for (auto &&i : to_decode) { |
59 | if(i.second.length() == 0) | |
60 | return 0; | |
7c673cae FG |
61 | } |
62 | ||
7c673cae FG |
63 | set<int> need; |
64 | for (map<int, bufferlist*>::iterator i = out.begin(); | |
65 | i != out.end(); | |
66 | ++i) { | |
11fdf7f2 TL |
67 | ceph_assert(i->second); |
68 | ceph_assert(i->second->length() == 0); | |
7c673cae FG |
69 | need.insert(i->first); |
70 | } | |
71 | ||
11fdf7f2 TL |
72 | set<int> avail; |
73 | for (auto &&i : to_decode) { | |
74 | ceph_assert(i.second.length() != 0); | |
75 | avail.insert(i.first); | |
76 | } | |
77 | ||
78 | map<int, vector<pair<int, int>>> min; | |
79 | int r = ec_impl->minimum_to_decode(need, avail, &min); | |
80 | ceph_assert(r == 0); | |
81 | ||
82 | int chunks_count = 0; | |
83 | int repair_data_per_chunk = 0; | |
84 | int subchunk_size = sinfo.get_chunk_size()/ec_impl->get_sub_chunk_count(); | |
85 | ||
86 | for (auto &&i : to_decode) { | |
87 | auto found = min.find(i.first); | |
88 | if (found != min.end()) { | |
89 | int repair_subchunk_count = 0; | |
90 | for (auto& subchunks : min[i.first]) { | |
91 | repair_subchunk_count += subchunks.second; | |
92 | } | |
93 | repair_data_per_chunk = repair_subchunk_count * subchunk_size; | |
94 | chunks_count = (int)i.second.length() / repair_data_per_chunk; | |
95 | break; | |
96 | } | |
97 | } | |
98 | ||
99 | for (int i = 0; i < chunks_count; i++) { | |
7c673cae | 100 | map<int, bufferlist> chunks; |
11fdf7f2 | 101 | for (auto j = to_decode.begin(); |
7c673cae FG |
102 | j != to_decode.end(); |
103 | ++j) { | |
11fdf7f2 TL |
104 | chunks[j->first].substr_of(j->second, |
105 | i*repair_data_per_chunk, | |
106 | repair_data_per_chunk); | |
7c673cae FG |
107 | } |
108 | map<int, bufferlist> out_bls; | |
11fdf7f2 TL |
109 | r = ec_impl->decode(need, chunks, &out_bls, sinfo.get_chunk_size()); |
110 | ceph_assert(r == 0); | |
111 | for (auto j = out.begin(); j != out.end(); ++j) { | |
112 | ceph_assert(out_bls.count(j->first)); | |
113 | ceph_assert(out_bls[j->first].length() == sinfo.get_chunk_size()); | |
7c673cae FG |
114 | j->second->claim_append(out_bls[j->first]); |
115 | } | |
116 | } | |
11fdf7f2 TL |
117 | for (auto &&i : out) { |
118 | ceph_assert(i.second->length() == chunks_count * sinfo.get_chunk_size()); | |
7c673cae FG |
119 | } |
120 | return 0; | |
121 | } | |
122 | ||
123 | int ECUtil::encode( | |
124 | const stripe_info_t &sinfo, | |
125 | ErasureCodeInterfaceRef &ec_impl, | |
126 | bufferlist &in, | |
127 | const set<int> &want, | |
128 | map<int, bufferlist> *out) { | |
129 | ||
130 | uint64_t logical_size = in.length(); | |
131 | ||
11fdf7f2 TL |
132 | ceph_assert(logical_size % sinfo.get_stripe_width() == 0); |
133 | ceph_assert(out); | |
134 | ceph_assert(out->empty()); | |
7c673cae FG |
135 | |
136 | if (logical_size == 0) | |
137 | return 0; | |
138 | ||
139 | for (uint64_t i = 0; i < logical_size; i += sinfo.get_stripe_width()) { | |
140 | map<int, bufferlist> encoded; | |
141 | bufferlist buf; | |
142 | buf.substr_of(in, i, sinfo.get_stripe_width()); | |
143 | int r = ec_impl->encode(want, buf, &encoded); | |
11fdf7f2 | 144 | ceph_assert(r == 0); |
7c673cae FG |
145 | for (map<int, bufferlist>::iterator i = encoded.begin(); |
146 | i != encoded.end(); | |
147 | ++i) { | |
11fdf7f2 | 148 | ceph_assert(i->second.length() == sinfo.get_chunk_size()); |
7c673cae FG |
149 | (*out)[i->first].claim_append(i->second); |
150 | } | |
151 | } | |
152 | ||
153 | for (map<int, bufferlist>::iterator i = out->begin(); | |
154 | i != out->end(); | |
155 | ++i) { | |
11fdf7f2 TL |
156 | ceph_assert(i->second.length() % sinfo.get_chunk_size() == 0); |
157 | ceph_assert( | |
7c673cae FG |
158 | sinfo.aligned_chunk_offset_to_logical_offset(i->second.length()) == |
159 | logical_size); | |
160 | } | |
161 | return 0; | |
162 | } | |
163 | ||
164 | void ECUtil::HashInfo::append(uint64_t old_size, | |
165 | map<int, bufferlist> &to_append) { | |
11fdf7f2 | 166 | ceph_assert(old_size == total_chunk_size); |
7c673cae FG |
167 | uint64_t size_to_append = to_append.begin()->second.length(); |
168 | if (has_chunk_hash()) { | |
11fdf7f2 | 169 | ceph_assert(to_append.size() == cumulative_shard_hashes.size()); |
7c673cae FG |
170 | for (map<int, bufferlist>::iterator i = to_append.begin(); |
171 | i != to_append.end(); | |
172 | ++i) { | |
11fdf7f2 TL |
173 | ceph_assert(size_to_append == i->second.length()); |
174 | ceph_assert((unsigned)i->first < cumulative_shard_hashes.size()); | |
7c673cae FG |
175 | uint32_t new_hash = i->second.crc32c(cumulative_shard_hashes[i->first]); |
176 | cumulative_shard_hashes[i->first] = new_hash; | |
177 | } | |
178 | } | |
179 | total_chunk_size += size_to_append; | |
180 | } | |
181 | ||
182 | void ECUtil::HashInfo::encode(bufferlist &bl) const | |
183 | { | |
184 | ENCODE_START(1, 1, bl); | |
11fdf7f2 TL |
185 | encode(total_chunk_size, bl); |
186 | encode(cumulative_shard_hashes, bl); | |
7c673cae FG |
187 | ENCODE_FINISH(bl); |
188 | } | |
189 | ||
11fdf7f2 | 190 | void ECUtil::HashInfo::decode(bufferlist::const_iterator &bl) |
7c673cae FG |
191 | { |
192 | DECODE_START(1, bl); | |
11fdf7f2 TL |
193 | decode(total_chunk_size, bl); |
194 | decode(cumulative_shard_hashes, bl); | |
7c673cae FG |
195 | projected_total_chunk_size = total_chunk_size; |
196 | DECODE_FINISH(bl); | |
197 | } | |
198 | ||
199 | void ECUtil::HashInfo::dump(Formatter *f) const | |
200 | { | |
201 | f->dump_unsigned("total_chunk_size", total_chunk_size); | |
94b18763 | 202 | f->open_array_section("cumulative_shard_hashes"); |
7c673cae FG |
203 | for (unsigned i = 0; i != cumulative_shard_hashes.size(); ++i) { |
204 | f->open_object_section("hash"); | |
205 | f->dump_unsigned("shard", i); | |
206 | f->dump_unsigned("hash", cumulative_shard_hashes[i]); | |
207 | f->close_section(); | |
208 | } | |
209 | f->close_section(); | |
210 | } | |
211 | ||
94b18763 FG |
212 | namespace ECUtil { |
213 | std::ostream& operator<<(std::ostream& out, const HashInfo& hi) | |
214 | { | |
215 | ostringstream hashes; | |
216 | for (auto hash: hi.cumulative_shard_hashes) | |
217 | hashes << " " << hex << hash; | |
218 | return out << "tcs=" << hi.total_chunk_size << hashes.str(); | |
219 | } | |
220 | } | |
221 | ||
7c673cae FG |
222 | void ECUtil::HashInfo::generate_test_instances(list<HashInfo*>& o) |
223 | { | |
224 | o.push_back(new HashInfo(3)); | |
225 | { | |
226 | bufferlist bl; | |
227 | bl.append_zero(20); | |
228 | map<int, bufferlist> buffers; | |
229 | buffers[0] = bl; | |
230 | buffers[1] = bl; | |
231 | buffers[2] = bl; | |
232 | o.back()->append(0, buffers); | |
233 | o.back()->append(20, buffers); | |
234 | } | |
235 | o.push_back(new HashInfo(4)); | |
236 | } | |
237 | ||
238 | const string HINFO_KEY = "hinfo_key"; | |
239 | ||
240 | bool ECUtil::is_hinfo_key_string(const string &key) | |
241 | { | |
242 | return key == HINFO_KEY; | |
243 | } | |
244 | ||
245 | const string &ECUtil::get_hinfo_key() | |
246 | { | |
247 | return HINFO_KEY; | |
248 | } |