ceph/src/osd/ECUtil.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2
   3 #include <errno.h>
   4 #include "include/encoding.h"
   5 #include "ECUtil.h"
   6 using namespace std;
   7
   8 int ECUtil::decode(
   9   const stripe_info_t &sinfo,
  10   ErasureCodeInterfaceRef &ec_impl,
  11   map<int, bufferlist> &to_decode,
  12   bufferlist *out) {
  13   assert(to_decode.size());
  14
  15   uint64_t total_data_size = to_decode.begin()->second.length();
  16   assert(total_data_size % sinfo.get_chunk_size() == 0);
  17
  18   assert(out);
  19   assert(out->length() == 0);
  20
  21   for (map<int, bufferlist>::iterator i = to_decode.begin();
  22        i != to_decode.end();
  23        ++i) {
  24     assert(i->second.length() == total_data_size);
  25   }
  26
  27   if (total_data_size == 0)
  28     return 0;
  29
  30   for (uint64_t i = 0; i < total_data_size; i += sinfo.get_chunk_size()) {
  31     map<int, bufferlist> chunks;
  32     for (map<int, bufferlist>::iterator j = to_decode.begin();
  33          j != to_decode.end();
  34          ++j) {
  35       chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
  36     }
  37     bufferlist bl;
  38     int r = ec_impl->decode_concat(chunks, &bl);
  39     assert(bl.length() == sinfo.get_stripe_width());
  40     assert(r == 0);
  41     out->claim_append(bl);
  42   }
  43   return 0;
  44 }
  45
  46 int ECUtil::decode(
  47   const stripe_info_t &sinfo,
  48   ErasureCodeInterfaceRef &ec_impl,
  49   map<int, bufferlist> &to_decode,
  50   map<int, bufferlist*> &out) {
  51   assert(to_decode.size());
  52
  53   uint64_t total_data_size = to_decode.begin()->second.length();
  54   assert(total_data_size % sinfo.get_chunk_size() == 0);
  55
  56   for (map<int, bufferlist>::iterator i = to_decode.begin();
  57        i != to_decode.end();
  58        ++i) {
  59     assert(i->second.length() == total_data_size);
  60   }
  61
  62   if (total_data_size == 0)
  63     return 0;
  64
  65   set<int> need;
  66   for (map<int, bufferlist*>::iterator i = out.begin();
  67        i != out.end();
  68        ++i) {
  69     assert(i->second);
  70     assert(i->second->length() == 0);
  71     need.insert(i->first);
  72   }
  73
  74   for (uint64_t i = 0; i < total_data_size; i += sinfo.get_chunk_size()) {
  75     map<int, bufferlist> chunks;
  76     for (map<int, bufferlist>::iterator j = to_decode.begin();
  77          j != to_decode.end();
  78          ++j) {
  79       chunks[j->first].substr_of(j->second, i, sinfo.get_chunk_size());
  80     }
  81     map<int, bufferlist> out_bls;
  82     int r = ec_impl->decode(need, chunks, &out_bls);
  83     assert(r == 0);
  84     for (map<int, bufferlist*>::iterator j = out.begin();
  85          j != out.end();
  86          ++j) {
  87       assert(out_bls.count(j->first));
  88       assert(out_bls[j->first].length() == sinfo.get_chunk_size());
  89       j->second->claim_append(out_bls[j->first]);
  90     }
  91   }
  92   for (map<int, bufferlist*>::iterator i = out.begin();
  93        i != out.end();
  94        ++i) {
  95     assert(i->second->length() == total_data_size);
  96   }
  97   return 0;
  98 }
  99
 100 int ECUtil::encode(
 101   const stripe_info_t &sinfo,
 102   ErasureCodeInterfaceRef &ec_impl,
 103   bufferlist &in,
 104   const set<int> &want,
 105   map<int, bufferlist> *out) {
 106
 107   uint64_t logical_size = in.length();
 108
 109   assert(logical_size % sinfo.get_stripe_width() == 0);
 110   assert(out);
 111   assert(out->empty());
 112
 113   if (logical_size == 0)
 114     return 0;
 115
 116   for (uint64_t i = 0; i < logical_size; i += sinfo.get_stripe_width()) {
 117     map<int, bufferlist> encoded;
 118     bufferlist buf;
 119     buf.substr_of(in, i, sinfo.get_stripe_width());
 120     int r = ec_impl->encode(want, buf, &encoded);
 121     assert(r == 0);
 122     for (map<int, bufferlist>::iterator i = encoded.begin();
 123          i != encoded.end();
 124          ++i) {
 125       assert(i->second.length() == sinfo.get_chunk_size());
 126       (*out)[i->first].claim_append(i->second);
 127     }
 128   }
 129
 130   for (map<int, bufferlist>::iterator i = out->begin();
 131        i != out->end();
 132        ++i) {
 133     assert(i->second.length() % sinfo.get_chunk_size() == 0);
 134     assert(
 135       sinfo.aligned_chunk_offset_to_logical_offset(i->second.length()) ==
 136       logical_size);
 137   }
 138   return 0;
 139 }
 140
 141 void ECUtil::HashInfo::append(uint64_t old_size,
 142                               map<int, bufferlist> &to_append) {
 143   assert(old_size == total_chunk_size);
 144   uint64_t size_to_append = to_append.begin()->second.length();
 145   if (has_chunk_hash()) {
 146     assert(to_append.size() == cumulative_shard_hashes.size());
 147     for (map<int, bufferlist>::iterator i = to_append.begin();
 148          i != to_append.end();
 149          ++i) {
 150       assert(size_to_append == i->second.length());
 151       assert((unsigned)i->first < cumulative_shard_hashes.size());
 152       uint32_t new_hash = i->second.crc32c(cumulative_shard_hashes[i->first]);
 153       cumulative_shard_hashes[i->first] = new_hash;
 154     }
 155   }
 156   total_chunk_size += size_to_append;
 157 }
 158
 159 void ECUtil::HashInfo::encode(bufferlist &bl) const
 160 {
 161   ENCODE_START(1, 1, bl);
 162   ::encode(total_chunk_size, bl);
 163   ::encode(cumulative_shard_hashes, bl);
 164   ENCODE_FINISH(bl);
 165 }
 166
 167 void ECUtil::HashInfo::decode(bufferlist::iterator &bl)
 168 {
 169   DECODE_START(1, bl);
 170   ::decode(total_chunk_size, bl);
 171   ::decode(cumulative_shard_hashes, bl);
 172   projected_total_chunk_size = total_chunk_size;
 173   DECODE_FINISH(bl);
 174 }
 175
 176 void ECUtil::HashInfo::dump(Formatter *f) const
 177 {
 178   f->dump_unsigned("total_chunk_size", total_chunk_size);
 179   f->open_array_section("cumulative_shard_hashes");
 180   for (unsigned i = 0; i != cumulative_shard_hashes.size(); ++i) {
 181     f->open_object_section("hash");
 182     f->dump_unsigned("shard", i);
 183     f->dump_unsigned("hash", cumulative_shard_hashes[i]);
 184     f->close_section();
 185   }
 186   f->close_section();
 187 }
 188
 189 namespace ECUtil {
 190 std::ostream& operator<<(std::ostream& out, const HashInfo& hi)
 191 {
 192   ostringstream hashes;
 193   for (auto hash: hi.cumulative_shard_hashes)
 194     hashes << " " << hex << hash;
 195   return out << "tcs=" << hi.total_chunk_size << hashes.str();
 196 }
 197 }
 198
 199 void ECUtil::HashInfo::generate_test_instances(list<HashInfo*>& o)
 200 {
 201   o.push_back(new HashInfo(3));
 202   {
 203     bufferlist bl;
 204     bl.append_zero(20);
 205     map<int, bufferlist> buffers;
 206     buffers[0] = bl;
 207     buffers[1] = bl;
 208     buffers[2] = bl;
 209     o.back()->append(0, buffers);
 210     o.back()->append(20, buffers);
 211   }
 212   o.push_back(new HashInfo(4));
 213 }
 214
 215 const string HINFO_KEY = "hinfo_key";
 216
 217 bool ECUtil::is_hinfo_key_string(const string &key)
 218 {
 219   return key == HINFO_KEY;
 220 }
 221
 222 const string &ECUtil::get_hinfo_key()
 223 {
 224   return HINFO_KEY;
 225 }