ceph/src/rgw/rgw_etag_verifier.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_etag_verifier.h"
   5
   6 #define dout_subsys ceph_subsys_rgw
   7
   8 namespace rgw::putobj {
   9
  10 int create_etag_verifier(CephContext* cct, DataProcessor* filter,
  11                          const bufferlist& manifest_bl,
  12                          const std::optional<RGWCompressionInfo>& compression,
  13                          etag_verifier_ptr& verifier)
  14 {
  15   RGWObjManifest manifest;
  16
  17   try {
  18     auto miter = manifest_bl.cbegin();
  19     decode(manifest, miter);
  20   } catch (buffer::error& err) {
  21     ldout(cct, 0) << "ERROR: couldn't decode manifest" << dendl;
  22     return -EIO;
  23   }
  24
  25   RGWObjManifestRule rule;
  26   bool found = manifest.get_rule(0, &rule);
  27   if (!found) {
  28     lderr(cct) << "ERROR: manifest->get_rule() could not find rule" << dendl;
  29     return -EIO;
  30   }
  31
  32   if (rule.start_part_num == 0) {
  33     /* Atomic object */
  34     verifier.emplace<ETagVerifier_Atomic>(cct, filter);
  35     return 0;
  36   }
  37
  38   uint64_t cur_part_ofs = UINT64_MAX;
  39   std::vector<uint64_t> part_ofs;
  40
  41   /*
  42    * We must store the offset of each part to calculate the ETAGs for each
  43    * MPU part. These part ETags then become the input for the MPU object
  44    * Etag.
  45    */
  46   for (auto mi = manifest.obj_begin(); mi != manifest.obj_end(); ++mi) {
  47     if (cur_part_ofs == mi.get_part_ofs())
  48       continue;
  49     cur_part_ofs = mi.get_part_ofs();
  50     ldout(cct, 20) << "MPU Part offset:" << cur_part_ofs << dendl;
  51     part_ofs.push_back(cur_part_ofs);
  52   }
  53
  54   if (compression) {
  55     // if the source object was compressed, the manifest is storing
  56     // compressed part offsets. transform the compressed offsets back to
  57     // their original offsets by finding the first block of each part
  58     const auto& blocks = compression->blocks;
  59     auto block = blocks.begin();
  60     for (auto& ofs : part_ofs) {
  61       // find the compression_block with new_ofs == ofs
  62       constexpr auto less = [] (const compression_block& block, uint64_t ofs) {
  63         return block.new_ofs < ofs;
  64       };
  65       block = std::lower_bound(block, blocks.end(), ofs, less);
  66       if (block == blocks.end() || block->new_ofs != ofs) {
  67         ldout(cct, 4) << "no match for compressed offset " << ofs
  68             << ", disabling etag verification" << dendl;
  69         return -EIO;
  70       }
  71       ofs = block->old_ofs;
  72       ldout(cct, 20) << "MPU Part uncompressed offset:" << ofs << dendl;
  73     }
  74   }
  75
  76   verifier.emplace<ETagVerifier_MPU>(cct, std::move(part_ofs), filter);
  77   return 0;
  78 }
  79
  80 int ETagVerifier_Atomic::process(bufferlist&& in, uint64_t logical_offset)
  81 {
  82   bufferlist out;
  83   if (in.length() > 0)
  84     hash.Update((const unsigned char *)in.c_str(), in.length());
  85
  86   return Pipe::process(std::move(in), logical_offset);
  87 }
  88
  89 void ETagVerifier_Atomic::calculate_etag()
  90 {
  91   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
  92   char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
  93
  94   /* Return early if ETag has already been calculated */
  95   if (!calculated_etag.empty())
  96     return;
  97
  98   hash.Final(m);
  99   buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
 100   calculated_etag = calc_md5;
 101   ldout(cct, 20) << "Single part object: " << " etag:" << calculated_etag
 102           << dendl;
 103 }
 104
 105 void ETagVerifier_MPU::process_end_of_MPU_part()
 106 {
 107   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
 108   char calc_md5_part[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
 109   std::string calculated_etag_part;
 110
 111   hash.Final(m);
 112   mpu_etag_hash.Update((const unsigned char *)m, sizeof(m));
 113   hash.Restart();
 114
 115   if (cct->_conf->subsys.should_gather(dout_subsys, 20)) {
 116     buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5_part);
 117     calculated_etag_part = calc_md5_part;
 118     ldout(cct, 20) << "Part etag: " << calculated_etag_part << dendl;
 119   }
 120
 121   cur_part_index++;
 122   next_part_index++;
 123 }
 124
 125 int ETagVerifier_MPU::process(bufferlist&& in, uint64_t logical_offset)
 126 {
 127   uint64_t bl_end = in.length() + logical_offset;
 128
 129   /* Handle the last MPU part */
 130   if (size_t(next_part_index) == part_ofs.size()) {
 131     hash.Update((const unsigned char *)in.c_str(), in.length());
 132     goto done;
 133   }
 134
 135   /* Incoming bufferlist spans two MPU parts. Calculate separate ETags */
 136   if (bl_end > part_ofs[next_part_index]) {
 137
 138     uint64_t part_one_len = part_ofs[next_part_index] - logical_offset;
 139     hash.Update((const unsigned char *)in.c_str(), part_one_len);
 140     process_end_of_MPU_part();
 141
 142     hash.Update((const unsigned char *)in.c_str() + part_one_len,
 143       bl_end - part_ofs[cur_part_index]);
 144     /*
 145      * If we've moved to the last part of the MPU, avoid usage of
 146      * parts_ofs[next_part_index] as it will lead to our-of-range access.
 147      */
 148     if (size_t(next_part_index) == part_ofs.size())
 149       goto done;
 150   } else {
 151     hash.Update((const unsigned char *)in.c_str(), in.length());
 152   }
 153
 154   /* Update the MPU Etag if the current part has ended */
 155   if (logical_offset + in.length() + 1 == part_ofs[next_part_index])
 156     process_end_of_MPU_part();
 157
 158 done:
 159   return Pipe::process(std::move(in), logical_offset);
 160 }
 161
 162 void ETagVerifier_MPU::calculate_etag()
 163 {
 164   const uint32_t parts = part_ofs.size();
 165   constexpr auto digits10 = std::numeric_limits<uint32_t>::digits10;
 166   constexpr auto extra = 2 + digits10; // add "-%u\0" at the end
 167
 168   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE], mpu_m[CEPH_CRYPTO_MD5_DIGESTSIZE];
 169   char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + extra];
 170
 171   /* Return early if ETag has already been calculated */
 172   if (!calculated_etag.empty())
 173     return;
 174
 175   hash.Final(m);
 176   mpu_etag_hash.Update((const unsigned char *)m, sizeof(m));
 177
 178   /* Refer RGWCompleteMultipart::execute() for ETag calculation for MPU object */
 179   mpu_etag_hash.Final(mpu_m);
 180   buf_to_hex(mpu_m, CEPH_CRYPTO_MD5_DIGESTSIZE, final_etag_str);
 181   snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2],
 182            sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
 183            "-%u", parts);
 184
 185   calculated_etag = final_etag_str;
 186   ldout(cct, 20) << "MPU calculated ETag:" << calculated_etag << dendl;
 187 }
 188
 189 } // namespace rgw::putobj