ceph/src/rgw/driver/rados/rgw_etag_verifier.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab ft=cpp
   3
   4 #include "rgw_etag_verifier.h"
   5 #include "rgw_obj_manifest.h"
   6
   7 #define dout_subsys ceph_subsys_rgw
   8
   9 namespace rgw::putobj {
  10
  11 int create_etag_verifier(const DoutPrefixProvider *dpp,
  12                          CephContext* cct, rgw::sal::DataProcessor* filter,
  13                          const bufferlist& manifest_bl,
  14                          const std::optional<RGWCompressionInfo>& compression,
  15                          etag_verifier_ptr& verifier)
  16 {
  17   RGWObjManifest manifest;
  18
  19   try {
  20     auto miter = manifest_bl.cbegin();
  21     decode(manifest, miter);
  22   } catch (buffer::error& err) {
  23     ldpp_dout(dpp, 0) << "ERROR: couldn't decode manifest" << dendl;
  24     return -EIO;
  25   }
  26
  27   RGWObjManifestRule rule;
  28   bool found = manifest.get_rule(0, &rule);
  29   if (!found) {
  30     ldpp_dout(dpp, -1) << "ERROR: manifest->get_rule() could not find rule" << dendl;
  31     return -EIO;
  32   }
  33
  34   if (rule.start_part_num == 0) {
  35     /* Atomic object */
  36     verifier.emplace<ETagVerifier_Atomic>(cct, filter);
  37     return 0;
  38   }
  39
  40   uint64_t cur_part_ofs = UINT64_MAX;
  41   std::vector<uint64_t> part_ofs;
  42
  43   /*
  44    * We must store the offset of each part to calculate the ETAGs for each
  45    * MPU part. These part ETags then become the input for the MPU object
  46    * Etag.
  47    */
  48   for (auto mi = manifest.obj_begin(dpp); mi != manifest.obj_end(dpp); ++mi) {
  49     if (cur_part_ofs == mi.get_part_ofs())
  50       continue;
  51     cur_part_ofs = mi.get_part_ofs();
  52     ldpp_dout(dpp, 20) << "MPU Part offset:" << cur_part_ofs << dendl;
  53     part_ofs.push_back(cur_part_ofs);
  54   }
  55
  56   if (compression) {
  57     // if the source object was compressed, the manifest is storing
  58     // compressed part offsets. transform the compressed offsets back to
  59     // their original offsets by finding the first block of each part
  60     const auto& blocks = compression->blocks;
  61     auto block = blocks.begin();
  62     for (auto& ofs : part_ofs) {
  63       // find the compression_block with new_ofs == ofs
  64       constexpr auto less = [] (const compression_block& block, uint64_t ofs) {
  65         return block.new_ofs < ofs;
  66       };
  67       block = std::lower_bound(block, blocks.end(), ofs, less);
  68       if (block == blocks.end() || block->new_ofs != ofs) {
  69         ldpp_dout(dpp, 4) << "no match for compressed offset " << ofs
  70             << ", disabling etag verification" << dendl;
  71         return -EIO;
  72       }
  73       ofs = block->old_ofs;
  74       ldpp_dout(dpp, 20) << "MPU Part uncompressed offset:" << ofs << dendl;
  75     }
  76   }
  77
  78   verifier.emplace<ETagVerifier_MPU>(cct, std::move(part_ofs), filter);
  79   return 0;
  80 }
  81
  82 int ETagVerifier_Atomic::process(bufferlist&& in, uint64_t logical_offset)
  83 {
  84   bufferlist out;
  85   if (in.length() > 0)
  86     hash.Update((const unsigned char *)in.c_str(), in.length());
  87
  88   return Pipe::process(std::move(in), logical_offset);
  89 }
  90
  91 void ETagVerifier_Atomic::calculate_etag()
  92 {
  93   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
  94   char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
  95
  96   /* Return early if ETag has already been calculated */
  97   if (!calculated_etag.empty())
  98     return;
  99
 100   hash.Final(m);
 101   buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5);
 102   calculated_etag = calc_md5;
 103   ldout(cct, 20) << "Single part object: " << " etag:" << calculated_etag
 104           << dendl;
 105 }
 106
 107 void ETagVerifier_MPU::process_end_of_MPU_part()
 108 {
 109   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE];
 110   char calc_md5_part[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1];
 111   std::string calculated_etag_part;
 112
 113   hash.Final(m);
 114   mpu_etag_hash.Update((const unsigned char *)m, sizeof(m));
 115   hash.Restart();
 116
 117   if (cct->_conf->subsys.should_gather(dout_subsys, 20)) {
 118     buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5_part);
 119     calculated_etag_part = calc_md5_part;
 120     ldout(cct, 20) << "Part etag: " << calculated_etag_part << dendl;
 121   }
 122
 123   cur_part_index++;
 124   next_part_index++;
 125 }
 126
 127 int ETagVerifier_MPU::process(bufferlist&& in, uint64_t logical_offset)
 128 {
 129   uint64_t bl_end = in.length() + logical_offset;
 130
 131   /* Handle the last MPU part */
 132   if (size_t(next_part_index) == part_ofs.size()) {
 133     hash.Update((const unsigned char *)in.c_str(), in.length());
 134     goto done;
 135   }
 136
 137   /* Incoming bufferlist spans two MPU parts. Calculate separate ETags */
 138   if (bl_end > part_ofs[next_part_index]) {
 139
 140     uint64_t part_one_len = part_ofs[next_part_index] - logical_offset;
 141     hash.Update((const unsigned char *)in.c_str(), part_one_len);
 142     process_end_of_MPU_part();
 143
 144     hash.Update((const unsigned char *)in.c_str() + part_one_len,
 145       bl_end - part_ofs[cur_part_index]);
 146     /*
 147      * If we've moved to the last part of the MPU, avoid usage of
 148      * parts_ofs[next_part_index] as it will lead to our-of-range access.
 149      */
 150     if (size_t(next_part_index) == part_ofs.size())
 151       goto done;
 152   } else {
 153     hash.Update((const unsigned char *)in.c_str(), in.length());
 154   }
 155
 156   /* Update the MPU Etag if the current part has ended */
 157   if (logical_offset + in.length() + 1 == part_ofs[next_part_index])
 158     process_end_of_MPU_part();
 159
 160 done:
 161   return Pipe::process(std::move(in), logical_offset);
 162 }
 163
 164 void ETagVerifier_MPU::calculate_etag()
 165 {
 166   const uint32_t parts = part_ofs.size();
 167   constexpr auto digits10 = std::numeric_limits<uint32_t>::digits10;
 168   constexpr auto extra = 2 + digits10; // add "-%u\0" at the end
 169
 170   unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE], mpu_m[CEPH_CRYPTO_MD5_DIGESTSIZE];
 171   char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + extra];
 172
 173   /* Return early if ETag has already been calculated */
 174   if (!calculated_etag.empty())
 175     return;
 176
 177   hash.Final(m);
 178   mpu_etag_hash.Update((const unsigned char *)m, sizeof(m));
 179
 180   /* Refer RGWCompleteMultipart::execute() for ETag calculation for MPU object */
 181   mpu_etag_hash.Final(mpu_m);
 182   buf_to_hex(mpu_m, CEPH_CRYPTO_MD5_DIGESTSIZE, final_etag_str);
 183   snprintf(&final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2],
 184            sizeof(final_etag_str) - CEPH_CRYPTO_MD5_DIGESTSIZE * 2,
 185            "-%u", parts);
 186
 187   calculated_etag = final_etag_str;
 188   ldout(cct, 20) << "MPU calculated ETag:" << calculated_etag << dendl;
 189 }
 190
 191 } // namespace rgw::putobj