1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab ft=cpp
4 #include "rgw_etag_verifier.h"
6 #define dout_subsys ceph_subsys_rgw
8 namespace rgw::putobj
{
10 int create_etag_verifier(const DoutPrefixProvider
*dpp
,
11 CephContext
* cct
, DataProcessor
* filter
,
12 const bufferlist
& manifest_bl
,
13 const std::optional
<RGWCompressionInfo
>& compression
,
14 etag_verifier_ptr
& verifier
)
16 RGWObjManifest manifest
;
19 auto miter
= manifest_bl
.cbegin();
20 decode(manifest
, miter
);
21 } catch (buffer::error
& err
) {
22 ldpp_dout(dpp
, 0) << "ERROR: couldn't decode manifest" << dendl
;
26 RGWObjManifestRule rule
;
27 bool found
= manifest
.get_rule(0, &rule
);
29 ldpp_dout(dpp
, -1) << "ERROR: manifest->get_rule() could not find rule" << dendl
;
33 if (rule
.start_part_num
== 0) {
35 verifier
.emplace
<ETagVerifier_Atomic
>(cct
, filter
);
39 uint64_t cur_part_ofs
= UINT64_MAX
;
40 std::vector
<uint64_t> part_ofs
;
43 * We must store the offset of each part to calculate the ETAGs for each
44 * MPU part. These part ETags then become the input for the MPU object
47 for (auto mi
= manifest
.obj_begin(dpp
); mi
!= manifest
.obj_end(dpp
); ++mi
) {
48 if (cur_part_ofs
== mi
.get_part_ofs())
50 cur_part_ofs
= mi
.get_part_ofs();
51 ldpp_dout(dpp
, 20) << "MPU Part offset:" << cur_part_ofs
<< dendl
;
52 part_ofs
.push_back(cur_part_ofs
);
56 // if the source object was compressed, the manifest is storing
57 // compressed part offsets. transform the compressed offsets back to
58 // their original offsets by finding the first block of each part
59 const auto& blocks
= compression
->blocks
;
60 auto block
= blocks
.begin();
61 for (auto& ofs
: part_ofs
) {
62 // find the compression_block with new_ofs == ofs
63 constexpr auto less
= [] (const compression_block
& block
, uint64_t ofs
) {
64 return block
.new_ofs
< ofs
;
66 block
= std::lower_bound(block
, blocks
.end(), ofs
, less
);
67 if (block
== blocks
.end() || block
->new_ofs
!= ofs
) {
68 ldpp_dout(dpp
, 4) << "no match for compressed offset " << ofs
69 << ", disabling etag verification" << dendl
;
73 ldpp_dout(dpp
, 20) << "MPU Part uncompressed offset:" << ofs
<< dendl
;
77 verifier
.emplace
<ETagVerifier_MPU
>(cct
, std::move(part_ofs
), filter
);
81 int ETagVerifier_Atomic::process(bufferlist
&& in
, uint64_t logical_offset
)
85 hash
.Update((const unsigned char *)in
.c_str(), in
.length());
87 return Pipe::process(std::move(in
), logical_offset
);
90 void ETagVerifier_Atomic::calculate_etag()
92 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
93 char calc_md5
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
95 /* Return early if ETag has already been calculated */
96 if (!calculated_etag
.empty())
100 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, calc_md5
);
101 calculated_etag
= calc_md5
;
102 ldout(cct
, 20) << "Single part object: " << " etag:" << calculated_etag
106 void ETagVerifier_MPU::process_end_of_MPU_part()
108 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
109 char calc_md5_part
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + 1];
110 std::string calculated_etag_part
;
113 mpu_etag_hash
.Update((const unsigned char *)m
, sizeof(m
));
116 if (cct
->_conf
->subsys
.should_gather(dout_subsys
, 20)) {
117 buf_to_hex(m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, calc_md5_part
);
118 calculated_etag_part
= calc_md5_part
;
119 ldout(cct
, 20) << "Part etag: " << calculated_etag_part
<< dendl
;
126 int ETagVerifier_MPU::process(bufferlist
&& in
, uint64_t logical_offset
)
128 uint64_t bl_end
= in
.length() + logical_offset
;
130 /* Handle the last MPU part */
131 if (size_t(next_part_index
) == part_ofs
.size()) {
132 hash
.Update((const unsigned char *)in
.c_str(), in
.length());
136 /* Incoming bufferlist spans two MPU parts. Calculate separate ETags */
137 if (bl_end
> part_ofs
[next_part_index
]) {
139 uint64_t part_one_len
= part_ofs
[next_part_index
] - logical_offset
;
140 hash
.Update((const unsigned char *)in
.c_str(), part_one_len
);
141 process_end_of_MPU_part();
143 hash
.Update((const unsigned char *)in
.c_str() + part_one_len
,
144 bl_end
- part_ofs
[cur_part_index
]);
146 * If we've moved to the last part of the MPU, avoid usage of
147 * parts_ofs[next_part_index] as it will lead to our-of-range access.
149 if (size_t(next_part_index
) == part_ofs
.size())
152 hash
.Update((const unsigned char *)in
.c_str(), in
.length());
155 /* Update the MPU Etag if the current part has ended */
156 if (logical_offset
+ in
.length() + 1 == part_ofs
[next_part_index
])
157 process_end_of_MPU_part();
160 return Pipe::process(std::move(in
), logical_offset
);
163 void ETagVerifier_MPU::calculate_etag()
165 const uint32_t parts
= part_ofs
.size();
166 constexpr auto digits10
= std::numeric_limits
<uint32_t>::digits10
;
167 constexpr auto extra
= 2 + digits10
; // add "-%u\0" at the end
169 unsigned char m
[CEPH_CRYPTO_MD5_DIGESTSIZE
], mpu_m
[CEPH_CRYPTO_MD5_DIGESTSIZE
];
170 char final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2 + extra
];
172 /* Return early if ETag has already been calculated */
173 if (!calculated_etag
.empty())
177 mpu_etag_hash
.Update((const unsigned char *)m
, sizeof(m
));
179 /* Refer RGWCompleteMultipart::execute() for ETag calculation for MPU object */
180 mpu_etag_hash
.Final(mpu_m
);
181 buf_to_hex(mpu_m
, CEPH_CRYPTO_MD5_DIGESTSIZE
, final_etag_str
);
182 snprintf(&final_etag_str
[CEPH_CRYPTO_MD5_DIGESTSIZE
* 2],
183 sizeof(final_etag_str
) - CEPH_CRYPTO_MD5_DIGESTSIZE
* 2,
186 calculated_etag
= final_etag_str
;
187 ldout(cct
, 20) << "MPU calculated ETag:" << calculated_etag
<< dendl
;
190 } // namespace rgw::putobj