]> git.proxmox.com Git - ceph.git/blame - ceph/src/rgw/rgw_log_backing.h
buildsys: change download over to reef release
[ceph.git] / ceph / src / rgw / rgw_log_backing.h
CommitLineData
f67539c2
TL
1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab ft=cpp
3
4#ifndef CEPH_RGW_LOGBACKING_H
5#define CEPH_RGW_LOGBACKING_H
6
7#include <optional>
8#include <iostream>
9#include <string>
10#include <string_view>
11
12#include <strings.h>
13
14#include <boost/container/flat_map.hpp>
15#include <boost/system/error_code.hpp>
16
17#undef FMT_HEADER_ONLY
18#define FMT_HEADER_ONLY 1
19#include <fmt/format.h>
20
21#include "include/rados/librados.hpp"
22#include "include/encoding.h"
23#include "include/expected.hpp"
24#include "include/function2.hpp"
25
26#include "cls/version/cls_version_types.h"
27
28#include "common/async/yield_context.h"
29#include "common/Formatter.h"
30#include "common/strtol.h"
31
32namespace bc = boost::container;
33namespace bs = boost::system;
34
35#include "cls_fifo_legacy.h"
36
37/// Type of log backing, stored in the mark used in the quick check,
38/// and passed to checking functions.
39enum class log_type {
40 omap = 0,
41 fifo = 1
42};
43
44inline void encode(const log_type& type, ceph::buffer::list& bl) {
45 auto t = static_cast<uint8_t>(type);
46 encode(t, bl);
47}
48
49inline void decode(log_type& type, bufferlist::const_iterator& bl) {
50 uint8_t t;
51 decode(t, bl);
52 type = static_cast<log_type>(t);
53}
54
55inline std::optional<log_type> to_log_type(std::string_view s) {
56 if (strncasecmp(s.data(), "omap", s.length()) == 0) {
57 return log_type::omap;
58 } else if (strncasecmp(s.data(), "fifo", s.length()) == 0) {
59 return log_type::fifo;
60 } else {
61 return std::nullopt;
62 }
63}
64inline std::ostream& operator <<(std::ostream& m, const log_type& t) {
65 switch (t) {
66 case log_type::omap:
67 return m << "log_type::omap";
68 case log_type::fifo:
69 return m << "log_type::fifo";
70 }
71
72 return m << "log_type::UNKNOWN=" << static_cast<uint32_t>(t);
73}
74
75/// Look over the shards in a log and determine the type.
76tl::expected<log_type, bs::error_code>
b3b6e05e
TL
77log_backing_type(const DoutPrefixProvider *dpp,
78 librados::IoCtx& ioctx,
f67539c2
TL
79 log_type def,
80 int shards, //< Total number of shards
81 /// A function taking a shard number and
82 /// returning an oid.
83 const fu2::unique_function<std::string(int) const>& get_oid,
84 optional_yield y);
85
86/// Remove all log shards and associated parts of fifos.
87bs::error_code log_remove(librados::IoCtx& ioctx,
88 int shards, //< Total number of shards
89 /// A function taking a shard number and
90 /// returning an oid.
91 const fu2::unique_function<std::string(int) const>& get_oid,
92 bool leave_zero,
93 optional_yield y);
94
95
96struct logback_generation {
97 uint64_t gen_id = 0;
98 log_type type;
99 std::optional<ceph::real_time> pruned;
100
101 void encode(ceph::buffer::list& bl) const {
102 ENCODE_START(1, 1, bl);
103 encode(gen_id, bl);
104 encode(type, bl);
105 encode(pruned, bl);
106 ENCODE_FINISH(bl);
107 }
108
109 void decode(bufferlist::const_iterator& bl) {
110 DECODE_START(1, bl);
111 decode(gen_id, bl);
112 decode(type, bl);
113 decode(pruned, bl);
114 DECODE_FINISH(bl);
115 }
116};
117WRITE_CLASS_ENCODER(logback_generation)
b3b6e05e
TL
118inline std::ostream& operator <<(std::ostream& m, const logback_generation& g) {
119 return m << "[" << g.gen_id << "," << g.type << ","
120 << (g.pruned ? "PRUNED" : "NOT PRUNED") << "]";
121}
f67539c2
TL
122
123class logback_generations : public librados::WatchCtx2 {
124public:
125 using entries_t = bc::flat_map<uint64_t, logback_generation>;
126
127protected:
128 librados::IoCtx& ioctx;
129 logback_generations(librados::IoCtx& ioctx,
130 std::string oid,
131 fu2::unique_function<std::string(
132 uint64_t, int) const>&& get_oid,
133 int shards) noexcept
134 : ioctx(ioctx), oid(oid), get_oid(std::move(get_oid)),
135 shards(shards) {}
136
137 uint64_t my_id = ioctx.get_instance_id();
138
139private:
140 const std::string oid;
141 const fu2::unique_function<std::string(uint64_t, int) const> get_oid;
142
143protected:
144 const int shards;
145
146private:
147
148 uint64_t watchcookie = 0;
149
150 obj_version version;
151 std::mutex m;
152 entries_t entries_;
153
154 tl::expected<std::pair<entries_t, obj_version>, bs::error_code>
b3b6e05e
TL
155 read(const DoutPrefixProvider *dpp, optional_yield y) noexcept;
156 bs::error_code write(const DoutPrefixProvider *dpp, entries_t&& e, std::unique_lock<std::mutex>&& l_,
f67539c2 157 optional_yield y) noexcept;
b3b6e05e 158 bs::error_code setup(const DoutPrefixProvider *dpp, log_type def, optional_yield y) noexcept;
f67539c2
TL
159
160 bs::error_code watch() noexcept;
161
162 auto lowest_nomempty(const entries_t& es) {
163 return std::find_if(es.begin(), es.end(),
164 [](const auto& e) {
165 return !e.second.pruned;
166 });
167 }
168
169public:
170
171 /// For the use of watch/notify.
172
173 void handle_notify(uint64_t notify_id,
174 uint64_t cookie,
175 uint64_t notifier_id,
176 bufferlist& bl) override final;
177
178 void handle_error(uint64_t cookie, int err) override final;
179
180 /// Public interface
181
182 virtual ~logback_generations();
183
184 template<typename T, typename... Args>
185 static tl::expected<std::unique_ptr<T>, bs::error_code>
b3b6e05e 186 init(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx_, std::string oid_,
f67539c2
TL
187 fu2::unique_function<std::string(uint64_t, int) const>&& get_oid_,
188 int shards_, log_type def, optional_yield y,
189 Args&& ...args) noexcept {
190 try {
191 T* lgp = new T(ioctx_, std::move(oid_),
192 std::move(get_oid_),
193 shards_, std::forward<Args>(args)...);
194 std::unique_ptr<T> lg(lgp);
195 lgp = nullptr;
b3b6e05e 196 auto ec = lg->setup(dpp, def, y);
f67539c2
TL
197 if (ec)
198 return tl::unexpected(ec);
199 // Obnoxiousness for C++ Compiler in Bionic Beaver
200 return tl::expected<std::unique_ptr<T>, bs::error_code>(std::move(lg));
201 } catch (const std::bad_alloc&) {
202 return tl::unexpected(bs::error_code(ENOMEM, bs::system_category()));
203 }
204 }
205
b3b6e05e 206 bs::error_code update(const DoutPrefixProvider *dpp, optional_yield y) noexcept;
f67539c2
TL
207
208 entries_t entries() const {
209 return entries_;
210 }
211
b3b6e05e 212 bs::error_code new_backing(const DoutPrefixProvider *dpp, log_type type, optional_yield y) noexcept;
f67539c2 213
b3b6e05e 214 bs::error_code empty_to(const DoutPrefixProvider *dpp, uint64_t gen_id, optional_yield y) noexcept;
f67539c2 215
b3b6e05e 216 bs::error_code remove_empty(const DoutPrefixProvider *dpp, optional_yield y) noexcept;
f67539c2
TL
217
218 // Callbacks, to be defined by descendant.
219
220 /// Handle initialization on startup
221 ///
222 /// @param e All non-empty generations
223 virtual bs::error_code handle_init(entries_t e) noexcept = 0;
224
225 /// Handle new generations.
226 ///
227 /// @param e Map of generations added since last update
228 virtual bs::error_code handle_new_gens(entries_t e) noexcept = 0;
229
230 /// Handle generations being marked empty
231 ///
232 /// @param new_tail Lowest non-empty generation
233 virtual bs::error_code handle_empty_to(uint64_t new_tail) noexcept = 0;
234};
235
236inline std::string gencursor(uint64_t gen_id, std::string_view cursor) {
237 return (gen_id > 0 ?
238 fmt::format("G{:0>20}@{}", gen_id, cursor) :
239 std::string(cursor));
240}
241
242inline std::pair<uint64_t, std::string_view>
243cursorgen(std::string_view cursor_) {
522d829b 244 if (cursor_.empty()) {
20effc67 245 return { 0, "" };
522d829b 246 }
f67539c2
TL
247 std::string_view cursor = cursor_;
248 if (cursor[0] != 'G') {
249 return { 0, cursor };
250 }
251 cursor.remove_prefix(1);
252 auto gen_id = ceph::consume<uint64_t>(cursor);
253 if (!gen_id || cursor[0] != '@') {
254 return { 0, cursor_ };
255 }
256 cursor.remove_prefix(1);
257 return { *gen_id, cursor };
258}
259
f67539c2
TL
260class LazyFIFO {
261 librados::IoCtx& ioctx;
262 std::string oid;
263 std::mutex m;
264 std::unique_ptr<rgw::cls::fifo::FIFO> fifo;
265
b3b6e05e 266 int lazy_init(const DoutPrefixProvider *dpp, optional_yield y) {
f67539c2
TL
267 std::unique_lock l(m);
268 if (fifo) return 0;
b3b6e05e 269 auto r = rgw::cls::fifo::FIFO::create(dpp, ioctx, oid, &fifo, y);
f67539c2
TL
270 if (r) {
271 fifo.reset();
272 }
273 return r;
274 }
275
276public:
277
278 LazyFIFO(librados::IoCtx& ioctx, std::string oid)
279 : ioctx(ioctx), oid(std::move(oid)) {}
280
b3b6e05e
TL
281 int read_meta(const DoutPrefixProvider *dpp, optional_yield y) {
282 auto r = lazy_init(dpp, y);
f67539c2 283 if (r < 0) return r;
b3b6e05e 284 return fifo->read_meta(dpp, y);
f67539c2
TL
285 }
286
b3b6e05e
TL
287 int meta(const DoutPrefixProvider *dpp, rados::cls::fifo::info& info, optional_yield y) {
288 auto r = lazy_init(dpp, y);
f67539c2
TL
289 if (r < 0) return r;
290 info = fifo->meta();
291 return 0;
292 }
293
b3b6e05e
TL
294 int get_part_layout_info(const DoutPrefixProvider *dpp,
295 std::uint32_t& part_header_size,
f67539c2
TL
296 std::uint32_t& part_entry_overhead,
297 optional_yield y) {
b3b6e05e 298 auto r = lazy_init(dpp, y);
f67539c2
TL
299 if (r < 0) return r;
300 std::tie(part_header_size, part_entry_overhead)
301 = fifo->get_part_layout_info();
302 return 0;
303 }
304
b3b6e05e
TL
305 int push(const DoutPrefixProvider *dpp,
306 const ceph::buffer::list& bl,
f67539c2 307 optional_yield y) {
b3b6e05e 308 auto r = lazy_init(dpp, y);
f67539c2 309 if (r < 0) return r;
b3b6e05e 310 return fifo->push(dpp, bl, y);
f67539c2
TL
311 }
312
b3b6e05e
TL
313 int push(const DoutPrefixProvider *dpp,
314 ceph::buffer::list& bl,
f67539c2
TL
315 librados::AioCompletion* c,
316 optional_yield y) {
b3b6e05e 317 auto r = lazy_init(dpp, y);
f67539c2 318 if (r < 0) return r;
b3b6e05e 319 fifo->push(dpp, bl, c);
f67539c2
TL
320 return 0;
321 }
322
b3b6e05e
TL
323 int push(const DoutPrefixProvider *dpp,
324 const std::vector<ceph::buffer::list>& data_bufs,
f67539c2 325 optional_yield y) {
b3b6e05e 326 auto r = lazy_init(dpp, y);
f67539c2 327 if (r < 0) return r;
b3b6e05e 328 return fifo->push(dpp, data_bufs, y);
f67539c2
TL
329 }
330
b3b6e05e
TL
331 int push(const DoutPrefixProvider *dpp,
332 const std::vector<ceph::buffer::list>& data_bufs,
f67539c2
TL
333 librados::AioCompletion* c,
334 optional_yield y) {
b3b6e05e 335 auto r = lazy_init(dpp, y);
f67539c2 336 if (r < 0) return r;
b3b6e05e 337 fifo->push(dpp, data_bufs, c);
f67539c2
TL
338 return 0;
339 }
340
b3b6e05e
TL
341 int list(const DoutPrefixProvider *dpp,
342 int max_entries, std::optional<std::string_view> markstr,
f67539c2
TL
343 std::vector<rgw::cls::fifo::list_entry>* out,
344 bool* more, optional_yield y) {
b3b6e05e 345 auto r = lazy_init(dpp, y);
f67539c2 346 if (r < 0) return r;
b3b6e05e 347 return fifo->list(dpp, max_entries, markstr, out, more, y);
f67539c2
TL
348 }
349
b3b6e05e 350 int list(const DoutPrefixProvider *dpp, int max_entries, std::optional<std::string_view> markstr,
f67539c2
TL
351 std::vector<rgw::cls::fifo::list_entry>* out, bool* more,
352 librados::AioCompletion* c, optional_yield y) {
b3b6e05e 353 auto r = lazy_init(dpp, y);
f67539c2 354 if (r < 0) return r;
b3b6e05e 355 fifo->list(dpp, max_entries, markstr, out, more, c);
f67539c2
TL
356 return 0;
357 }
358
b3b6e05e
TL
359 int trim(const DoutPrefixProvider *dpp, std::string_view markstr, bool exclusive, optional_yield y) {
360 auto r = lazy_init(dpp, y);
f67539c2 361 if (r < 0) return r;
b3b6e05e 362 return fifo->trim(dpp, markstr, exclusive, y);
f67539c2
TL
363 }
364
b3b6e05e 365 int trim(const DoutPrefixProvider *dpp, std::string_view markstr, bool exclusive, librados::AioCompletion* c,
f67539c2 366 optional_yield y) {
b3b6e05e 367 auto r = lazy_init(dpp, y);
f67539c2 368 if (r < 0) return r;
b3b6e05e 369 fifo->trim(dpp, markstr, exclusive, c);
f67539c2
TL
370 return 0;
371 }
372
b3b6e05e 373 int get_part_info(const DoutPrefixProvider *dpp, int64_t part_num, rados::cls::fifo::part_header* header,
f67539c2 374 optional_yield y) {
b3b6e05e 375 auto r = lazy_init(dpp, y);
f67539c2 376 if (r < 0) return r;
b3b6e05e 377 return fifo->get_part_info(dpp, part_num, header, y);
f67539c2
TL
378 }
379
b3b6e05e 380 int get_part_info(const DoutPrefixProvider *dpp, int64_t part_num, rados::cls::fifo::part_header* header,
f67539c2 381 librados::AioCompletion* c, optional_yield y) {
b3b6e05e 382 auto r = lazy_init(dpp, y);
f67539c2
TL
383 if (r < 0) return r;
384 fifo->get_part_info(part_num, header, c);
385 return 0;
386 }
387
b3b6e05e 388 int get_head_info(const DoutPrefixProvider *dpp, fu2::unique_function<
f67539c2
TL
389 void(int r, rados::cls::fifo::part_header&&)>&& f,
390 librados::AioCompletion* c,
391 optional_yield y) {
b3b6e05e 392 auto r = lazy_init(dpp, y);
f67539c2 393 if (r < 0) return r;
b3b6e05e 394 fifo->get_head_info(dpp, std::move(f), c);
f67539c2
TL
395 return 0;
396 }
397};
398
399#endif