]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Author: Myoungwon Oh <ohmyoungwon@gmail.com> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | #include "include/types.h" | |
15 | ||
16 | #include "include/rados/buffer.h" | |
17 | #include "include/rados/librados.hpp" | |
18 | #include "include/rados/rados_types.hpp" | |
19 | ||
20 | #include "acconfig.h" | |
21 | ||
11fdf7f2 | 22 | #include "common/Cond.h" |
9f95a23c TL |
23 | #include "common/Formatter.h" |
24 | #include "common/ceph_argparse.h" | |
25 | #include "common/ceph_crypto.h" | |
26 | #include "common/config.h" | |
11fdf7f2 TL |
27 | #include "common/debug.h" |
28 | #include "common/errno.h" | |
11fdf7f2 | 29 | #include "common/obj_bencher.h" |
9f95a23c | 30 | #include "global/global_init.h" |
11fdf7f2 TL |
31 | |
32 | #include <iostream> | |
33 | #include <fstream> | |
34 | #include <stdlib.h> | |
35 | #include <time.h> | |
36 | #include <sstream> | |
37 | #include <errno.h> | |
38 | #include <dirent.h> | |
39 | #include <stdexcept> | |
40 | #include <climits> | |
41 | #include <locale> | |
42 | #include <memory> | |
f67539c2 | 43 | #include <math.h> |
11fdf7f2 TL |
44 | |
45 | #include "tools/RadosDump.h" | |
46 | #include "cls/cas/cls_cas_client.h" | |
f67539c2 | 47 | #include "cls/cas/cls_cas_internal.h" |
11fdf7f2 TL |
48 | #include "include/stringify.h" |
49 | #include "global/signal_handler.h" | |
f67539c2 TL |
50 | #include "common/CDC.h" |
51 | ||
52 | struct EstimateResult { | |
53 | std::unique_ptr<CDC> cdc; | |
54 | ||
55 | uint64_t chunk_size; | |
56 | ||
57 | ceph::mutex lock = ceph::make_mutex("EstimateResult::lock"); | |
58 | ||
59 | // < key, <count, chunk_size> > | |
60 | map< string, pair <uint64_t, uint64_t> > chunk_statistics; | |
61 | uint64_t total_bytes = 0; | |
62 | std::atomic<uint64_t> total_objects = {0}; | |
63 | ||
64 | EstimateResult(std::string alg, int chunk_size) | |
65 | : cdc(CDC::create(alg, chunk_size)), | |
66 | chunk_size(1ull << chunk_size) {} | |
67 | ||
68 | void add_chunk(bufferlist& chunk, const std::string& fp_algo) { | |
69 | string fp; | |
70 | if (fp_algo == "sha1") { | |
71 | sha1_digest_t sha1_val = crypto::digest<crypto::SHA1>(chunk); | |
72 | fp = sha1_val.to_str(); | |
73 | } else if (fp_algo == "sha256") { | |
74 | sha256_digest_t sha256_val = crypto::digest<crypto::SHA256>(chunk); | |
75 | fp = sha256_val.to_str(); | |
76 | } else if (fp_algo == "sha512") { | |
77 | sha512_digest_t sha512_val = crypto::digest<crypto::SHA512>(chunk); | |
78 | fp = sha512_val.to_str(); | |
79 | } else { | |
80 | ceph_assert(0 == "no support fingerperint algorithm"); | |
81 | } | |
82 | ||
83 | std::lock_guard l(lock); | |
84 | auto p = chunk_statistics.find(fp); | |
85 | if (p != chunk_statistics.end()) { | |
86 | p->second.first++; | |
87 | if (p->second.second != chunk.length()) { | |
88 | cerr << "warning: hash collision on " << fp | |
89 | << ": was " << p->second.second | |
90 | << " now " << chunk.length() << std::endl; | |
91 | } | |
92 | } else { | |
93 | chunk_statistics[fp] = make_pair(1, chunk.length()); | |
94 | } | |
95 | total_bytes += chunk.length(); | |
96 | } | |
97 | ||
98 | void dump(Formatter *f) const { | |
99 | f->dump_unsigned("target_chunk_size", chunk_size); | |
100 | ||
101 | uint64_t dedup_bytes = 0; | |
102 | uint64_t dedup_objects = chunk_statistics.size(); | |
103 | for (auto& j : chunk_statistics) { | |
104 | dedup_bytes += j.second.second; | |
105 | } | |
106 | //f->dump_unsigned("dedup_bytes", dedup_bytes); | |
107 | //f->dump_unsigned("original_bytes", total_bytes); | |
108 | f->dump_float("dedup_bytes_ratio", | |
109 | (double)dedup_bytes / (double)total_bytes); | |
110 | f->dump_float("dedup_objects_ratio", | |
111 | (double)dedup_objects / (double)total_objects); | |
112 | ||
113 | uint64_t avg = total_bytes / dedup_objects; | |
114 | uint64_t sqsum = 0; | |
115 | for (auto& j : chunk_statistics) { | |
116 | sqsum += (avg - j.second.second) * (avg - j.second.second); | |
117 | } | |
118 | uint64_t stddev = sqrt(sqsum / dedup_objects); | |
119 | f->dump_unsigned("chunk_size_average", avg); | |
120 | f->dump_unsigned("chunk_size_stddev", stddev); | |
121 | } | |
122 | }; | |
123 | ||
124 | map<uint64_t, EstimateResult> dedup_estimates; // chunk size -> result | |
11fdf7f2 TL |
125 | |
126 | using namespace librados; | |
f67539c2 | 127 | unsigned default_op_size = 1 << 26; |
11fdf7f2 | 128 | unsigned default_max_thread = 2; |
f67539c2 TL |
129 | int32_t default_report_period = 10; |
130 | ceph::mutex glock = ceph::make_mutex("glock"); | |
11fdf7f2 TL |
131 | |
132 | void usage() | |
133 | { | |
f67539c2 | 134 | cout << " usage: [--op <estimate|chunk-scrub|chunk-get-ref|chunk-put-ref|dump-chunk-refs>] [--pool <pool_name> ] " << std::endl; |
11fdf7f2 TL |
135 | cout << " --object <object_name> " << std::endl; |
136 | cout << " --chunk-size <size> chunk-size (byte) " << std::endl; | |
f67539c2 | 137 | cout << " --chunk-algorithm <fixed|fastcdc> " << std::endl; |
9f95a23c | 138 | cout << " --fingerprint-algorithm <sha1|sha256|sha512> " << std::endl; |
11fdf7f2 TL |
139 | cout << " --chunk-pool <pool name> " << std::endl; |
140 | cout << " --max-thread <threads> " << std::endl; | |
f67539c2 TL |
141 | cout << " --report-period <seconds> " << std::endl; |
142 | cout << " --max-seconds <seconds>" << std::endl; | |
9f95a23c | 143 | cout << " --max-read-size <bytes> " << std::endl; |
11fdf7f2 TL |
144 | exit(1); |
145 | } | |
146 | ||
147 | template <typename I, typename T> | |
148 | static int rados_sistrtoll(I &i, T *val) { | |
149 | std::string err; | |
150 | *val = strict_iecstrtoll(i->second.c_str(), &err); | |
151 | if (err != "") { | |
152 | cerr << "Invalid value for " << i->first << ": " << err << std::endl; | |
153 | return -EINVAL; | |
154 | } else { | |
155 | return 0; | |
156 | } | |
157 | } | |
158 | ||
159 | class EstimateDedupRatio; | |
160 | class ChunkScrub; | |
f67539c2 | 161 | class CrawlerThread : public Thread |
11fdf7f2 TL |
162 | { |
163 | IoCtx io_ctx; | |
164 | int n; | |
165 | int m; | |
166 | ObjectCursor begin; | |
167 | ObjectCursor end; | |
f67539c2 | 168 | ceph::mutex m_lock = ceph::make_mutex("CrawlerThread::Locker"); |
9f95a23c | 169 | ceph::condition_variable m_cond; |
f67539c2 | 170 | int32_t report_period; |
11fdf7f2 TL |
171 | bool m_stop = false; |
172 | uint64_t total_bytes = 0; | |
f67539c2 | 173 | uint64_t total_objects = 0; |
11fdf7f2 | 174 | uint64_t examined_objects = 0; |
f67539c2 | 175 | uint64_t examined_bytes = 0; |
9f95a23c TL |
176 | uint64_t max_read_size = 0; |
177 | bool debug = false; | |
11fdf7f2 TL |
178 | #define COND_WAIT_INTERVAL 10 |
179 | ||
180 | public: | |
f67539c2 TL |
181 | CrawlerThread(IoCtx& io_ctx, int n, int m, |
182 | ObjectCursor begin, ObjectCursor end, int32_t report_period, | |
9f95a23c TL |
183 | uint64_t num_objects, uint64_t max_read_size = default_op_size): |
184 | io_ctx(io_ctx), n(n), m(m), begin(begin), end(end), | |
f67539c2 | 185 | report_period(report_period), total_objects(num_objects), max_read_size(max_read_size) |
11fdf7f2 TL |
186 | {} |
187 | void signal(int signum) { | |
9f95a23c | 188 | std::lock_guard l{m_lock}; |
11fdf7f2 | 189 | m_stop = true; |
9f95a23c | 190 | m_cond.notify_all(); |
11fdf7f2 | 191 | } |
f67539c2 | 192 | virtual void print_status(Formatter *f, ostream &out) {} |
11fdf7f2 | 193 | uint64_t get_examined_objects() { return examined_objects; } |
f67539c2 | 194 | uint64_t get_examined_bytes() { return examined_bytes; } |
11fdf7f2 TL |
195 | uint64_t get_total_bytes() { return total_bytes; } |
196 | uint64_t get_total_objects() { return total_objects; } | |
f67539c2 | 197 | void set_debug(const bool debug_) { debug = debug_; } |
11fdf7f2 TL |
198 | friend class EstimateDedupRatio; |
199 | friend class ChunkScrub; | |
200 | }; | |
201 | ||
f67539c2 | 202 | class EstimateDedupRatio : public CrawlerThread |
11fdf7f2 TL |
203 | { |
204 | string chunk_algo; | |
205 | string fp_algo; | |
206 | uint64_t chunk_size; | |
f67539c2 | 207 | uint64_t max_seconds; |
11fdf7f2 TL |
208 | |
209 | public: | |
f67539c2 TL |
210 | EstimateDedupRatio( |
211 | IoCtx& io_ctx, int n, int m, ObjectCursor begin, ObjectCursor end, | |
212 | string chunk_algo, string fp_algo, uint64_t chunk_size, int32_t report_period, | |
213 | uint64_t num_objects, uint64_t max_read_size, | |
214 | uint64_t max_seconds): | |
215 | CrawlerThread(io_ctx, n, m, begin, end, report_period, num_objects, | |
216 | max_read_size), | |
217 | chunk_algo(chunk_algo), | |
218 | fp_algo(fp_algo), | |
219 | chunk_size(chunk_size), | |
220 | max_seconds(max_seconds) { | |
221 | } | |
11fdf7f2 TL |
222 | |
223 | void* entry() { | |
11fdf7f2 TL |
224 | estimate_dedup_ratio(); |
225 | return NULL; | |
226 | } | |
227 | void estimate_dedup_ratio(); | |
11fdf7f2 TL |
228 | }; |
229 | ||
f67539c2 | 230 | class ChunkScrub: public CrawlerThread |
11fdf7f2 TL |
231 | { |
232 | IoCtx chunk_io_ctx; | |
f67539c2 | 233 | int damaged_objects = 0; |
11fdf7f2 TL |
234 | |
235 | public: | |
236 | ChunkScrub(IoCtx& io_ctx, int n, int m, ObjectCursor begin, ObjectCursor end, | |
f67539c2 TL |
237 | IoCtx& chunk_io_ctx, int32_t report_period, uint64_t num_objects): |
238 | CrawlerThread(io_ctx, n, m, begin, end, report_period, num_objects), chunk_io_ctx(chunk_io_ctx) | |
11fdf7f2 TL |
239 | { } |
240 | void* entry() { | |
11fdf7f2 TL |
241 | chunk_scrub_common(); |
242 | return NULL; | |
243 | } | |
244 | void chunk_scrub_common(); | |
f67539c2 | 245 | int get_damaged_objects() { return damaged_objects; } |
11fdf7f2 TL |
246 | void print_status(Formatter *f, ostream &out); |
247 | }; | |
248 | ||
f67539c2 | 249 | vector<std::unique_ptr<CrawlerThread>> estimate_threads; |
11fdf7f2 | 250 | |
f67539c2 | 251 | static void print_dedup_estimate(std::ostream& out, std::string chunk_algo) |
11fdf7f2 | 252 | { |
f67539c2 TL |
253 | /* |
254 | uint64_t total_bytes = 0; | |
11fdf7f2 | 255 | uint64_t total_objects = 0; |
f67539c2 TL |
256 | */ |
257 | uint64_t examined_objects = 0; | |
258 | uint64_t examined_bytes = 0; | |
11fdf7f2 TL |
259 | |
260 | for (auto &et : estimate_threads) { | |
11fdf7f2 | 261 | examined_objects += et->get_examined_objects(); |
f67539c2 TL |
262 | examined_bytes += et->get_examined_bytes(); |
263 | } | |
264 | ||
265 | auto f = Formatter::create("json-pretty"); | |
266 | f->open_object_section("results"); | |
267 | f->dump_string("chunk_algo", chunk_algo); | |
268 | f->open_array_section("chunk_sizes"); | |
269 | for (auto& i : dedup_estimates) { | |
270 | f->dump_object("chunker", i.second); | |
271 | } | |
272 | f->close_section(); | |
273 | ||
274 | f->open_object_section("summary"); | |
275 | f->dump_unsigned("examined_objects", examined_objects); | |
276 | f->dump_unsigned("examined_bytes", examined_bytes); | |
277 | /* | |
278 | f->dump_unsigned("total_objects", total_objects); | |
279 | f->dump_unsigned("total_bytes", total_bytes); | |
280 | f->dump_float("examined_ratio", (float)examined_bytes / (float)total_bytes); | |
281 | */ | |
282 | f->close_section(); | |
283 | f->close_section(); | |
284 | f->flush(out); | |
11fdf7f2 TL |
285 | } |
286 | ||
287 | static void handle_signal(int signum) | |
288 | { | |
9f95a23c | 289 | std::lock_guard l{glock}; |
11fdf7f2 TL |
290 | for (auto &p : estimate_threads) { |
291 | p->signal(signum); | |
292 | } | |
293 | } | |
294 | ||
11fdf7f2 TL |
295 | void EstimateDedupRatio::estimate_dedup_ratio() |
296 | { | |
297 | ObjectCursor shard_start; | |
298 | ObjectCursor shard_end; | |
11fdf7f2 TL |
299 | |
300 | io_ctx.object_list_slice( | |
301 | begin, | |
302 | end, | |
303 | n, | |
304 | m, | |
305 | &shard_start, | |
306 | &shard_end); | |
307 | ||
f67539c2 TL |
308 | utime_t start = ceph_clock_now(); |
309 | utime_t end; | |
310 | if (max_seconds) { | |
311 | end = start; | |
312 | end += max_seconds; | |
313 | } | |
314 | ||
315 | utime_t next_report; | |
316 | if (report_period) { | |
317 | next_report = start; | |
318 | next_report += report_period; | |
319 | } | |
320 | ||
11fdf7f2 | 321 | ObjectCursor c(shard_start); |
9f95a23c | 322 | while (c < shard_end) |
11fdf7f2 TL |
323 | { |
324 | std::vector<ObjectItem> result; | |
325 | int r = io_ctx.object_list(c, shard_end, 12, {}, &result, &c); | |
326 | if (r < 0 ){ | |
327 | cerr << "error object_list : " << cpp_strerror(r) << std::endl; | |
328 | return; | |
329 | } | |
330 | ||
f67539c2 TL |
331 | unsigned op_size = max_read_size; |
332 | ||
11fdf7f2 TL |
333 | for (const auto & i : result) { |
334 | const auto &oid = i.oid; | |
f67539c2 TL |
335 | |
336 | utime_t now = ceph_clock_now(); | |
337 | if (max_seconds && now > end) { | |
338 | m_stop = true; | |
339 | } | |
340 | if (m_stop) { | |
341 | return; | |
342 | } | |
343 | ||
344 | if (n == 0 && // first thread only | |
345 | next_report != utime_t() && now > next_report) { | |
346 | cerr << (int)(now - start) << "s : read " | |
347 | << dedup_estimates.begin()->second.total_bytes << " bytes so far..." | |
348 | << std::endl; | |
349 | print_dedup_estimate(cerr, chunk_algo); | |
350 | next_report = now; | |
351 | next_report += report_period; | |
352 | } | |
353 | ||
354 | // read entire object | |
355 | bufferlist bl; | |
11fdf7f2 TL |
356 | uint64_t offset = 0; |
357 | while (true) { | |
f67539c2 TL |
358 | bufferlist t; |
359 | int ret = io_ctx.read(oid, t, op_size, offset); | |
360 | if (ret <= 0) { | |
11fdf7f2 TL |
361 | break; |
362 | } | |
f67539c2 TL |
363 | offset += ret; |
364 | bl.claim_append(t); | |
11fdf7f2 TL |
365 | } |
366 | examined_objects++; | |
f67539c2 TL |
367 | examined_bytes += bl.length(); |
368 | ||
369 | // do the chunking | |
370 | for (auto& i : dedup_estimates) { | |
371 | vector<pair<uint64_t, uint64_t>> chunks; | |
372 | i.second.cdc->calc_chunks(bl, &chunks); | |
373 | for (auto& p : chunks) { | |
374 | bufferlist chunk; | |
375 | chunk.substr_of(bl, p.first, p.second); | |
376 | i.second.add_chunk(chunk, fp_algo); | |
377 | if (debug) { | |
378 | cout << " " << oid << " " << p.first << "~" << p.second << std::endl; | |
379 | } | |
380 | } | |
381 | ++i.second.total_objects; | |
382 | } | |
11fdf7f2 TL |
383 | } |
384 | } | |
385 | } | |
386 | ||
11fdf7f2 TL |
387 | void ChunkScrub::chunk_scrub_common() |
388 | { | |
389 | ObjectCursor shard_start; | |
390 | ObjectCursor shard_end; | |
391 | int ret; | |
9f95a23c TL |
392 | Rados rados; |
393 | ||
394 | ret = rados.init_with_context(g_ceph_context); | |
395 | if (ret < 0) { | |
396 | cerr << "couldn't initialize rados: " << cpp_strerror(ret) << std::endl; | |
397 | return; | |
398 | } | |
399 | ret = rados.connect(); | |
400 | if (ret) { | |
401 | cerr << "couldn't connect to cluster: " << cpp_strerror(ret) << std::endl; | |
402 | return; | |
403 | } | |
11fdf7f2 TL |
404 | |
405 | chunk_io_ctx.object_list_slice( | |
406 | begin, | |
407 | end, | |
408 | n, | |
409 | m, | |
410 | &shard_start, | |
411 | &shard_end); | |
412 | ||
413 | ObjectCursor c(shard_start); | |
414 | while(c < shard_end) | |
415 | { | |
416 | std::vector<ObjectItem> result; | |
417 | int r = chunk_io_ctx.object_list(c, shard_end, 12, {}, &result, &c); | |
418 | if (r < 0 ){ | |
419 | cerr << "error object_list : " << cpp_strerror(r) << std::endl; | |
420 | return; | |
421 | } | |
422 | ||
423 | for (const auto & i : result) { | |
9f95a23c | 424 | std::unique_lock l{m_lock}; |
11fdf7f2 TL |
425 | if (m_stop) { |
426 | Formatter *formatter = Formatter::create("json-pretty"); | |
427 | print_status(formatter, cout); | |
428 | delete formatter; | |
429 | return; | |
430 | } | |
431 | auto oid = i.oid; | |
f67539c2 TL |
432 | cout << oid << std::endl; |
433 | chunk_refs_t refs; | |
434 | { | |
435 | bufferlist t; | |
436 | ret = chunk_io_ctx.getxattr(oid, CHUNK_REFCOUNT_ATTR, t); | |
437 | if (ret < 0) { | |
438 | continue; | |
439 | } | |
440 | auto p = t.cbegin(); | |
441 | decode(refs, p); | |
442 | } | |
443 | ||
444 | examined_objects++; | |
445 | if (refs.get_type() != chunk_refs_t::TYPE_BY_OBJECT) { | |
446 | // we can't do anything here | |
11fdf7f2 TL |
447 | continue; |
448 | } | |
449 | ||
f67539c2 TL |
450 | // check all objects |
451 | chunk_refs_by_object_t *byo = | |
452 | static_cast<chunk_refs_by_object_t*>(refs.r.get()); | |
453 | set<hobject_t> real_refs; | |
454 | ||
455 | uint64_t pool_missing = 0; | |
456 | uint64_t object_missing = 0; | |
457 | uint64_t does_not_ref = 0; | |
458 | for (auto& pp : byo->by_object) { | |
9f95a23c TL |
459 | IoCtx target_io_ctx; |
460 | ret = rados.ioctx_create2(pp.pool, target_io_ctx); | |
461 | if (ret < 0) { | |
f67539c2 TL |
462 | cerr << oid << " ref " << pp |
463 | << ": referencing pool does not exist" << std::endl; | |
464 | ++pool_missing; | |
9f95a23c TL |
465 | continue; |
466 | } | |
467 | ||
f67539c2 TL |
468 | ret = cls_cas_references_chunk(target_io_ctx, pp.oid.name, oid); |
469 | if (ret == -ENOENT) { | |
470 | cerr << oid << " ref " << pp | |
471 | << ": referencing object missing" << std::endl; | |
472 | ++object_missing; | |
473 | } else if (ret == -ENOLINK) { | |
474 | cerr << oid << " ref " << pp | |
475 | << ": referencing object does not reference chunk" | |
476 | << std::endl; | |
477 | ++does_not_ref; | |
11fdf7f2 | 478 | } |
11fdf7f2 | 479 | } |
f67539c2 TL |
480 | if (pool_missing || object_missing || does_not_ref) { |
481 | ++damaged_objects; | |
11fdf7f2 TL |
482 | } |
483 | } | |
484 | } | |
f67539c2 | 485 | cout << "--done--" << std::endl; |
11fdf7f2 TL |
486 | } |
487 | ||
488 | void ChunkScrub::print_status(Formatter *f, ostream &out) | |
489 | { | |
490 | if (f) { | |
491 | f->open_array_section("chunk_scrub"); | |
492 | f->dump_string("PID", stringify(get_pid())); | |
493 | f->open_object_section("Status"); | |
494 | f->dump_string("Total object", stringify(total_objects)); | |
f67539c2 TL |
495 | f->dump_string("Examined objects", stringify(examined_objects)); |
496 | f->dump_string("damaged objects", stringify(damaged_objects)); | |
11fdf7f2 TL |
497 | f->close_section(); |
498 | f->flush(out); | |
499 | cout << std::endl; | |
500 | } | |
501 | } | |
502 | ||
503 | int estimate_dedup_ratio(const std::map < std::string, std::string > &opts, | |
504 | std::vector<const char*> &nargs) | |
505 | { | |
506 | Rados rados; | |
507 | IoCtx io_ctx; | |
f67539c2 TL |
508 | std::string chunk_algo = "fastcdc"; |
509 | string fp_algo = "sha1"; | |
11fdf7f2 TL |
510 | string pool_name; |
511 | uint64_t chunk_size = 0; | |
f67539c2 TL |
512 | uint64_t min_chunk_size = 8192; |
513 | uint64_t max_chunk_size = 4*1024*1024; | |
11fdf7f2 TL |
514 | unsigned max_thread = default_max_thread; |
515 | uint32_t report_period = default_report_period; | |
9f95a23c | 516 | uint64_t max_read_size = default_op_size; |
f67539c2 | 517 | uint64_t max_seconds = 0; |
11fdf7f2 TL |
518 | int ret; |
519 | std::map<std::string, std::string>::const_iterator i; | |
520 | bool debug = false; | |
521 | ObjectCursor begin; | |
522 | ObjectCursor end; | |
9f95a23c TL |
523 | librados::pool_stat_t s; |
524 | list<string> pool_names; | |
525 | map<string, librados::pool_stat_t> stats; | |
11fdf7f2 TL |
526 | |
527 | i = opts.find("pool"); | |
528 | if (i != opts.end()) { | |
529 | pool_name = i->second.c_str(); | |
530 | } | |
531 | i = opts.find("chunk-algorithm"); | |
532 | if (i != opts.end()) { | |
533 | chunk_algo = i->second.c_str(); | |
f67539c2 TL |
534 | if (!CDC::create(chunk_algo, 12)) { |
535 | cerr << "unrecognized chunk-algorithm " << chunk_algo << std::endl; | |
536 | exit(1); | |
11fdf7f2 TL |
537 | } |
538 | } else { | |
f67539c2 TL |
539 | cerr << "must specify chunk-algorithm" << std::endl; |
540 | exit(1); | |
11fdf7f2 TL |
541 | } |
542 | ||
543 | i = opts.find("fingerprint-algorithm"); | |
544 | if (i != opts.end()) { | |
545 | fp_algo = i->second.c_str(); | |
f67539c2 | 546 | if (fp_algo != "sha1" |
9f95a23c | 547 | && fp_algo != "sha256" && fp_algo != "sha512") { |
f67539c2 TL |
548 | cerr << "unrecognized fingerprint-algorithm " << fp_algo << std::endl; |
549 | exit(1); | |
11fdf7f2 | 550 | } |
11fdf7f2 TL |
551 | } |
552 | ||
553 | i = opts.find("chunk-size"); | |
554 | if (i != opts.end()) { | |
555 | if (rados_sistrtoll(i, &chunk_size)) { | |
556 | return -EINVAL; | |
557 | } | |
11fdf7f2 TL |
558 | } |
559 | ||
f67539c2 | 560 | i = opts.find("min-chunk-size"); |
11fdf7f2 | 561 | if (i != opts.end()) { |
f67539c2 | 562 | if (rados_sistrtoll(i, &min_chunk_size)) { |
11fdf7f2 TL |
563 | return -EINVAL; |
564 | } | |
f67539c2 TL |
565 | } |
566 | i = opts.find("max-chunk-size"); | |
9f95a23c | 567 | if (i != opts.end()) { |
f67539c2 | 568 | if (rados_sistrtoll(i, &max_chunk_size)) { |
9f95a23c TL |
569 | return -EINVAL; |
570 | } | |
f67539c2 TL |
571 | } |
572 | ||
573 | i = opts.find("max-thread"); | |
9f95a23c | 574 | if (i != opts.end()) { |
f67539c2 | 575 | if (rados_sistrtoll(i, &max_thread)) { |
9f95a23c TL |
576 | return -EINVAL; |
577 | } | |
578 | } | |
f67539c2 TL |
579 | |
580 | i = opts.find("report-period"); | |
9f95a23c | 581 | if (i != opts.end()) { |
f67539c2 | 582 | if (rados_sistrtoll(i, &report_period)) { |
9f95a23c TL |
583 | return -EINVAL; |
584 | } | |
f67539c2 TL |
585 | } |
586 | i = opts.find("max-seconds"); | |
9f95a23c | 587 | if (i != opts.end()) { |
f67539c2 | 588 | if (rados_sistrtoll(i, &max_seconds)) { |
9f95a23c TL |
589 | return -EINVAL; |
590 | } | |
f67539c2 TL |
591 | } |
592 | i = opts.find("max-read-size"); | |
9f95a23c | 593 | if (i != opts.end()) { |
f67539c2 | 594 | if (rados_sistrtoll(i, &max_read_size)) { |
9f95a23c TL |
595 | return -EINVAL; |
596 | } | |
597 | } | |
11fdf7f2 TL |
598 | i = opts.find("debug"); |
599 | if (i != opts.end()) { | |
600 | debug = true; | |
601 | } | |
602 | ||
603 | i = opts.find("pgid"); | |
604 | boost::optional<pg_t> pgid(i != opts.end(), pg_t()); | |
605 | ||
606 | ret = rados.init_with_context(g_ceph_context); | |
607 | if (ret < 0) { | |
608 | cerr << "couldn't initialize rados: " << cpp_strerror(ret) << std::endl; | |
609 | goto out; | |
610 | } | |
611 | ret = rados.connect(); | |
612 | if (ret) { | |
613 | cerr << "couldn't connect to cluster: " << cpp_strerror(ret) << std::endl; | |
614 | ret = -1; | |
615 | goto out; | |
616 | } | |
617 | if (pool_name.empty()) { | |
618 | cerr << "--create-pool requested but pool_name was not specified!" << std::endl; | |
f67539c2 | 619 | exit(1); |
11fdf7f2 TL |
620 | } |
621 | ret = rados.ioctx_create(pool_name.c_str(), io_ctx); | |
622 | if (ret < 0) { | |
623 | cerr << "error opening pool " | |
624 | << pool_name << ": " | |
625 | << cpp_strerror(ret) << std::endl; | |
626 | goto out; | |
627 | } | |
628 | ||
f67539c2 TL |
629 | // set up chunkers |
630 | if (chunk_size) { | |
631 | dedup_estimates.emplace(std::piecewise_construct, | |
632 | std::forward_as_tuple(chunk_size), | |
633 | std::forward_as_tuple(chunk_algo, cbits(chunk_size)-1)); | |
634 | } else { | |
635 | for (size_t cs = min_chunk_size; cs <= max_chunk_size; cs *= 2) { | |
636 | dedup_estimates.emplace(std::piecewise_construct, | |
637 | std::forward_as_tuple(cs), | |
638 | std::forward_as_tuple(chunk_algo, cbits(cs)-1)); | |
639 | } | |
640 | } | |
641 | ||
9f95a23c | 642 | glock.lock(); |
11fdf7f2 TL |
643 | begin = io_ctx.object_list_begin(); |
644 | end = io_ctx.object_list_end(); | |
9f95a23c TL |
645 | pool_names.push_back(pool_name); |
646 | ret = rados.get_pool_stats(pool_names, stats); | |
647 | if (ret < 0) { | |
648 | cerr << "error fetching pool stats: " << cpp_strerror(ret) << std::endl; | |
649 | glock.unlock(); | |
650 | return ret; | |
651 | } | |
652 | if (stats.find(pool_name) == stats.end()) { | |
653 | cerr << "stats can not find pool name: " << pool_name << std::endl; | |
654 | glock.unlock(); | |
655 | return ret; | |
656 | } | |
657 | s = stats[pool_name]; | |
658 | ||
11fdf7f2 | 659 | for (unsigned i = 0; i < max_thread; i++) { |
f67539c2 TL |
660 | std::unique_ptr<CrawlerThread> ptr ( |
661 | new EstimateDedupRatio(io_ctx, i, max_thread, begin, end, | |
662 | chunk_algo, fp_algo, chunk_size, | |
663 | report_period, s.num_objects, max_read_size, | |
664 | max_seconds)); | |
11fdf7f2 | 665 | ptr->create("estimate_thread"); |
f67539c2 | 666 | ptr->set_debug(debug); |
11fdf7f2 TL |
667 | estimate_threads.push_back(move(ptr)); |
668 | } | |
9f95a23c | 669 | glock.unlock(); |
11fdf7f2 TL |
670 | |
671 | for (auto &p : estimate_threads) { | |
672 | p->join(); | |
673 | } | |
674 | ||
f67539c2 | 675 | print_dedup_estimate(cout, chunk_algo); |
11fdf7f2 TL |
676 | |
677 | out: | |
678 | return (ret < 0) ? 1 : 0; | |
679 | } | |
680 | ||
681 | static void print_chunk_scrub() | |
682 | { | |
683 | uint64_t total_objects = 0; | |
684 | uint64_t examined_objects = 0; | |
f67539c2 | 685 | int damaged_objects = 0; |
11fdf7f2 TL |
686 | |
687 | for (auto &et : estimate_threads) { | |
9f95a23c TL |
688 | if (!total_objects) { |
689 | total_objects = et->get_total_objects(); | |
690 | } | |
11fdf7f2 TL |
691 | examined_objects += et->get_examined_objects(); |
692 | ChunkScrub *ptr = static_cast<ChunkScrub*>(et.get()); | |
f67539c2 | 693 | damaged_objects += ptr->get_damaged_objects(); |
11fdf7f2 TL |
694 | } |
695 | ||
696 | cout << " Total object : " << total_objects << std::endl; | |
697 | cout << " Examined object : " << examined_objects << std::endl; | |
f67539c2 | 698 | cout << " Damaged object : " << damaged_objects << std::endl; |
11fdf7f2 TL |
699 | } |
700 | ||
701 | int chunk_scrub_common(const std::map < std::string, std::string > &opts, | |
702 | std::vector<const char*> &nargs) | |
703 | { | |
704 | Rados rados; | |
705 | IoCtx io_ctx, chunk_io_ctx; | |
706 | std::string object_name, target_object_name; | |
9f95a23c | 707 | string chunk_pool_name, op_name; |
11fdf7f2 TL |
708 | int ret; |
709 | unsigned max_thread = default_max_thread; | |
710 | std::map<std::string, std::string>::const_iterator i; | |
711 | uint32_t report_period = default_report_period; | |
712 | ObjectCursor begin; | |
713 | ObjectCursor end; | |
9f95a23c TL |
714 | librados::pool_stat_t s; |
715 | list<string> pool_names; | |
716 | map<string, librados::pool_stat_t> stats; | |
11fdf7f2 | 717 | |
11fdf7f2 TL |
718 | i = opts.find("op_name"); |
719 | if (i != opts.end()) { | |
720 | op_name= i->second.c_str(); | |
721 | } else { | |
f67539c2 TL |
722 | cerr << "must specify op" << std::endl; |
723 | exit(1); | |
11fdf7f2 TL |
724 | } |
725 | ||
726 | i = opts.find("chunk-pool"); | |
727 | if (i != opts.end()) { | |
728 | chunk_pool_name = i->second.c_str(); | |
729 | } else { | |
f67539c2 TL |
730 | cerr << "must specify --chunk-pool" << std::endl; |
731 | exit(1); | |
11fdf7f2 TL |
732 | } |
733 | i = opts.find("max-thread"); | |
734 | if (i != opts.end()) { | |
735 | if (rados_sistrtoll(i, &max_thread)) { | |
736 | return -EINVAL; | |
737 | } | |
738 | } | |
739 | i = opts.find("report-period"); | |
740 | if (i != opts.end()) { | |
741 | if (rados_sistrtoll(i, &report_period)) { | |
742 | return -EINVAL; | |
743 | } | |
744 | } | |
745 | i = opts.find("pgid"); | |
746 | boost::optional<pg_t> pgid(i != opts.end(), pg_t()); | |
747 | ||
748 | ret = rados.init_with_context(g_ceph_context); | |
749 | if (ret < 0) { | |
750 | cerr << "couldn't initialize rados: " << cpp_strerror(ret) << std::endl; | |
751 | goto out; | |
752 | } | |
753 | ret = rados.connect(); | |
754 | if (ret) { | |
755 | cerr << "couldn't connect to cluster: " << cpp_strerror(ret) << std::endl; | |
756 | ret = -1; | |
757 | goto out; | |
758 | } | |
11fdf7f2 TL |
759 | ret = rados.ioctx_create(chunk_pool_name.c_str(), chunk_io_ctx); |
760 | if (ret < 0) { | |
761 | cerr << "error opening pool " | |
762 | << chunk_pool_name << ": " | |
763 | << cpp_strerror(ret) << std::endl; | |
764 | goto out; | |
765 | } | |
766 | ||
f67539c2 TL |
767 | if (op_name == "chunk-get-ref" || |
768 | op_name == "chunk-put-ref") { | |
11fdf7f2 | 769 | string target_object_name; |
9f95a23c | 770 | uint64_t pool_id; |
11fdf7f2 TL |
771 | i = opts.find("object"); |
772 | if (i != opts.end()) { | |
773 | object_name = i->second.c_str(); | |
774 | } else { | |
f67539c2 TL |
775 | cerr << "must specify object" << std::endl; |
776 | exit(1); | |
11fdf7f2 TL |
777 | } |
778 | i = opts.find("target-ref"); | |
779 | if (i != opts.end()) { | |
780 | target_object_name = i->second.c_str(); | |
781 | } else { | |
f67539c2 TL |
782 | cerr << "must specify target ref" << std::endl; |
783 | exit(1); | |
11fdf7f2 | 784 | } |
9f95a23c TL |
785 | i = opts.find("target-ref-pool-id"); |
786 | if (i != opts.end()) { | |
787 | if (rados_sistrtoll(i, &pool_id)) { | |
788 | return -EINVAL; | |
789 | } | |
790 | } else { | |
f67539c2 TL |
791 | cerr << "must specify target-ref-pool-id" << std::endl; |
792 | exit(1); | |
11fdf7f2 TL |
793 | } |
794 | ||
795 | uint32_t hash; | |
796 | ret = chunk_io_ctx.get_object_hash_position2(object_name, &hash); | |
797 | if (ret < 0) { | |
798 | return ret; | |
799 | } | |
9f95a23c | 800 | hobject_t oid(sobject_t(target_object_name, CEPH_NOSNAP), "", hash, pool_id, ""); |
11fdf7f2 TL |
801 | |
802 | ObjectWriteOperation op; | |
f67539c2 TL |
803 | if (op_name == "chunk-get-ref") { |
804 | cls_cas_chunk_get_ref(op, oid); | |
805 | } else { | |
806 | cls_cas_chunk_put_ref(op, oid); | |
807 | } | |
11fdf7f2 TL |
808 | ret = chunk_io_ctx.operate(object_name, &op); |
809 | if (ret < 0) { | |
810 | cerr << " operate fail : " << cpp_strerror(ret) << std::endl; | |
811 | } | |
812 | ||
813 | return ret; | |
814 | ||
f67539c2 | 815 | } else if (op_name == "dump-chunk-refs") { |
11fdf7f2 TL |
816 | i = opts.find("object"); |
817 | if (i != opts.end()) { | |
818 | object_name = i->second.c_str(); | |
819 | } else { | |
f67539c2 TL |
820 | cerr << "must specify object" << std::endl; |
821 | exit(1); | |
11fdf7f2 | 822 | } |
f67539c2 TL |
823 | bufferlist t; |
824 | ret = chunk_io_ctx.getxattr(object_name, CHUNK_REFCOUNT_ATTR, t); | |
825 | if (ret < 0) { | |
826 | return ret; | |
11fdf7f2 | 827 | } |
f67539c2 TL |
828 | chunk_refs_t refs; |
829 | auto p = t.cbegin(); | |
830 | decode(refs, p); | |
831 | auto f = Formatter::create("json-pretty"); | |
832 | f->dump_object("refs", refs); | |
833 | f->flush(cout); | |
834 | return 0; | |
11fdf7f2 | 835 | } |
f67539c2 | 836 | |
9f95a23c TL |
837 | glock.lock(); |
838 | begin = chunk_io_ctx.object_list_begin(); | |
839 | end = chunk_io_ctx.object_list_end(); | |
840 | pool_names.push_back(chunk_pool_name); | |
841 | ret = rados.get_pool_stats(pool_names, stats); | |
842 | if (ret < 0) { | |
843 | cerr << "error fetching pool stats: " << cpp_strerror(ret) << std::endl; | |
844 | glock.unlock(); | |
845 | return ret; | |
846 | } | |
847 | if (stats.find(chunk_pool_name) == stats.end()) { | |
848 | cerr << "stats can not find pool name: " << chunk_pool_name << std::endl; | |
849 | glock.unlock(); | |
850 | return ret; | |
851 | } | |
852 | s = stats[chunk_pool_name]; | |
853 | ||
11fdf7f2 | 854 | for (unsigned i = 0; i < max_thread; i++) { |
f67539c2 TL |
855 | std::unique_ptr<CrawlerThread> ptr ( |
856 | new ChunkScrub(io_ctx, i, max_thread, begin, end, chunk_io_ctx, | |
857 | report_period, s.num_objects)); | |
11fdf7f2 TL |
858 | ptr->create("estimate_thread"); |
859 | estimate_threads.push_back(move(ptr)); | |
860 | } | |
9f95a23c | 861 | glock.unlock(); |
11fdf7f2 TL |
862 | |
863 | for (auto &p : estimate_threads) { | |
f67539c2 | 864 | cout << "join " << std::endl; |
11fdf7f2 | 865 | p->join(); |
f67539c2 | 866 | cout << "joined " << std::endl; |
11fdf7f2 TL |
867 | } |
868 | ||
869 | print_chunk_scrub(); | |
870 | ||
871 | out: | |
872 | return (ret < 0) ? 1 : 0; | |
873 | } | |
874 | ||
875 | int main(int argc, const char **argv) | |
876 | { | |
877 | vector<const char*> args; | |
878 | argv_to_vec(argc, argv, args); | |
879 | if (args.empty()) { | |
880 | cerr << argv[0] << ": -h or --help for usage" << std::endl; | |
881 | exit(1); | |
882 | } | |
883 | if (ceph_argparse_need_usage(args)) { | |
884 | usage(); | |
885 | exit(0); | |
886 | } | |
887 | ||
888 | std::string fn; | |
889 | string op_name; | |
890 | ||
891 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
892 | CODE_ENVIRONMENT_UTILITY, 0); | |
893 | common_init_finish(g_ceph_context); | |
894 | init_async_signal_handler(); | |
895 | register_async_signal_handler_oneshot(SIGINT, handle_signal); | |
896 | register_async_signal_handler_oneshot(SIGTERM, handle_signal); | |
897 | std::map < std::string, std::string > opts; | |
898 | std::string val; | |
899 | std::vector<const char*>::iterator i; | |
900 | for (i = args.begin(); i != args.end(); ) { | |
901 | if (ceph_argparse_double_dash(args, i)) { | |
902 | break; | |
903 | } else if (ceph_argparse_witharg(args, i, &val, "--op", (char*)NULL)) { | |
904 | opts["op_name"] = val; | |
905 | op_name = val; | |
906 | } else if (ceph_argparse_witharg(args, i, &val, "--pool", (char*)NULL)) { | |
907 | opts["pool"] = val; | |
908 | } else if (ceph_argparse_witharg(args, i, &val, "--object", (char*)NULL)) { | |
909 | opts["object"] = val; | |
910 | } else if (ceph_argparse_witharg(args, i, &val, "--chunk-algorithm", (char*)NULL)) { | |
911 | opts["chunk-algorithm"] = val; | |
912 | } else if (ceph_argparse_witharg(args, i, &val, "--chunk-size", (char*)NULL)) { | |
913 | opts["chunk-size"] = val; | |
914 | } else if (ceph_argparse_witharg(args, i, &val, "--fingerprint-algorithm", (char*)NULL)) { | |
915 | opts["fingerprint-algorithm"] = val; | |
916 | } else if (ceph_argparse_witharg(args, i, &val, "--chunk-pool", (char*)NULL)) { | |
917 | opts["chunk-pool"] = val; | |
918 | } else if (ceph_argparse_witharg(args, i, &val, "--target-ref", (char*)NULL)) { | |
919 | opts["target-ref"] = val; | |
9f95a23c TL |
920 | } else if (ceph_argparse_witharg(args, i, &val, "--target-ref-pool-id", (char*)NULL)) { |
921 | opts["target-ref-pool-id"] = val; | |
11fdf7f2 TL |
922 | } else if (ceph_argparse_witharg(args, i, &val, "--max-thread", (char*)NULL)) { |
923 | opts["max-thread"] = val; | |
924 | } else if (ceph_argparse_witharg(args, i, &val, "--report-period", (char*)NULL)) { | |
925 | opts["report-period"] = val; | |
9f95a23c | 926 | } else if (ceph_argparse_witharg(args, i, &val, "--max-read-size", (char*)NULL)) { |
f67539c2 TL |
927 | opts["max-seconds"] = val; |
928 | } else if (ceph_argparse_witharg(args, i, &val, "--max-seconds", (char*)NULL)) { | |
929 | opts["max-seconds"] = val; | |
930 | } else if (ceph_argparse_witharg(args, i, &val, "--min-chunk-size", (char*)NULL)) { | |
931 | opts["min-chunk-size"] = val; | |
932 | } else if (ceph_argparse_witharg(args, i, &val, "--max-chunk-size", (char*)NULL)) { | |
933 | opts["max-chunk-size"] = val; | |
11fdf7f2 TL |
934 | } else if (ceph_argparse_flag(args, i, "--debug", (char*)NULL)) { |
935 | opts["debug"] = "true"; | |
936 | } else { | |
f67539c2 TL |
937 | if (val[0] == '-') { |
938 | cerr << "unrecognized option " << val << std::endl; | |
939 | exit(1); | |
940 | } | |
11fdf7f2 TL |
941 | ++i; |
942 | } | |
943 | } | |
944 | ||
945 | if (op_name == "estimate") { | |
946 | return estimate_dedup_ratio(opts, args); | |
9f95a23c | 947 | } else if (op_name == "chunk-scrub") { |
11fdf7f2 | 948 | return chunk_scrub_common(opts, args); |
f67539c2 TL |
949 | } else if (op_name == "chunk-get-ref" || |
950 | op_name == "chunk-put-ref") { | |
11fdf7f2 | 951 | return chunk_scrub_common(opts, args); |
f67539c2 | 952 | } else if (op_name == "dump-chunk-refs") { |
11fdf7f2 TL |
953 | return chunk_scrub_common(opts, args); |
954 | } else { | |
f67539c2 TL |
955 | cerr << "unrecognized op " << op_name << std::endl; |
956 | exit(1); | |
11fdf7f2 TL |
957 | } |
958 | ||
959 | unregister_async_signal_handler(SIGINT, handle_signal); | |
960 | unregister_async_signal_handler(SIGTERM, handle_signal); | |
961 | shutdown_async_signal_handler(); | |
962 | ||
963 | return 0; | |
964 | } |