]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <glob.h> | |
16 | #include <stdio.h> | |
17 | #include <string.h> | |
18 | #include <iostream> | |
20effc67 | 19 | #include <memory> |
7c673cae FG |
20 | #include <time.h> |
21 | #include <sys/mount.h> | |
7c673cae FG |
22 | #include <boost/random/mersenne_twister.hpp> |
23 | #include <boost/random/uniform_int.hpp> | |
24 | #include <boost/random/binomial_distribution.hpp> | |
20effc67 | 25 | #include <fmt/format.h> |
7c673cae FG |
26 | #include <gtest/gtest.h> |
27 | ||
28 | #include "os/ObjectStore.h" | |
29 | #include "os/filestore/FileStore.h" | |
11fdf7f2 | 30 | #if defined(WITH_BLUESTORE) |
7c673cae | 31 | #include "os/bluestore/BlueStore.h" |
9f95a23c | 32 | #include "os/bluestore/BlueFS.h" |
7c673cae FG |
33 | #endif |
34 | #include "include/Context.h" | |
20effc67 | 35 | #include "common/buffer_instrumentation.h" |
7c673cae | 36 | #include "common/ceph_argparse.h" |
9f95a23c | 37 | #include "common/admin_socket.h" |
7c673cae | 38 | #include "global/global_init.h" |
9f95a23c | 39 | #include "common/ceph_mutex.h" |
7c673cae FG |
40 | #include "common/Cond.h" |
41 | #include "common/errno.h" | |
20effc67 | 42 | #include "common/options.h" // for the size literals |
a4b75251 | 43 | #include "common/pretty_binary.h" |
7c673cae FG |
44 | #include "include/stringify.h" |
45 | #include "include/coredumpctl.h" | |
7c673cae | 46 | #include "include/unordered_map.h" |
20effc67 | 47 | #include "os/kv.h" |
7c673cae FG |
48 | #include "store_test_fixture.h" |
49 | ||
20effc67 TL |
50 | |
51 | using namespace std; | |
11fdf7f2 | 52 | using namespace std::placeholders; |
7c673cae FG |
53 | |
54 | typedef boost::mt11213b gen_type; | |
55 | ||
94b18763 | 56 | const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000; |
7c673cae FG |
57 | #define dout_context g_ceph_context |
58 | ||
20effc67 TL |
59 | bool smr = false; |
60 | ||
7c673cae FG |
61 | static bool bl_eq(bufferlist& expected, bufferlist& actual) |
62 | { | |
63 | if (expected.contents_equal(actual)) | |
64 | return true; | |
65 | ||
66 | unsigned first = 0; | |
67 | if(expected.length() != actual.length()) { | |
68 | cout << "--- buffer lengths mismatch " << std::hex | |
69 | << "expected 0x" << expected.length() << " != actual 0x" | |
70 | << actual.length() << std::dec << std::endl; | |
71 | derr << "--- buffer lengths mismatch " << std::hex | |
72 | << "expected 0x" << expected.length() << " != actual 0x" | |
73 | << actual.length() << std::dec << dendl; | |
74 | } | |
11fdf7f2 | 75 | auto len = std::min(expected.length(), actual.length()); |
7c673cae FG |
76 | while ( first<len && expected[first] == actual[first]) |
77 | ++first; | |
78 | unsigned last = len; | |
79 | while (last > 0 && expected[last-1] == actual[last-1]) | |
80 | --last; | |
81 | if(len > 0) { | |
82 | cout << "--- buffer mismatch between offset 0x" << std::hex << first | |
83 | << " and 0x" << last << ", total 0x" << len << std::dec | |
84 | << std::endl; | |
85 | derr << "--- buffer mismatch between offset 0x" << std::hex << first | |
86 | << " and 0x" << last << ", total 0x" << len << std::dec | |
87 | << dendl; | |
88 | cout << "--- expected:\n"; | |
89 | expected.hexdump(cout); | |
90 | cout << "--- actual:\n"; | |
91 | actual.hexdump(cout); | |
92 | } | |
93 | return false; | |
94 | } | |
95 | ||
96 | ||
97 | ||
98 | template <typename T> | |
11fdf7f2 | 99 | int queue_transaction( |
7c673cae | 100 | T &store, |
11fdf7f2 | 101 | ObjectStore::CollectionHandle ch, |
7c673cae FG |
102 | ObjectStore::Transaction &&t) { |
103 | if (rand() % 2) { | |
104 | ObjectStore::Transaction t2; | |
105 | t2.append(t); | |
11fdf7f2 | 106 | return store->queue_transaction(ch, std::move(t2)); |
7c673cae | 107 | } else { |
11fdf7f2 | 108 | return store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
109 | } |
110 | } | |
111 | ||
f91f0fd5 TL |
112 | template <typename T> |
113 | int collection_list(T &store, ObjectStore::CollectionHandle &c, | |
114 | const ghobject_t& start, const ghobject_t& end, int max, | |
115 | vector<ghobject_t> *ls, ghobject_t *pnext, | |
116 | bool disable_legacy = false) { | |
117 | if (disable_legacy || rand() % 2) { | |
118 | return store->collection_list(c, start, end, max, ls, pnext); | |
119 | } else { | |
120 | return store->collection_list_legacy(c, start, end, max, ls, pnext); | |
121 | } | |
122 | } | |
7c673cae FG |
123 | |
124 | bool sorted(const vector<ghobject_t> &in) { | |
125 | ghobject_t start; | |
126 | for (vector<ghobject_t>::const_iterator i = in.begin(); | |
127 | i != in.end(); | |
128 | ++i) { | |
129 | if (start > *i) { | |
130 | cout << start << " should follow " << *i << std::endl; | |
131 | return false; | |
132 | } | |
133 | start = *i; | |
134 | } | |
135 | return true; | |
136 | } | |
137 | ||
138 | class StoreTest : public StoreTestFixture, | |
139 | public ::testing::WithParamInterface<const char*> { | |
140 | public: | |
141 | StoreTest() | |
142 | : StoreTestFixture(GetParam()) | |
143 | {} | |
11fdf7f2 TL |
144 | void doCompressionTest(); |
145 | void doSyntheticTest( | |
146 | int num_ops, | |
147 | uint64_t max_obj, uint64_t max_wr, uint64_t align); | |
7c673cae FG |
148 | }; |
149 | ||
150 | class StoreTestDeferredSetup : public StoreTest { | |
151 | void SetUp() override { | |
152 | //do nothing | |
153 | } | |
154 | ||
155 | protected: | |
156 | void DeferredSetup() { | |
157 | StoreTest::SetUp(); | |
158 | } | |
159 | ||
160 | public: | |
161 | }; | |
162 | ||
20effc67 | 163 | |
7c673cae FG |
164 | class StoreTestSpecificAUSize : public StoreTestDeferredSetup { |
165 | ||
166 | public: | |
167 | typedef | |
168 | std::function<void( | |
7c673cae FG |
169 | uint64_t num_ops, |
170 | uint64_t max_obj, | |
171 | uint64_t max_wr, | |
172 | uint64_t align)> MatrixTest; | |
173 | ||
174 | void StartDeferred(size_t min_alloc_size) { | |
11fdf7f2 | 175 | SetVal(g_conf(), "bluestore_min_alloc_size", stringify(min_alloc_size).c_str()); |
7c673cae FG |
176 | DeferredSetup(); |
177 | } | |
7c673cae FG |
178 | |
179 | private: | |
180 | // bluestore matrix testing | |
181 | uint64_t max_write = 40 * 1024; | |
182 | uint64_t max_size = 400 * 1024; | |
183 | uint64_t alignment = 0; | |
184 | uint64_t num_ops = 10000; | |
185 | ||
186 | protected: | |
187 | string matrix_get(const char *k) { | |
188 | if (string(k) == "max_write") { | |
189 | return stringify(max_write); | |
190 | } else if (string(k) == "max_size") { | |
191 | return stringify(max_size); | |
192 | } else if (string(k) == "alignment") { | |
193 | return stringify(alignment); | |
194 | } else if (string(k) == "num_ops") { | |
195 | return stringify(num_ops); | |
196 | } else { | |
197 | char *buf; | |
11fdf7f2 | 198 | g_conf().get_val(k, &buf, -1); |
7c673cae FG |
199 | string v = buf; |
200 | free(buf); | |
201 | return v; | |
202 | } | |
203 | } | |
204 | ||
205 | void matrix_set(const char *k, const char *v) { | |
206 | if (string(k) == "max_write") { | |
207 | max_write = atoll(v); | |
208 | } else if (string(k) == "max_size") { | |
209 | max_size = atoll(v); | |
210 | } else if (string(k) == "alignment") { | |
211 | alignment = atoll(v); | |
212 | } else if (string(k) == "num_ops") { | |
213 | num_ops = atoll(v); | |
214 | } else { | |
11fdf7f2 | 215 | SetVal(g_conf(), k, v); |
7c673cae FG |
216 | } |
217 | } | |
218 | ||
219 | void do_matrix_choose(const char *matrix[][10], | |
220 | int i, int pos, int num, | |
7c673cae FG |
221 | MatrixTest fn) { |
222 | if (matrix[i][0]) { | |
223 | int count; | |
224 | for (count = 0; matrix[i][count+1]; ++count) ; | |
225 | for (int j = 1; matrix[i][j]; ++j) { | |
226 | matrix_set(matrix[i][0], matrix[i][j]); | |
227 | do_matrix_choose(matrix, | |
228 | i + 1, | |
229 | pos * count + j - 1, | |
230 | num * count, | |
7c673cae FG |
231 | fn); |
232 | } | |
233 | } else { | |
234 | cout << "---------------------- " << (pos + 1) << " / " << num | |
235 | << " ----------------------" << std::endl; | |
236 | for (unsigned k=0; matrix[k][0]; ++k) { | |
237 | cout << " " << matrix[k][0] << " = " << matrix_get(matrix[k][0]) | |
238 | << std::endl; | |
239 | } | |
11fdf7f2 TL |
240 | g_ceph_context->_conf.apply_changes(nullptr); |
241 | fn(num_ops, max_size, max_write, alignment); | |
7c673cae FG |
242 | } |
243 | } | |
244 | ||
245 | void do_matrix(const char *matrix[][10], | |
7c673cae | 246 | MatrixTest fn) { |
7c673cae FG |
247 | |
248 | if (strcmp(matrix[0][0], "bluestore_min_alloc_size") == 0) { | |
249 | int count; | |
250 | for (count = 0; matrix[0][count+1]; ++count) ; | |
251 | for (size_t j = 1; matrix[0][j]; ++j) { | |
252 | if (j > 1) { | |
253 | TearDown(); | |
254 | } | |
255 | StartDeferred(strtoll(matrix[0][j], NULL, 10)); | |
11fdf7f2 | 256 | do_matrix_choose(matrix, 1, j - 1, count, fn); |
7c673cae FG |
257 | } |
258 | } else { | |
259 | StartDeferred(0); | |
11fdf7f2 | 260 | do_matrix_choose(matrix, 0, 0, 1, fn); |
7c673cae | 261 | } |
7c673cae FG |
262 | } |
263 | ||
264 | }; | |
265 | ||
a4b75251 TL |
266 | class StoreTestOmapUpgrade : public StoreTestDeferredSetup { |
267 | protected: | |
268 | void StartDeferred() { | |
269 | DeferredSetup(); | |
270 | } | |
271 | ||
272 | public: | |
273 | struct generator { | |
274 | double r = 3.6; | |
275 | double x = 0.5; | |
276 | double operator()(){ | |
277 | double v = x; | |
278 | x = r * x * (1 - x); | |
279 | return v; | |
280 | } | |
281 | }; | |
282 | ||
283 | std::string generate_monotonic_name(uint32_t SUM, uint32_t i, double r, double x) | |
284 | { | |
285 | generator gen{r, x}; | |
286 | //std::cout << "r=" << r << " x=" << x << std::endl; | |
287 | std::string s; | |
288 | while (SUM > 1) { | |
289 | uint32_t lo = 0; | |
290 | uint32_t hi = 1 + gen() * 10; | |
291 | uint32_t start = ('z' - 'a' + 1 - hi) * gen(); | |
292 | while (hi - lo > 0) { | |
293 | uint32_t mid = (lo + hi + 1 + (SUM&1)) / 2; // round up or down, depending on SUM | |
294 | // std::cout << "SUM=" << SUM << " x=" << gen.x << std::endl; | |
295 | uint32_t mid_val = gen() * (SUM - 1) + 1; | |
296 | // LEFT = lo .. mid - 1 | |
297 | // RIGHT = mid .. hi | |
298 | // std::cout << "lo=" << lo << " hi=" << hi << " mid=" << mid | |
299 | // << " SUM=" << SUM << " i=" << i << " x=" << gen.x << " mid_val=" << mid_val << std::endl; | |
300 | if (i < mid_val) { | |
301 | hi = mid - 1; | |
302 | SUM = mid_val; | |
303 | } else { | |
304 | lo = mid; | |
305 | SUM = SUM - mid_val; | |
306 | i = i - mid_val; | |
307 | } | |
308 | } | |
309 | //std::cout << "lo=" << lo << " hi=" << hi | |
310 | // << " SUM=" << SUM << " i=" << i << std::endl; | |
311 | ||
312 | s.push_back('a' + lo + start); // to keep alphabetic order | |
313 | uint32_t cnt = gen() * 8; | |
314 | for (uint32_t j = 0; j < cnt; j++) { | |
315 | s.push_back('a' + ('z' - 'a' + 1) * gen()); | |
316 | } | |
317 | s.push_back('.'); | |
318 | } | |
319 | return s; | |
320 | } | |
321 | ||
322 | std::string gen_string(size_t size, generator& gen) { | |
323 | std::string s; | |
324 | for (size_t i = 0; i < size; i++) { | |
325 | s.push_back('a' + ('z' - 'a' + 1 ) * gen()); | |
326 | } | |
327 | return s; | |
328 | } | |
329 | ||
330 | void make_omap_data(size_t object_count, | |
331 | int64_t poolid, | |
332 | coll_t cid) { | |
333 | int r; | |
334 | ObjectStore::CollectionHandle ch = store->open_collection(cid); | |
335 | for (size_t o = 0; o < object_count; o++) | |
336 | { | |
337 | ObjectStore::Transaction t; | |
338 | std::string oid = generate_monotonic_name(object_count, o, 3.71, 0.5); | |
339 | ghobject_t hoid(hobject_t(oid, "", CEPH_NOSNAP, 0, poolid, "")); | |
340 | t.touch(cid, hoid); | |
341 | generator gen{3.85 + 0.1 * o / object_count, 1 - double(o) / object_count}; | |
342 | ||
343 | map<string, bufferlist> start_set; | |
344 | size_t omap_count = 1 + gen() * 20; | |
345 | bool do_omap_header = gen() > 0.5; | |
346 | if (do_omap_header) { | |
347 | bufferlist header; | |
348 | header.append(gen_string(50, gen)); | |
349 | t.omap_setheader(cid, hoid, header); | |
350 | } | |
351 | for (size_t i = 0; i < omap_count; i++) { | |
352 | std::string name = generate_monotonic_name(omap_count, i, 3.66 + 0.22 * o / object_count, 0.5); | |
353 | bufferlist val; | |
354 | val.append(gen_string(100, gen)); | |
355 | start_set.emplace(name, val); | |
356 | } | |
357 | t.omap_setkeys(cid, hoid, start_set); | |
358 | r = queue_transaction(store, ch, std::move(t)); | |
359 | ASSERT_EQ(r, 0); | |
360 | } | |
361 | } | |
362 | ||
363 | void check_omap_data(size_t object_count, | |
364 | int64_t poolid, | |
365 | coll_t cid) { | |
366 | int r; | |
367 | ObjectStore::CollectionHandle ch = store->open_collection(cid); | |
368 | ||
369 | for (size_t o = 0; o < object_count; o++) | |
370 | { | |
371 | ObjectStore::Transaction t; | |
372 | std::string oid = generate_monotonic_name(object_count, o, 3.71, 0.5); | |
373 | ghobject_t hoid(hobject_t(oid, "", CEPH_NOSNAP, 0, poolid, "")); | |
374 | generator gen{3.85 + 0.1 * o / object_count, 1 - double(o) / object_count}; | |
375 | ||
376 | bufferlist omap_header; | |
377 | map<string, bufferlist> omap_set; | |
378 | r = store->omap_get(ch, hoid, &omap_header, &omap_set); | |
379 | ASSERT_EQ(r, 0); | |
380 | size_t omap_count = 1 + gen() * 20; | |
381 | bool do_omap_header = gen() > 0.5; | |
382 | if (do_omap_header) { | |
383 | std::string header_str = gen_string(50, gen); | |
384 | ASSERT_EQ(header_str, omap_header.to_str()); | |
385 | } | |
386 | auto it = omap_set.begin(); | |
387 | for (size_t i = 0; i < omap_count; i++) { | |
388 | ASSERT_TRUE(it != omap_set.end()); | |
389 | std::string name = generate_monotonic_name(omap_count, i, 3.66 + 0.22 * o / object_count, 0.5); | |
390 | std::string val_gen = gen_string(100, gen); | |
391 | ASSERT_EQ(it->first, name); | |
392 | ASSERT_EQ(it->second.to_str(), val_gen); | |
393 | ++it; | |
394 | } | |
395 | } | |
396 | } | |
397 | }; | |
398 | ||
7c673cae FG |
399 | TEST_P(StoreTest, collect_metadata) { |
400 | map<string,string> pm; | |
401 | store->collect_metadata(&pm); | |
402 | if (GetParam() == string("filestore")) { | |
403 | ASSERT_NE(pm.count("filestore_backend"), 0u); | |
404 | ASSERT_NE(pm.count("filestore_f_type"), 0u); | |
405 | ASSERT_NE(pm.count("backend_filestore_partition_path"), 0u); | |
406 | ASSERT_NE(pm.count("backend_filestore_dev_node"), 0u); | |
407 | } | |
408 | } | |
409 | ||
410 | TEST_P(StoreTest, Trivial) { | |
411 | } | |
412 | ||
413 | TEST_P(StoreTest, TrivialRemount) { | |
414 | int r = store->umount(); | |
415 | ASSERT_EQ(0, r); | |
416 | r = store->mount(); | |
417 | ASSERT_EQ(0, r); | |
418 | } | |
419 | ||
20effc67 TL |
420 | TEST_P(StoreTest, TrivialRemountFsck) { |
421 | if(string(GetParam()) != "bluestore") | |
422 | return; | |
423 | int r = store->umount(); | |
424 | ASSERT_EQ(0, r); | |
425 | r = store->fsck(false); | |
426 | ASSERT_EQ(0, r); | |
427 | r = store->mount(); | |
428 | ASSERT_EQ(0, r); | |
429 | } | |
430 | ||
7c673cae | 431 | TEST_P(StoreTest, SimpleRemount) { |
7c673cae FG |
432 | coll_t cid; |
433 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
434 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
435 | bufferlist bl; | |
436 | bl.append("1234512345"); | |
437 | int r; | |
11fdf7f2 | 438 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
439 | { |
440 | cerr << "create collection + write" << std::endl; | |
441 | ObjectStore::Transaction t; | |
442 | t.create_collection(cid, 0); | |
443 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 444 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
445 | ASSERT_EQ(r, 0); |
446 | } | |
11fdf7f2 | 447 | ch.reset(); |
7c673cae FG |
448 | r = store->umount(); |
449 | ASSERT_EQ(0, r); | |
450 | r = store->mount(); | |
451 | ASSERT_EQ(0, r); | |
11fdf7f2 | 452 | ch = store->open_collection(cid); |
7c673cae FG |
453 | { |
454 | ObjectStore::Transaction t; | |
455 | t.write(cid, hoid2, 0, bl.length(), bl); | |
11fdf7f2 | 456 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
457 | ASSERT_EQ(r, 0); |
458 | } | |
459 | { | |
460 | ObjectStore::Transaction t; | |
461 | t.remove(cid, hoid); | |
462 | t.remove(cid, hoid2); | |
463 | t.remove_collection(cid); | |
464 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 465 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
466 | ASSERT_EQ(r, 0); |
467 | } | |
11fdf7f2 | 468 | ch.reset(); |
7c673cae FG |
469 | r = store->umount(); |
470 | ASSERT_EQ(0, r); | |
471 | r = store->mount(); | |
472 | ASSERT_EQ(0, r); | |
11fdf7f2 | 473 | ch = store->create_new_collection(cid); |
7c673cae FG |
474 | { |
475 | ObjectStore::Transaction t; | |
476 | t.create_collection(cid, 0); | |
11fdf7f2 | 477 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 478 | ASSERT_EQ(r, 0); |
11fdf7f2 | 479 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
480 | ASSERT_TRUE(!exists); |
481 | } | |
482 | { | |
483 | ObjectStore::Transaction t; | |
484 | t.remove_collection(cid); | |
485 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 486 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
487 | ASSERT_EQ(r, 0); |
488 | } | |
489 | } | |
490 | ||
491 | TEST_P(StoreTest, IORemount) { | |
7c673cae FG |
492 | coll_t cid; |
493 | bufferlist bl; | |
494 | bl.append("1234512345"); | |
495 | int r; | |
11fdf7f2 | 496 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
497 | { |
498 | cerr << "create collection + objects" << std::endl; | |
499 | ObjectStore::Transaction t; | |
500 | t.create_collection(cid, 0); | |
501 | for (int n=1; n<=100; ++n) { | |
502 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
503 | t.write(cid, hoid, 0, bl.length(), bl); | |
504 | } | |
11fdf7f2 | 505 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
506 | ASSERT_EQ(r, 0); |
507 | } | |
508 | // overwrites | |
509 | { | |
510 | cout << "overwrites" << std::endl; | |
511 | for (int n=1; n<=100; ++n) { | |
512 | ObjectStore::Transaction t; | |
513 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
514 | t.write(cid, hoid, 1, bl.length(), bl); | |
11fdf7f2 | 515 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
516 | ASSERT_EQ(r, 0); |
517 | } | |
518 | } | |
11fdf7f2 | 519 | ch.reset(); |
7c673cae FG |
520 | r = store->umount(); |
521 | ASSERT_EQ(0, r); | |
522 | r = store->mount(); | |
523 | ASSERT_EQ(0, r); | |
524 | { | |
525 | ObjectStore::Transaction t; | |
526 | for (int n=1; n<=100; ++n) { | |
527 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
528 | t.remove(cid, hoid); | |
529 | } | |
530 | t.remove_collection(cid); | |
11fdf7f2 TL |
531 | auto ch = store->open_collection(cid); |
532 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
533 | ASSERT_EQ(r, 0); |
534 | } | |
535 | } | |
536 | ||
537 | TEST_P(StoreTest, UnprintableCharsName) { | |
7c673cae FG |
538 | coll_t cid; |
539 | string name = "funnychars_"; | |
540 | for (unsigned i = 0; i < 256; ++i) { | |
541 | name.push_back(i); | |
542 | } | |
543 | ghobject_t oid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
544 | int r; | |
11fdf7f2 | 545 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
546 | { |
547 | cerr << "create collection + object" << std::endl; | |
548 | ObjectStore::Transaction t; | |
549 | t.create_collection(cid, 0); | |
550 | t.touch(cid, oid); | |
11fdf7f2 | 551 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
552 | ASSERT_EQ(r, 0); |
553 | } | |
11fdf7f2 | 554 | ch.reset(); |
7c673cae FG |
555 | r = store->umount(); |
556 | ASSERT_EQ(0, r); | |
557 | r = store->mount(); | |
558 | ASSERT_EQ(0, r); | |
559 | { | |
560 | cout << "removing" << std::endl; | |
561 | ObjectStore::Transaction t; | |
562 | t.remove(cid, oid); | |
563 | t.remove_collection(cid); | |
11fdf7f2 TL |
564 | auto ch = store->open_collection(cid); |
565 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
566 | ASSERT_EQ(r, 0); |
567 | } | |
568 | } | |
569 | ||
570 | TEST_P(StoreTest, FiemapEmpty) { | |
7c673cae FG |
571 | coll_t cid; |
572 | int r = 0; | |
573 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
11fdf7f2 | 574 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
575 | { |
576 | ObjectStore::Transaction t; | |
577 | t.create_collection(cid, 0); | |
578 | t.touch(cid, oid); | |
579 | t.truncate(cid, oid, 100000); | |
11fdf7f2 | 580 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
581 | ASSERT_EQ(r, 0); |
582 | } | |
583 | { | |
584 | bufferlist bl; | |
11fdf7f2 | 585 | store->fiemap(ch, oid, 0, 100000, bl); |
7c673cae | 586 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
587 | auto p = bl.cbegin(); |
588 | decode(m, p); | |
7c673cae FG |
589 | cout << " got " << m << std::endl; |
590 | e[0] = 100000; | |
591 | EXPECT_TRUE(m == e || m.empty()); | |
592 | } | |
593 | { | |
594 | ObjectStore::Transaction t; | |
595 | t.remove(cid, oid); | |
596 | t.remove_collection(cid); | |
597 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 598 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
599 | ASSERT_EQ(r, 0); |
600 | } | |
601 | } | |
602 | ||
603 | TEST_P(StoreTest, FiemapHoles) { | |
7c673cae FG |
604 | const uint64_t MAX_EXTENTS = 4000; |
605 | const uint64_t SKIP_STEP = 65536; | |
606 | coll_t cid; | |
607 | int r = 0; | |
608 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
609 | bufferlist bl; | |
610 | bl.append("foo"); | |
11fdf7f2 | 611 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
612 | { |
613 | ObjectStore::Transaction t; | |
614 | t.create_collection(cid, 0); | |
615 | t.touch(cid, oid); | |
616 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) | |
617 | t.write(cid, oid, SKIP_STEP * i, 3, bl); | |
11fdf7f2 | 618 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
619 | ASSERT_EQ(r, 0); |
620 | } | |
621 | { | |
622 | //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3 | |
623 | bufferlist bl; | |
11fdf7f2 | 624 | store->fiemap(ch, oid, 0, SKIP_STEP * (MAX_EXTENTS - 1) + 3, bl); |
7c673cae | 625 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
626 | auto p = bl.cbegin(); |
627 | decode(m, p); | |
7c673cae FG |
628 | cout << " got " << m << std::endl; |
629 | ASSERT_TRUE(!m.empty()); | |
630 | ASSERT_GE(m[0], 3u); | |
11fdf7f2 TL |
631 | auto last = m.crbegin(); |
632 | if (m.size() == 1) { | |
633 | ASSERT_EQ(0u, last->first); | |
634 | } else if (m.size() == MAX_EXTENTS) { | |
635 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) { | |
636 | ASSERT_TRUE(m.count(SKIP_STEP * i)); | |
637 | } | |
7c673cae | 638 | } |
11fdf7f2 TL |
639 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); |
640 | } | |
641 | { | |
7c673cae | 642 | // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3 |
11fdf7f2 TL |
643 | bufferlist bl; |
644 | store->fiemap(ch, oid, SKIP_STEP, SKIP_STEP * (MAX_EXTENTS - 2) + 3, bl); | |
645 | map<uint64_t,uint64_t> m, e; | |
646 | auto p = bl.cbegin(); | |
647 | decode(m, p); | |
7c673cae FG |
648 | cout << " got " << m << std::endl; |
649 | ASSERT_TRUE(!m.empty()); | |
11fdf7f2 TL |
650 | // kstore always returns [0, object_size] regardless of offset and length |
651 | // FIXME: if fiemap logic in kstore is refined | |
652 | if (string(GetParam()) != "kstore") { | |
653 | ASSERT_GE(m[SKIP_STEP], 3u); | |
654 | auto last = m.crbegin(); | |
655 | if (m.size() == 1) { | |
656 | ASSERT_EQ(SKIP_STEP, last->first); | |
657 | } else if (m.size() == MAX_EXTENTS - 2) { | |
658 | for (uint64_t i = 1; i < MAX_EXTENTS - 1; i++) { | |
659 | ASSERT_TRUE(m.count(SKIP_STEP*i)); | |
660 | } | |
661 | } | |
662 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); | |
7c673cae | 663 | } |
7c673cae FG |
664 | } |
665 | { | |
666 | ObjectStore::Transaction t; | |
667 | t.remove(cid, oid); | |
668 | t.remove_collection(cid); | |
669 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 670 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
671 | ASSERT_EQ(r, 0); |
672 | } | |
673 | } | |
674 | ||
675 | TEST_P(StoreTest, SimpleMetaColTest) { | |
7c673cae FG |
676 | coll_t cid; |
677 | int r = 0; | |
678 | { | |
11fdf7f2 | 679 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
680 | ObjectStore::Transaction t; |
681 | t.create_collection(cid, 0); | |
682 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 683 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
684 | ASSERT_EQ(r, 0); |
685 | } | |
686 | { | |
687 | ObjectStore::Transaction t; | |
688 | t.remove_collection(cid); | |
689 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
690 | auto ch = store->open_collection(cid); |
691 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
692 | ASSERT_EQ(r, 0); |
693 | } | |
694 | { | |
11fdf7f2 | 695 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
696 | ObjectStore::Transaction t; |
697 | t.create_collection(cid, 0); | |
698 | cerr << "add collection" << std::endl; | |
11fdf7f2 | 699 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
700 | ASSERT_EQ(r, 0); |
701 | } | |
702 | { | |
703 | ObjectStore::Transaction t; | |
704 | t.remove_collection(cid); | |
705 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
706 | auto ch = store->open_collection(cid); |
707 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
708 | ASSERT_EQ(r, 0); |
709 | } | |
710 | } | |
711 | ||
712 | TEST_P(StoreTest, SimplePGColTest) { | |
7c673cae FG |
713 | coll_t cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD)); |
714 | int r = 0; | |
715 | { | |
716 | ObjectStore::Transaction t; | |
11fdf7f2 | 717 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
718 | t.create_collection(cid, 4); |
719 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 720 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
721 | ASSERT_EQ(r, 0); |
722 | } | |
723 | { | |
724 | ObjectStore::Transaction t; | |
725 | t.remove_collection(cid); | |
726 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
727 | auto ch = store->open_collection(cid); |
728 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
729 | ASSERT_EQ(r, 0); |
730 | } | |
731 | { | |
732 | ObjectStore::Transaction t; | |
733 | t.create_collection(cid, 4); | |
734 | cerr << "add collection" << std::endl; | |
11fdf7f2 TL |
735 | auto ch = store->create_new_collection(cid); |
736 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
737 | ASSERT_EQ(r, 0); |
738 | } | |
739 | { | |
740 | ObjectStore::Transaction t; | |
741 | t.remove_collection(cid); | |
742 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
743 | auto ch = store->open_collection(cid); |
744 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
745 | ASSERT_EQ(r, 0); |
746 | } | |
747 | } | |
748 | ||
749 | TEST_P(StoreTest, SimpleColPreHashTest) { | |
7c673cae FG |
750 | // Firstly we will need to revert the value making sure |
751 | // collection hint actually works | |
752 | int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold; | |
753 | std::ostringstream oss; | |
754 | if (merge_threshold > 0) { | |
755 | oss << "-" << merge_threshold; | |
11fdf7f2 | 756 | SetVal(g_conf(), "filestore_merge_threshold", oss.str().c_str()); |
7c673cae FG |
757 | } |
758 | ||
759 | uint32_t pg_num = 128; | |
760 | ||
761 | boost::uniform_int<> pg_id_range(0, pg_num); | |
762 | gen_type rng(time(NULL)); | |
763 | int pg_id = pg_id_range(rng); | |
764 | ||
765 | int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple; | |
766 | boost::uniform_int<> folders_range(5, 256); | |
767 | uint64_t expected_num_objs = (uint64_t)objs_per_folder * (uint64_t)folders_range(rng); | |
768 | ||
769 | coll_t cid(spg_t(pg_t(pg_id, 15), shard_id_t::NO_SHARD)); | |
770 | int r; | |
11fdf7f2 | 771 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
772 | { |
773 | // Create a collection along with a hint | |
774 | ObjectStore::Transaction t; | |
775 | t.create_collection(cid, 5); | |
776 | cerr << "create collection" << std::endl; | |
777 | bufferlist hint; | |
11fdf7f2 TL |
778 | encode(pg_num, hint); |
779 | encode(expected_num_objs, hint); | |
7c673cae FG |
780 | t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); |
781 | cerr << "collection hint" << std::endl; | |
11fdf7f2 | 782 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
783 | ASSERT_EQ(r, 0); |
784 | } | |
785 | { | |
786 | // Remove the collection | |
787 | ObjectStore::Transaction t; | |
788 | t.remove_collection(cid); | |
789 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 790 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
791 | ASSERT_EQ(r, 0); |
792 | } | |
7c673cae FG |
793 | } |
794 | ||
795 | TEST_P(StoreTest, SmallBlockWrites) { | |
7c673cae FG |
796 | int r; |
797 | coll_t cid; | |
11fdf7f2 | 798 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
799 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
800 | { | |
801 | ObjectStore::Transaction t; | |
802 | t.create_collection(cid, 0); | |
803 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 804 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
805 | ASSERT_EQ(r, 0); |
806 | } | |
807 | bufferlist a; | |
808 | bufferptr ap(0x1000); | |
809 | memset(ap.c_str(), 'a', 0x1000); | |
810 | a.append(ap); | |
811 | bufferlist b; | |
812 | bufferptr bp(0x1000); | |
813 | memset(bp.c_str(), 'b', 0x1000); | |
814 | b.append(bp); | |
815 | bufferlist c; | |
816 | bufferptr cp(0x1000); | |
817 | memset(cp.c_str(), 'c', 0x1000); | |
818 | c.append(cp); | |
819 | bufferptr zp(0x1000); | |
820 | zp.zero(); | |
821 | bufferlist z; | |
822 | z.append(zp); | |
823 | { | |
824 | ObjectStore::Transaction t; | |
825 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 826 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
827 | ASSERT_EQ(r, 0); |
828 | ||
829 | bufferlist in, exp; | |
11fdf7f2 | 830 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
831 | ASSERT_EQ(0x1000, r); |
832 | exp.append(a); | |
833 | ASSERT_TRUE(bl_eq(exp, in)); | |
834 | } | |
835 | { | |
836 | ObjectStore::Transaction t; | |
837 | t.write(cid, hoid, 0x1000, 0x1000, b); | |
11fdf7f2 | 838 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
839 | ASSERT_EQ(r, 0); |
840 | ||
841 | bufferlist in, exp; | |
11fdf7f2 | 842 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
843 | ASSERT_EQ(0x2000, r); |
844 | exp.append(a); | |
845 | exp.append(b); | |
846 | ASSERT_TRUE(bl_eq(exp, in)); | |
847 | } | |
848 | { | |
849 | ObjectStore::Transaction t; | |
850 | t.write(cid, hoid, 0x3000, 0x1000, c); | |
11fdf7f2 | 851 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
852 | ASSERT_EQ(r, 0); |
853 | ||
854 | bufferlist in, exp; | |
11fdf7f2 | 855 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
856 | ASSERT_EQ(0x4000, r); |
857 | exp.append(a); | |
858 | exp.append(b); | |
859 | exp.append(z); | |
860 | exp.append(c); | |
861 | ASSERT_TRUE(bl_eq(exp, in)); | |
862 | } | |
863 | { | |
864 | ObjectStore::Transaction t; | |
865 | t.write(cid, hoid, 0x2000, 0x1000, a); | |
11fdf7f2 | 866 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
867 | ASSERT_EQ(r, 0); |
868 | ||
869 | bufferlist in, exp; | |
11fdf7f2 | 870 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
871 | ASSERT_EQ(0x4000, r); |
872 | exp.append(a); | |
873 | exp.append(b); | |
874 | exp.append(a); | |
875 | exp.append(c); | |
876 | ASSERT_TRUE(bl_eq(exp, in)); | |
877 | } | |
878 | { | |
879 | ObjectStore::Transaction t; | |
880 | t.write(cid, hoid, 0, 0x1000, c); | |
11fdf7f2 | 881 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
882 | ASSERT_EQ(r, 0); |
883 | } | |
884 | { | |
885 | bufferlist in, exp; | |
11fdf7f2 | 886 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
887 | ASSERT_EQ(0x4000, r); |
888 | exp.append(c); | |
889 | exp.append(b); | |
890 | exp.append(a); | |
891 | exp.append(c); | |
892 | ASSERT_TRUE(bl_eq(exp, in)); | |
893 | } | |
894 | { | |
895 | ObjectStore::Transaction t; | |
896 | t.remove(cid, hoid); | |
897 | t.remove_collection(cid); | |
898 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 899 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
900 | ASSERT_EQ(r, 0); |
901 | } | |
902 | } | |
903 | ||
904 | TEST_P(StoreTest, BufferCacheReadTest) { | |
7c673cae FG |
905 | int r; |
906 | coll_t cid; | |
907 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
908 | { | |
11fdf7f2 TL |
909 | auto ch = store->open_collection(cid); |
910 | ASSERT_FALSE(ch); | |
7c673cae | 911 | } |
11fdf7f2 | 912 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
913 | { |
914 | ObjectStore::Transaction t; | |
915 | t.create_collection(cid, 0); | |
916 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 917 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
918 | ASSERT_EQ(r, 0); |
919 | } | |
920 | { | |
11fdf7f2 | 921 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
922 | ASSERT_TRUE(!exists); |
923 | ||
924 | ObjectStore::Transaction t; | |
925 | t.touch(cid, hoid); | |
926 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 927 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
928 | ASSERT_EQ(r, 0); |
929 | ||
11fdf7f2 | 930 | exists = store->exists(ch, hoid); |
7c673cae FG |
931 | ASSERT_EQ(true, exists); |
932 | } | |
933 | { | |
934 | ObjectStore::Transaction t; | |
935 | bufferlist bl, newdata; | |
936 | bl.append("abcde"); | |
937 | t.write(cid, hoid, 0, 5, bl); | |
938 | t.write(cid, hoid, 10, 5, bl); | |
939 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 940 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
941 | ASSERT_EQ(r, 0); |
942 | ||
11fdf7f2 | 943 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
944 | ASSERT_EQ(r, 15); |
945 | { | |
946 | bufferlist expected; | |
947 | expected.append(bl); | |
948 | expected.append_zero(5); | |
949 | expected.append(bl); | |
950 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
951 | } | |
952 | } | |
953 | //overwrite over the same extents | |
954 | { | |
955 | ObjectStore::Transaction t; | |
956 | bufferlist bl, newdata; | |
957 | bl.append("edcba"); | |
958 | t.write(cid, hoid, 0, 5, bl); | |
959 | t.write(cid, hoid, 10, 5, bl); | |
960 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 961 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
962 | ASSERT_EQ(r, 0); |
963 | ||
11fdf7f2 | 964 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
965 | ASSERT_EQ(r, 15); |
966 | { | |
967 | bufferlist expected; | |
968 | expected.append(bl); | |
969 | expected.append_zero(5); | |
970 | expected.append(bl); | |
971 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
972 | } | |
973 | } | |
974 | //additional write to an unused region of some blob | |
975 | { | |
976 | ObjectStore::Transaction t; | |
977 | bufferlist bl2, newdata; | |
978 | bl2.append("1234567890"); | |
979 | ||
980 | t.write(cid, hoid, 20, bl2.length(), bl2); | |
981 | cerr << "Append" << std::endl; | |
11fdf7f2 | 982 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
983 | ASSERT_EQ(r, 0); |
984 | ||
11fdf7f2 | 985 | r = store->read(ch, hoid, 0, 30, newdata); |
7c673cae FG |
986 | ASSERT_EQ(r, 30); |
987 | { | |
988 | bufferlist expected; | |
989 | expected.append("edcba"); | |
990 | expected.append_zero(5); | |
991 | expected.append("edcba"); | |
992 | expected.append_zero(5); | |
993 | expected.append(bl2); | |
994 | ||
995 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
996 | } | |
997 | } | |
998 | //additional write to an unused region of some blob and partial owerite over existing extents | |
999 | { | |
1000 | ObjectStore::Transaction t; | |
1001 | bufferlist bl, bl2, bl3, newdata; | |
1002 | bl.append("DCB"); | |
1003 | bl2.append("1234567890"); | |
1004 | bl3.append("BA"); | |
1005 | ||
1006 | t.write(cid, hoid, 30, bl2.length(), bl2); | |
1007 | t.write(cid, hoid, 1, bl.length(), bl); | |
1008 | t.write(cid, hoid, 13, bl3.length(), bl3); | |
1009 | cerr << "TripleWrite" << std::endl; | |
11fdf7f2 | 1010 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1011 | ASSERT_EQ(r, 0); |
1012 | ||
11fdf7f2 | 1013 | r = store->read(ch, hoid, 0, 40, newdata); |
7c673cae FG |
1014 | ASSERT_EQ(r, 40); |
1015 | { | |
1016 | bufferlist expected; | |
1017 | expected.append("eDCBa"); | |
1018 | expected.append_zero(5); | |
1019 | expected.append("edcBA"); | |
1020 | expected.append_zero(5); | |
1021 | expected.append(bl2); | |
1022 | expected.append(bl2); | |
1023 | ||
1024 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1025 | } | |
1026 | } | |
1027 | } | |
1028 | ||
11fdf7f2 | 1029 | void StoreTest::doCompressionTest() |
7c673cae | 1030 | { |
7c673cae FG |
1031 | int r; |
1032 | coll_t cid; | |
1033 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1034 | { | |
11fdf7f2 TL |
1035 | auto ch = store->open_collection(cid); |
1036 | ASSERT_FALSE(ch); | |
7c673cae | 1037 | } |
11fdf7f2 | 1038 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1039 | { |
1040 | ObjectStore::Transaction t; | |
1041 | t.create_collection(cid, 0); | |
1042 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1043 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1044 | ASSERT_EQ(r, 0); |
1045 | } | |
1046 | { | |
11fdf7f2 | 1047 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1048 | ASSERT_TRUE(!exists); |
1049 | ||
1050 | ObjectStore::Transaction t; | |
1051 | t.touch(cid, hoid); | |
1052 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1053 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1054 | ASSERT_EQ(r, 0); |
1055 | ||
11fdf7f2 | 1056 | exists = store->exists(ch, hoid); |
7c673cae FG |
1057 | ASSERT_EQ(true, exists); |
1058 | } | |
1059 | std::string data; | |
1060 | data.resize(0x10000 * 4); | |
1061 | for(size_t i = 0;i < data.size(); i++) | |
1062 | data[i] = i / 256; | |
1063 | { | |
1064 | ObjectStore::Transaction t; | |
1065 | bufferlist bl, newdata; | |
1066 | bl.append(data); | |
1067 | t.write(cid, hoid, 0, bl.length(), bl); | |
1068 | cerr << "CompressibleData (4xAU) Write" << std::endl; | |
11fdf7f2 | 1069 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1070 | ASSERT_EQ(r, 0); |
1071 | ||
11fdf7f2 | 1072 | r = store->read(ch, hoid, 0, data.size() , newdata); |
7c673cae FG |
1073 | |
1074 | ASSERT_EQ(r, (int)data.size()); | |
1075 | { | |
1076 | bufferlist expected; | |
1077 | expected.append(data); | |
1078 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1079 | } | |
1080 | newdata.clear(); | |
11fdf7f2 | 1081 | r = store->read(ch, hoid, 0, 711 , newdata); |
7c673cae FG |
1082 | ASSERT_EQ(r, 711); |
1083 | { | |
1084 | bufferlist expected; | |
1085 | expected.append(data.substr(0,711)); | |
1086 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1087 | } | |
1088 | newdata.clear(); | |
11fdf7f2 | 1089 | r = store->read(ch, hoid, 0xf00f, data.size(), newdata); |
7c673cae FG |
1090 | ASSERT_EQ(r, int(data.size() - 0xf00f) ); |
1091 | { | |
1092 | bufferlist expected; | |
1093 | expected.append(data.substr(0xf00f)); | |
1094 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1095 | } | |
1096 | { | |
1097 | struct store_statfs_t statfs; | |
1098 | int r = store->statfs(&statfs); | |
1099 | ASSERT_EQ(r, 0); | |
11fdf7f2 TL |
1100 | ASSERT_EQ(statfs.data_stored, (unsigned)data.size()); |
1101 | ASSERT_LE(statfs.data_compressed, (unsigned)data.size()); | |
1102 | ASSERT_EQ(statfs.data_compressed_original, (unsigned)data.size()); | |
1103 | ASSERT_LE(statfs.data_compressed_allocated, (unsigned)data.size()); | |
7c673cae FG |
1104 | } |
1105 | } | |
1106 | std::string data2; | |
1107 | data2.resize(0x10000 * 4 - 0x9000); | |
1108 | for(size_t i = 0;i < data2.size(); i++) | |
1109 | data2[i] = (i+1) / 256; | |
1110 | { | |
1111 | ObjectStore::Transaction t; | |
1112 | bufferlist bl, newdata; | |
1113 | bl.append(data2); | |
1114 | t.write(cid, hoid, 0x8000, bl.length(), bl); | |
1115 | cerr << "CompressibleData partial overwrite" << std::endl; | |
11fdf7f2 | 1116 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1117 | ASSERT_EQ(r, 0); |
1118 | ||
11fdf7f2 | 1119 | r = store->read(ch, hoid, 0, 0x10000, newdata); |
7c673cae FG |
1120 | ASSERT_EQ(r, (int)0x10000); |
1121 | { | |
1122 | bufferlist expected; | |
1123 | expected.append(data.substr(0, 0x8000)); | |
1124 | expected.append(data2.substr(0, 0x8000)); | |
1125 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1126 | } | |
1127 | newdata.clear(); | |
11fdf7f2 | 1128 | r = store->read(ch, hoid, 0x9000, 711 , newdata); |
7c673cae FG |
1129 | ASSERT_EQ(r, 711); |
1130 | { | |
1131 | bufferlist expected; | |
1132 | expected.append(data2.substr(0x1000,711)); | |
1133 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1134 | } | |
1135 | newdata.clear(); | |
11fdf7f2 | 1136 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
1137 | ASSERT_EQ(r, int(0x40000) ); |
1138 | { | |
1139 | bufferlist expected; | |
1140 | expected.append(data.substr(0, 0x8000)); | |
1141 | expected.append(data2.substr(0, 0x37000)); | |
1142 | expected.append(data.substr(0x3f000, 0x1000)); | |
1143 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1144 | } | |
1145 | } | |
1146 | data2.resize(0x3f000); | |
1147 | for(size_t i = 0;i < data2.size(); i++) | |
1148 | data2[i] = (i+2) / 256; | |
1149 | { | |
1150 | ObjectStore::Transaction t; | |
1151 | bufferlist bl, newdata; | |
1152 | bl.append(data2); | |
1153 | t.write(cid, hoid, 0, bl.length(), bl); | |
1154 | cerr << "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 1155 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1156 | ASSERT_EQ(r, 0); |
1157 | ||
11fdf7f2 | 1158 | r = store->read(ch, hoid, 0, 0x3e000 - 1, newdata); |
7c673cae FG |
1159 | ASSERT_EQ(r, (int)0x3e000 - 1); |
1160 | { | |
1161 | bufferlist expected; | |
1162 | expected.append(data2.substr(0, 0x3e000 - 1)); | |
1163 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1164 | } | |
1165 | newdata.clear(); | |
11fdf7f2 | 1166 | r = store->read(ch, hoid, 0x3e000-1, 0x2001, newdata); |
7c673cae FG |
1167 | ASSERT_EQ(r, 0x2001); |
1168 | { | |
1169 | bufferlist expected; | |
1170 | expected.append(data2.substr(0x3e000-1, 0x1001)); | |
1171 | expected.append(data.substr(0x3f000, 0x1000)); | |
1172 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1173 | } | |
1174 | newdata.clear(); | |
11fdf7f2 | 1175 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
1176 | ASSERT_EQ(r, int(0x40000) ); |
1177 | { | |
1178 | bufferlist expected; | |
1179 | expected.append(data2.substr(0, 0x3f000)); | |
1180 | expected.append(data.substr(0x3f000, 0x1000)); | |
1181 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1182 | } | |
1183 | } | |
1184 | data.resize(0x1001); | |
1185 | for(size_t i = 0;i < data.size(); i++) | |
1186 | data[i] = (i+3) / 256; | |
1187 | { | |
1188 | ObjectStore::Transaction t; | |
1189 | bufferlist bl, newdata; | |
1190 | bl.append(data); | |
1191 | t.write(cid, hoid, 0x3f000-1, bl.length(), bl); | |
1192 | cerr << "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 1193 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1194 | ASSERT_EQ(r, 0); |
1195 | ||
11fdf7f2 | 1196 | r = store->read(ch, hoid, 0x3e000, 0x2000, newdata); |
7c673cae FG |
1197 | ASSERT_EQ(r, (int)0x2000); |
1198 | { | |
1199 | bufferlist expected; | |
1200 | expected.append(data2.substr(0x3e000, 0x1000 - 1)); | |
1201 | expected.append(data.substr(0, 0x1001)); | |
1202 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1203 | } | |
1204 | } | |
1205 | { | |
1206 | ObjectStore::Transaction t; | |
1207 | t.remove(cid, hoid); | |
1208 | cerr << "Cleaning object" << std::endl; | |
11fdf7f2 | 1209 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1210 | ASSERT_EQ(r, 0); |
1211 | } | |
1212 | //force fsck | |
11fdf7f2 | 1213 | ch.reset(); |
7c673cae | 1214 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1215 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1216 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 TL |
1217 | ch = store->open_collection(cid); |
1218 | auto settingsBookmark = BookmarkSettings(); | |
1219 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
1220 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 1221 | { |
7c673cae FG |
1222 | data.resize(0x10000*6); |
1223 | ||
1224 | for(size_t i = 0;i < data.size(); i++) | |
1225 | data[i] = i / 256; | |
1226 | ObjectStore::Transaction t; | |
1227 | bufferlist bl, newdata; | |
1228 | bl.append(data); | |
1229 | t.write(cid, hoid, 0, bl.length(), bl); | |
1230 | cerr << "CompressibleData large blob" << std::endl; | |
11fdf7f2 | 1231 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1232 | ASSERT_EQ(r, 0); |
1233 | } | |
1234 | //force fsck | |
11fdf7f2 | 1235 | ch.reset(); |
7c673cae | 1236 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1237 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1238 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1239 | ch = store->open_collection(cid); |
7c673cae FG |
1240 | { |
1241 | ObjectStore::Transaction t; | |
1242 | t.remove(cid, hoid); | |
1243 | t.remove_collection(cid); | |
1244 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1245 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1246 | ASSERT_EQ(r, 0); |
1247 | } | |
7c673cae FG |
1248 | } |
1249 | ||
1250 | TEST_P(StoreTest, CompressionTest) { | |
1251 | if (string(GetParam()) != "bluestore") | |
1252 | return; | |
20effc67 TL |
1253 | if (smr) { |
1254 | cout << "TODO: need to adjust statfs check for smr" << std::endl; | |
1255 | return; | |
1256 | } | |
7c673cae | 1257 | |
11fdf7f2 TL |
1258 | SetVal(g_conf(), "bluestore_compression_algorithm", "snappy"); |
1259 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
1260 | g_ceph_context->_conf.apply_changes(nullptr); | |
1261 | doCompressionTest(); | |
7c673cae | 1262 | |
11fdf7f2 TL |
1263 | SetVal(g_conf(), "bluestore_compression_algorithm", "zlib"); |
1264 | SetVal(g_conf(), "bluestore_compression_mode", "aggressive"); | |
1265 | g_ceph_context->_conf.apply_changes(nullptr); | |
1266 | doCompressionTest(); | |
7c673cae FG |
1267 | } |
1268 | ||
1269 | TEST_P(StoreTest, SimpleObjectTest) { | |
7c673cae FG |
1270 | int r; |
1271 | coll_t cid; | |
1272 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1273 | { | |
11fdf7f2 TL |
1274 | auto ch = store->open_collection(cid); |
1275 | ASSERT_FALSE(ch); | |
7c673cae | 1276 | } |
11fdf7f2 | 1277 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1278 | { |
1279 | ObjectStore::Transaction t; | |
1280 | t.create_collection(cid, 0); | |
1281 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1282 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1283 | ASSERT_EQ(r, 0); |
1284 | } | |
1285 | { | |
11fdf7f2 | 1286 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1287 | ASSERT_TRUE(!exists); |
1288 | ||
1289 | ObjectStore::Transaction t; | |
1290 | t.touch(cid, hoid); | |
1291 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1292 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1293 | ASSERT_EQ(r, 0); |
1294 | ||
11fdf7f2 | 1295 | exists = store->exists(ch, hoid); |
7c673cae FG |
1296 | ASSERT_EQ(true, exists); |
1297 | } | |
1298 | { | |
1299 | ObjectStore::Transaction t; | |
1300 | t.remove(cid, hoid); | |
1301 | t.touch(cid, hoid); | |
1302 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1303 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1304 | ASSERT_EQ(r, 0); |
1305 | } | |
1306 | { | |
1307 | ObjectStore::Transaction t; | |
1308 | bufferlist bl, orig; | |
1309 | bl.append("abcde"); | |
1310 | orig = bl; | |
1311 | t.remove(cid, hoid); | |
1312 | t.write(cid, hoid, 0, 5, bl); | |
1313 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1314 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1315 | ASSERT_EQ(r, 0); |
1316 | ||
1317 | bufferlist in; | |
11fdf7f2 | 1318 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
1319 | ASSERT_EQ(5, r); |
1320 | ASSERT_TRUE(bl_eq(orig, in)); | |
1321 | } | |
1322 | { | |
1323 | ObjectStore::Transaction t; | |
1324 | bufferlist bl, exp; | |
1325 | bl.append("abcde"); | |
1326 | exp = bl; | |
1327 | exp.append(bl); | |
1328 | t.write(cid, hoid, 5, 5, bl); | |
1329 | cerr << "Append" << std::endl; | |
11fdf7f2 | 1330 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1331 | ASSERT_EQ(r, 0); |
1332 | ||
1333 | bufferlist in; | |
11fdf7f2 | 1334 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1335 | ASSERT_EQ(10, r); |
1336 | ASSERT_TRUE(bl_eq(exp, in)); | |
1337 | } | |
1338 | { | |
1339 | ObjectStore::Transaction t; | |
1340 | bufferlist bl, exp; | |
1341 | bl.append("abcdeabcde"); | |
1342 | exp = bl; | |
1343 | t.write(cid, hoid, 0, 10, bl); | |
1344 | cerr << "Full overwrite" << std::endl; | |
11fdf7f2 | 1345 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1346 | ASSERT_EQ(r, 0); |
1347 | ||
1348 | bufferlist in; | |
11fdf7f2 | 1349 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1350 | ASSERT_EQ(10, r); |
1351 | ASSERT_TRUE(bl_eq(exp, in)); | |
1352 | } | |
1353 | { | |
1354 | ObjectStore::Transaction t; | |
1355 | bufferlist bl; | |
1356 | bl.append("abcde"); | |
1357 | t.write(cid, hoid, 3, 5, bl); | |
1358 | cerr << "Partial overwrite" << std::endl; | |
11fdf7f2 | 1359 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1360 | ASSERT_EQ(r, 0); |
1361 | ||
1362 | bufferlist in, exp; | |
1363 | exp.append("abcabcdede"); | |
11fdf7f2 | 1364 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1365 | ASSERT_EQ(10, r); |
1366 | in.hexdump(cout); | |
1367 | ASSERT_TRUE(bl_eq(exp, in)); | |
1368 | } | |
1369 | { | |
1370 | { | |
1371 | ObjectStore::Transaction t; | |
1372 | bufferlist bl; | |
1373 | bl.append("fghij"); | |
1374 | t.truncate(cid, hoid, 0); | |
1375 | t.write(cid, hoid, 5, 5, bl); | |
1376 | cerr << "Truncate + hole" << std::endl; | |
11fdf7f2 | 1377 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1378 | ASSERT_EQ(r, 0); |
1379 | } | |
1380 | { | |
1381 | ObjectStore::Transaction t; | |
1382 | bufferlist bl; | |
1383 | bl.append("abcde"); | |
1384 | t.write(cid, hoid, 0, 5, bl); | |
1385 | cerr << "Reverse fill-in" << std::endl; | |
11fdf7f2 | 1386 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1387 | ASSERT_EQ(r, 0); |
1388 | } | |
1389 | ||
1390 | bufferlist in, exp; | |
1391 | exp.append("abcdefghij"); | |
11fdf7f2 | 1392 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1393 | ASSERT_EQ(10, r); |
1394 | in.hexdump(cout); | |
1395 | ASSERT_TRUE(bl_eq(exp, in)); | |
1396 | } | |
1397 | { | |
1398 | ObjectStore::Transaction t; | |
1399 | bufferlist bl; | |
1400 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1401 | t.write(cid, hoid, 0, bl.length(), bl); | |
1402 | cerr << "larger overwrite" << std::endl; | |
11fdf7f2 | 1403 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1404 | ASSERT_EQ(r, 0); |
1405 | ||
1406 | bufferlist in; | |
11fdf7f2 | 1407 | r = store->read(ch, hoid, 0, bl.length(), in); |
7c673cae FG |
1408 | ASSERT_EQ((int)bl.length(), r); |
1409 | in.hexdump(cout); | |
1410 | ASSERT_TRUE(bl_eq(bl, in)); | |
1411 | } | |
1412 | { | |
1413 | bufferlist bl; | |
1414 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1415 | ||
1416 | //test: offset=len=0 mean read all data | |
1417 | bufferlist in; | |
11fdf7f2 | 1418 | r = store->read(ch, hoid, 0, 0, in); |
7c673cae FG |
1419 | ASSERT_EQ((int)bl.length(), r); |
1420 | in.hexdump(cout); | |
1421 | ASSERT_TRUE(bl_eq(bl, in)); | |
1422 | } | |
1423 | { | |
1424 | //verifying unaligned csums | |
1425 | std::string s1("1"), s2(0x1000, '2'), s3("00"); | |
1426 | { | |
1427 | ObjectStore::Transaction t; | |
1428 | bufferlist bl; | |
1429 | bl.append(s1); | |
1430 | bl.append(s2); | |
1431 | t.truncate(cid, hoid, 0); | |
1432 | t.write(cid, hoid, 0x1000-1, bl.length(), bl); | |
1433 | cerr << "Write unaligned csum, stage 1" << std::endl; | |
11fdf7f2 | 1434 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1435 | ASSERT_EQ(r, 0); |
1436 | } | |
1437 | ||
1438 | bufferlist in, exp1, exp2, exp3; | |
1439 | exp1.append(s1); | |
1440 | exp2.append(s2); | |
1441 | exp3.append(s3); | |
11fdf7f2 | 1442 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1443 | ASSERT_EQ(1, r); |
1444 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1445 | in.clear(); | |
11fdf7f2 | 1446 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1447 | ASSERT_EQ(0x1000, r); |
1448 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1449 | ||
1450 | { | |
1451 | ObjectStore::Transaction t; | |
1452 | bufferlist bl; | |
1453 | bl.append(s3); | |
1454 | t.write(cid, hoid, 1, bl.length(), bl); | |
1455 | cerr << "Write unaligned csum, stage 2" << std::endl; | |
11fdf7f2 | 1456 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1457 | ASSERT_EQ(r, 0); |
1458 | } | |
1459 | in.clear(); | |
11fdf7f2 | 1460 | r = store->read(ch, hoid, 1, 2, in); |
7c673cae FG |
1461 | ASSERT_EQ(2, r); |
1462 | ASSERT_TRUE(bl_eq(exp3, in)); | |
1463 | in.clear(); | |
11fdf7f2 | 1464 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1465 | ASSERT_EQ(1, r); |
1466 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1467 | in.clear(); | |
11fdf7f2 | 1468 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1469 | ASSERT_EQ(0x1000, r); |
1470 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1471 | ||
1472 | } | |
1473 | ||
1474 | { | |
1475 | ObjectStore::Transaction t; | |
1476 | t.remove(cid, hoid); | |
1477 | t.remove_collection(cid); | |
1478 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1479 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1480 | ASSERT_EQ(r, 0); |
1481 | } | |
1482 | } | |
1483 | ||
11fdf7f2 TL |
1484 | #if defined(WITH_BLUESTORE) |
1485 | ||
1911f103 TL |
1486 | TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { |
1487 | if(string(GetParam()) != "bluestore") | |
1488 | return; | |
20effc67 TL |
1489 | if (smr) { |
1490 | cout << "SKIP (smr)" << std::endl; | |
1491 | return; | |
1492 | } | |
1493 | ||
f67539c2 | 1494 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); |
1911f103 TL |
1495 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); |
1496 | g_conf().apply_changes(nullptr); | |
1497 | StartDeferred(65536); | |
1498 | ||
1499 | int r; | |
1500 | coll_t cid; | |
1501 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1502 | const PerfCounters* logger = store->get_perf_counters(); | |
1503 | auto ch = store->create_new_collection(cid); | |
1504 | { | |
1505 | ObjectStore::Transaction t; | |
1506 | t.create_collection(cid, 0); | |
1507 | cerr << "Creating collection " << cid << std::endl; | |
1508 | r = queue_transaction(store, ch, std::move(t)); | |
1509 | ASSERT_EQ(r, 0); | |
1510 | } | |
1511 | { | |
1512 | bool exists = store->exists(ch, hoid); | |
1513 | ASSERT_TRUE(!exists); | |
1514 | ||
1515 | ObjectStore::Transaction t; | |
1516 | t.touch(cid, hoid); | |
1517 | cerr << "Creating object " << hoid << std::endl; | |
1518 | r = queue_transaction(store, ch, std::move(t)); | |
1519 | ASSERT_EQ(r, 0); | |
1520 | ||
1521 | exists = store->exists(ch, hoid); | |
1522 | ASSERT_EQ(true, exists); | |
1523 | } | |
1524 | { | |
1525 | ObjectStore::Transaction t; | |
1526 | bufferlist bl, orig; | |
1527 | string s(4096, 'a'); | |
1528 | bl.append(s); | |
1529 | t.write(cid, hoid, 0x11000, bl.length(), bl); | |
1530 | cerr << "write1" << std::endl; | |
1531 | r = queue_transaction(store, ch, std::move(t)); | |
1532 | ASSERT_EQ(r, 0); | |
1533 | } | |
1534 | { | |
1535 | ObjectStore::Transaction t; | |
1536 | bufferlist bl, orig; | |
1537 | string s(4096 * 3, 'a'); | |
1538 | bl.append(s); | |
1539 | t.write(cid, hoid, 0x15000, bl.length(), bl); | |
1540 | cerr << "write2" << std::endl; | |
1541 | r = queue_transaction(store, ch, std::move(t)); | |
1542 | ASSERT_EQ(r, 0); | |
1543 | } | |
1544 | ASSERT_EQ(logger->get(l_bluestore_write_small), 2u); | |
1545 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 1u); | |
1546 | ||
1547 | { | |
1548 | ObjectStore::Transaction t; | |
1549 | bufferlist bl, orig; | |
1550 | string s(4096 * 2, 'a'); | |
1551 | bl.append(s); | |
1552 | t.write(cid, hoid, 0xe000, bl.length(), bl); | |
1553 | cerr << "write3" << std::endl; | |
1554 | r = queue_transaction(store, ch, std::move(t)); | |
1555 | ASSERT_EQ(r, 0); | |
1556 | } | |
1557 | ASSERT_EQ(logger->get(l_bluestore_write_small), 3u); | |
1558 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1559 | ||
1560 | ||
1561 | { | |
1562 | ObjectStore::Transaction t; | |
1563 | bufferlist bl, orig; | |
1564 | string s(4096, 'a'); | |
1565 | bl.append(s); | |
1566 | t.write(cid, hoid, 0xf000, bl.length(), bl); | |
1567 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1568 | cerr << "write3" << std::endl; | |
1569 | r = queue_transaction(store, ch, std::move(t)); | |
1570 | ASSERT_EQ(r, 0); | |
1571 | } | |
1572 | ASSERT_EQ(logger->get(l_bluestore_write_small), 5u); | |
1573 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1574 | { | |
1575 | ObjectStore::Transaction t; | |
1576 | t.remove(cid, hoid); | |
1577 | t.remove_collection(cid); | |
1578 | cerr << "Cleaning" << std::endl; | |
1579 | r = queue_transaction(store, ch, std::move(t)); | |
1580 | ASSERT_EQ(r, 0); | |
1581 | } | |
1582 | } | |
1583 | ||
1584 | ||
7c673cae FG |
1585 | TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { |
1586 | if(string(GetParam()) != "bluestore") | |
1587 | return; | |
20effc67 TL |
1588 | if (smr) { |
1589 | cout << "TODO: fix this for smr" << std::endl; | |
1590 | return; | |
1591 | } | |
f67539c2 | 1592 | SetVal(g_conf(), "bluestore_block_db_path", ""); |
7c673cae | 1593 | StartDeferred(65536); |
11fdf7f2 | 1594 | SetVal(g_conf(), "bluestore_compression_mode", "force"); |
eafe8130 | 1595 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); |
7c673cae | 1596 | // just a big number to disble gc |
11fdf7f2 TL |
1597 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "100000"); |
1598 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
1599 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
1600 | int r; |
1601 | ||
11fdf7f2 TL |
1602 | int poolid = 4373; |
1603 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
1604 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
1605 | string(), | |
1606 | 0, | |
1607 | poolid, | |
1608 | string())); | |
7c673cae FG |
1609 | ghobject_t hoid2 = hoid; |
1610 | hoid2.hobj.snap = 1; | |
1611 | { | |
11fdf7f2 TL |
1612 | auto ch = store->open_collection(cid); |
1613 | ASSERT_FALSE(ch); | |
7c673cae | 1614 | } |
11fdf7f2 | 1615 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1616 | { |
1617 | ObjectStore::Transaction t; | |
1618 | t.create_collection(cid, 0); | |
1619 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1620 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1621 | ASSERT_EQ(r, 0); |
1622 | } | |
1623 | { | |
11fdf7f2 | 1624 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1625 | ASSERT_TRUE(!exists); |
1626 | ||
1627 | ObjectStore::Transaction t; | |
1628 | t.touch(cid, hoid); | |
1629 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1630 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1631 | ASSERT_EQ(r, 0); |
1632 | ||
11fdf7f2 | 1633 | exists = store->exists(ch, hoid); |
7c673cae FG |
1634 | ASSERT_EQ(true, exists); |
1635 | } | |
1636 | { | |
1637 | struct store_statfs_t statfs; | |
1638 | int r = store->statfs(&statfs); | |
1639 | ASSERT_EQ(r, 0); | |
1640 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
1641 | ASSERT_EQ( 0u, statfs.data_stored); |
1642 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); | |
1643 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
1644 | ||
1645 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1646 | bool per_pool_omap; |
1647 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1648 | ASSERT_EQ(r, 0); |
1649 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
1650 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
1651 | ||
7c673cae | 1652 | //force fsck |
11fdf7f2 | 1653 | ch.reset(); |
7c673cae | 1654 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1655 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1656 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1657 | ch = store->open_collection(cid); |
7c673cae FG |
1658 | } |
1659 | { | |
1660 | ObjectStore::Transaction t; | |
1661 | bufferlist bl; | |
1662 | bl.append("abcde"); | |
1663 | t.write(cid, hoid, 0, 5, bl); | |
1664 | cerr << "Append 5 bytes" << std::endl; | |
11fdf7f2 | 1665 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1666 | ASSERT_EQ(r, 0); |
1667 | ||
1668 | struct store_statfs_t statfs; | |
1669 | int r = store->statfs(&statfs); | |
1670 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1671 | ASSERT_EQ(5, statfs.data_stored); |
7c673cae | 1672 | ASSERT_EQ(0x10000, statfs.allocated); |
11fdf7f2 TL |
1673 | ASSERT_EQ(0, statfs.data_compressed); |
1674 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1675 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1676 | ||
1677 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1678 | bool per_pool_omap; |
1679 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1680 | ASSERT_EQ(r, 0); |
1681 | ASSERT_EQ(5, statfs_pool.data_stored); | |
1682 | ASSERT_EQ(0x10000, statfs_pool.allocated); | |
1683 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1684 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1685 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1686 | ||
1687 | // accessing unknown pool | |
9f95a23c | 1688 | r = store->pool_statfs(poolid + 1, &statfs_pool, &per_pool_omap); |
11fdf7f2 TL |
1689 | ASSERT_EQ(r, 0); |
1690 | ASSERT_EQ(0, statfs_pool.data_stored); | |
1691 | ASSERT_EQ(0, statfs_pool.allocated); | |
1692 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1693 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1694 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1695 | ||
7c673cae | 1696 | //force fsck |
11fdf7f2 | 1697 | ch.reset(); |
7c673cae | 1698 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1699 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1700 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1701 | ch = store->open_collection(cid); |
7c673cae FG |
1702 | } |
1703 | { | |
1704 | ObjectStore::Transaction t; | |
1705 | std::string s(0x30000, 'a'); | |
1706 | bufferlist bl; | |
1707 | bl.append(s); | |
1708 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1709 | cerr << "Append 0x30000 compressible bytes" << std::endl; | |
11fdf7f2 | 1710 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1711 | ASSERT_EQ(r, 0); |
1712 | ||
1713 | struct store_statfs_t statfs; | |
1714 | int r = store->statfs(&statfs); | |
1715 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1716 | ASSERT_EQ(0x30005, statfs.data_stored); |
7c673cae | 1717 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1718 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1719 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1720 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1721 | ||
1722 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1723 | bool per_pool_omap; |
1724 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1725 | ASSERT_EQ(r, 0); |
1726 | ASSERT_EQ(0x30005, statfs_pool.data_stored); | |
1727 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1728 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1729 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1730 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1731 | //force fsck |
11fdf7f2 | 1732 | ch.reset(); |
7c673cae | 1733 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1734 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1735 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1736 | ch = store->open_collection(cid); |
7c673cae FG |
1737 | } |
1738 | { | |
1739 | ObjectStore::Transaction t; | |
1740 | t.zero(cid, hoid, 1, 3); | |
1741 | t.zero(cid, hoid, 0x20000, 9); | |
1742 | cerr << "Punch hole at 1~3, 0x20000~9" << std::endl; | |
11fdf7f2 | 1743 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1744 | ASSERT_EQ(r, 0); |
1745 | ||
1746 | struct store_statfs_t statfs; | |
1747 | int r = store->statfs(&statfs); | |
1748 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1749 | ASSERT_EQ(0x30005 - 3 - 9, statfs.data_stored); |
7c673cae | 1750 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1751 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1752 | ASSERT_EQ(0x20000 - 9, statfs.data_compressed_original); | |
1753 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1754 | ||
1755 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1756 | bool per_pool_omap; |
1757 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1758 | ASSERT_EQ(r, 0); |
1759 | ASSERT_EQ(0x30005 - 3 - 9, statfs_pool.data_stored); | |
1760 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1761 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1762 | ASSERT_EQ(0x20000 - 9, statfs_pool.data_compressed_original); | |
1763 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1764 | //force fsck |
11fdf7f2 | 1765 | ch.reset(); |
7c673cae | 1766 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1767 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1768 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1769 | ch = store->open_collection(cid); |
7c673cae FG |
1770 | } |
1771 | { | |
1772 | ObjectStore::Transaction t; | |
1773 | std::string s(0x1000, 'b'); | |
1774 | bufferlist bl; | |
1775 | bl.append(s); | |
1776 | t.write(cid, hoid, 1, bl.length(), bl); | |
1777 | t.write(cid, hoid, 0x10001, bl.length(), bl); | |
1778 | cerr << "Overwrite first and second(compressible) extents" << std::endl; | |
11fdf7f2 | 1779 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1780 | ASSERT_EQ(r, 0); |
1781 | ||
1782 | struct store_statfs_t statfs; | |
1783 | int r = store->statfs(&statfs); | |
1784 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1785 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs.data_stored); |
7c673cae | 1786 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1787 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1788 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs.data_compressed_original); | |
1789 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1790 | ||
1791 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1792 | bool per_pool_omap; |
1793 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1794 | ASSERT_EQ(r, 0); |
1795 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs_pool.data_stored); | |
1796 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1797 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1798 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs_pool.data_compressed_original); | |
1799 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1800 | //force fsck |
11fdf7f2 | 1801 | ch.reset(); |
7c673cae | 1802 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1803 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1804 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1805 | ch = store->open_collection(cid); |
7c673cae FG |
1806 | } |
1807 | { | |
1808 | ObjectStore::Transaction t; | |
1809 | std::string s(0x10000, 'c'); | |
1810 | bufferlist bl; | |
1811 | bl.append(s); | |
1812 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1813 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
1814 | t.write(cid, hoid, 0x30000, bl.length(), bl); | |
1815 | cerr << "Overwrite compressed extent with 3 uncompressible ones" << std::endl; | |
11fdf7f2 | 1816 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1817 | ASSERT_EQ(r, 0); |
1818 | ||
1819 | struct store_statfs_t statfs; | |
1820 | int r = store->statfs(&statfs); | |
1821 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1822 | ASSERT_EQ(0x30000 + 0x1001, statfs.data_stored); |
7c673cae | 1823 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1824 | ASSERT_LE(statfs.data_compressed, 0); |
1825 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1826 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1827 | ||
1828 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1829 | bool per_pool_omap; |
1830 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1831 | ASSERT_EQ(r, 0); |
1832 | ASSERT_EQ(0x30000 + 0x1001, statfs_pool.data_stored); | |
1833 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1834 | ASSERT_LE(statfs_pool.data_compressed, 0); | |
1835 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1836 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
7c673cae | 1837 | //force fsck |
11fdf7f2 | 1838 | ch.reset(); |
7c673cae | 1839 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1840 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1841 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1842 | ch = store->open_collection(cid); |
7c673cae FG |
1843 | } |
1844 | { | |
1845 | ObjectStore::Transaction t; | |
1846 | t.zero(cid, hoid, 0, 0x40000); | |
1847 | cerr << "Zero object" << std::endl; | |
11fdf7f2 | 1848 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1849 | ASSERT_EQ(r, 0); |
1850 | struct store_statfs_t statfs; | |
1851 | int r = store->statfs(&statfs); | |
1852 | ASSERT_EQ(r, 0); | |
1853 | ASSERT_EQ(0u, statfs.allocated); | |
11fdf7f2 TL |
1854 | ASSERT_EQ(0u, statfs.data_stored); |
1855 | ASSERT_EQ(0u, statfs.data_compressed_original); | |
1856 | ASSERT_EQ(0u, statfs.data_compressed); | |
1857 | ASSERT_EQ(0u, statfs.data_compressed_allocated); | |
1858 | ||
1859 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1860 | bool per_pool_omap; |
1861 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1862 | ASSERT_EQ(r, 0); |
1863 | ASSERT_EQ(0u, statfs_pool.allocated); | |
1864 | ASSERT_EQ(0u, statfs_pool.data_stored); | |
1865 | ASSERT_EQ(0u, statfs_pool.data_compressed_original); | |
1866 | ASSERT_EQ(0u, statfs_pool.data_compressed); | |
1867 | ASSERT_EQ(0u, statfs_pool.data_compressed_allocated); | |
7c673cae | 1868 | //force fsck |
11fdf7f2 | 1869 | ch.reset(); |
7c673cae | 1870 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1871 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1872 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1873 | ch = store->open_collection(cid); |
7c673cae FG |
1874 | } |
1875 | { | |
1876 | ObjectStore::Transaction t; | |
1877 | std::string s(0x10000, 'c'); | |
1878 | bufferlist bl; | |
1879 | bl.append(s); | |
1880 | bl.append(s); | |
1881 | bl.append(s); | |
1882 | bl.append(s.substr(0, 0x10000-2)); | |
1883 | t.write(cid, hoid, 0, bl.length(), bl); | |
1884 | cerr << "Yet another compressible write" << std::endl; | |
11fdf7f2 | 1885 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1886 | ASSERT_EQ(r, 0); |
1887 | struct store_statfs_t statfs; | |
1888 | r = store->statfs(&statfs); | |
1889 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1890 | ASSERT_EQ(0x40000 - 2, statfs.data_stored); |
7c673cae | 1891 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1892 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1893 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1894 | ASSERT_EQ(0x10000, statfs.data_compressed_allocated); | |
1895 | ||
1896 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1897 | bool per_pool_omap; |
1898 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1899 | ASSERT_EQ(r, 0); |
1900 | ASSERT_EQ(0x40000 - 2, statfs_pool.data_stored); | |
1901 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1902 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1903 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1904 | ASSERT_EQ(0x10000, statfs_pool.data_compressed_allocated); | |
7c673cae | 1905 | //force fsck |
11fdf7f2 | 1906 | ch.reset(); |
7c673cae | 1907 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1908 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1909 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1910 | ch = store->open_collection(cid); |
7c673cae FG |
1911 | } |
1912 | { | |
1913 | struct store_statfs_t statfs; | |
1914 | r = store->statfs(&statfs); | |
1915 | ASSERT_EQ(r, 0); | |
1916 | ||
11fdf7f2 | 1917 | struct store_statfs_t statfs_pool; |
9f95a23c TL |
1918 | bool per_pool_omap; |
1919 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1920 | ASSERT_EQ(r, 0); |
1921 | ||
7c673cae FG |
1922 | ObjectStore::Transaction t; |
1923 | t.clone(cid, hoid, hoid2); | |
1924 | cerr << "Clone compressed objecte" << std::endl; | |
11fdf7f2 | 1925 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1926 | ASSERT_EQ(r, 0); |
1927 | struct store_statfs_t statfs2; | |
1928 | r = store->statfs(&statfs2); | |
1929 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1930 | ASSERT_GT(statfs2.data_stored, statfs.data_stored); |
7c673cae | 1931 | ASSERT_EQ(statfs2.allocated, statfs.allocated); |
11fdf7f2 TL |
1932 | ASSERT_GT(statfs2.data_compressed, statfs.data_compressed); |
1933 | ASSERT_GT(statfs2.data_compressed_original, statfs.data_compressed_original); | |
1934 | ASSERT_EQ(statfs2.data_compressed_allocated, statfs.data_compressed_allocated); | |
1935 | ||
1936 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1937 | r = store->pool_statfs(poolid, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1938 | ASSERT_EQ(r, 0); |
1939 | ASSERT_GT(statfs2_pool.data_stored, statfs_pool.data_stored); | |
1940 | ASSERT_EQ(statfs2_pool.allocated, statfs_pool.allocated); | |
1941 | ASSERT_GT(statfs2_pool.data_compressed, statfs_pool.data_compressed); | |
1942 | ASSERT_GT(statfs2_pool.data_compressed_original, | |
1943 | statfs_pool.data_compressed_original); | |
1944 | ASSERT_EQ(statfs2_pool.data_compressed_allocated, | |
1945 | statfs_pool.data_compressed_allocated); | |
1946 | } | |
1947 | ||
1948 | { | |
1949 | // verify no | |
1950 | auto poolid2 = poolid + 1; | |
1951 | coll_t cid2 = coll_t(spg_t(pg_t(20, poolid2), shard_id_t::NO_SHARD)); | |
1952 | ghobject_t hoid(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
1953 | string(), | |
1954 | 0, | |
1955 | poolid2, | |
1956 | string())); | |
1957 | auto ch = store->create_new_collection(cid2); | |
1958 | ||
1959 | { | |
1960 | ||
1961 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
1962 | bool per_pool_omap; |
1963 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
1964 | ASSERT_EQ(r, 0); |
1965 | ||
1966 | cerr << "Creating second collection " << cid2 << std::endl; | |
1967 | ObjectStore::Transaction t; | |
1968 | t.create_collection(cid2, 0); | |
1969 | r = queue_transaction(store, ch, std::move(t)); | |
1970 | ASSERT_EQ(r, 0); | |
1971 | ||
1972 | t = ObjectStore::Transaction(); | |
1973 | bufferlist bl; | |
1974 | bl.append("abcde"); | |
1975 | t.write(cid2, hoid, 0, 5, bl); | |
1976 | r = queue_transaction(store, ch, std::move(t)); | |
1977 | ASSERT_EQ(r, 0); | |
1978 | ||
1979 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1980 | r = store->pool_statfs(poolid2, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1981 | ASSERT_EQ(r, 0); |
1982 | ASSERT_EQ(5, statfs2_pool.data_stored); | |
1983 | ASSERT_EQ(0x10000, statfs2_pool.allocated); | |
1984 | ASSERT_EQ(0, statfs2_pool.data_compressed); | |
1985 | ASSERT_EQ(0, statfs2_pool.data_compressed_original); | |
1986 | ASSERT_EQ(0, statfs2_pool.data_compressed_allocated); | |
1987 | ||
1988 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 1989 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1990 | ASSERT_EQ(r, 0); |
1991 | // adjust 'available' since it has changed | |
1992 | statfs1_pool_again.available = statfs1_pool.available; | |
1993 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
1994 | ||
1995 | t = ObjectStore::Transaction(); | |
1996 | t.remove(cid2, hoid); | |
1997 | t.remove_collection(cid2); | |
1998 | cerr << "Cleaning" << std::endl; | |
1999 | r = queue_transaction(store, ch, std::move(t)); | |
2000 | ASSERT_EQ(r, 0); | |
2001 | } | |
2002 | } | |
2003 | ||
2004 | { | |
2005 | // verify ops on temporary object | |
2006 | ||
2007 | auto poolid3 = poolid + 2; | |
2008 | coll_t cid3 = coll_t(spg_t(pg_t(20, poolid3), shard_id_t::NO_SHARD)); | |
2009 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP), | |
2010 | string(), | |
2011 | 0, | |
2012 | poolid3, | |
2013 | string())); | |
2014 | ghobject_t hoid3_temp; | |
2015 | hoid3_temp.hobj = hoid3.hobj.make_temp_hobject("Object 3 temp"); | |
2016 | auto ch3 = store->create_new_collection(cid3); | |
2017 | { | |
2018 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
2019 | bool per_pool_omap; |
2020 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
2021 | ASSERT_EQ(r, 0); |
2022 | ||
2023 | cerr << "Creating third collection " << cid3 << std::endl; | |
2024 | ObjectStore::Transaction t; | |
2025 | t.create_collection(cid3, 0); | |
2026 | r = queue_transaction(store, ch3, std::move(t)); | |
2027 | ASSERT_EQ(r, 0); | |
2028 | ||
2029 | t = ObjectStore::Transaction(); | |
2030 | bufferlist bl; | |
2031 | bl.append("abcde"); | |
2032 | t.write(cid3, hoid3_temp, 0, 5, bl); | |
2033 | r = queue_transaction(store, ch3, std::move(t)); | |
2034 | ASSERT_EQ(r, 0); | |
2035 | ||
2036 | struct store_statfs_t statfs3_pool; | |
9f95a23c | 2037 | r = store->pool_statfs(poolid3, &statfs3_pool, &per_pool_omap); |
11fdf7f2 TL |
2038 | ASSERT_EQ(r, 0); |
2039 | ASSERT_EQ(5, statfs3_pool.data_stored); | |
2040 | ASSERT_EQ(0x10000, statfs3_pool.allocated); | |
2041 | ASSERT_EQ(0, statfs3_pool.data_compressed); | |
2042 | ASSERT_EQ(0, statfs3_pool.data_compressed_original); | |
2043 | ASSERT_EQ(0, statfs3_pool.data_compressed_allocated); | |
2044 | ||
2045 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 2046 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
2047 | ASSERT_EQ(r, 0); |
2048 | // adjust 'available' since it has changed | |
2049 | statfs1_pool_again.available = statfs1_pool.available; | |
2050 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
2051 | ||
2052 | //force fsck | |
2053 | ch.reset(); | |
2054 | ch3.reset(); | |
2055 | EXPECT_EQ(store->umount(), 0); | |
2056 | EXPECT_EQ(store->mount(), 0); | |
2057 | ch = store->open_collection(cid); | |
2058 | ch3 = store->open_collection(cid3); | |
2059 | ||
2060 | t = ObjectStore::Transaction(); | |
2061 | t.collection_move_rename( | |
2062 | cid3, hoid3_temp, | |
2063 | cid3, hoid3); | |
2064 | r = queue_transaction(store, ch3, std::move(t)); | |
2065 | ASSERT_EQ(r, 0); | |
2066 | ||
2067 | struct store_statfs_t statfs3_pool_again; | |
9f95a23c | 2068 | r = store->pool_statfs(poolid3, &statfs3_pool_again, &per_pool_omap); |
11fdf7f2 TL |
2069 | ASSERT_EQ(r, 0); |
2070 | ASSERT_EQ(statfs3_pool_again, statfs3_pool); | |
2071 | ||
2072 | //force fsck | |
2073 | ch.reset(); | |
2074 | ch3.reset(); | |
2075 | EXPECT_EQ(store->umount(), 0); | |
2076 | EXPECT_EQ(store->mount(), 0); | |
2077 | ch = store->open_collection(cid); | |
2078 | ch3 = store->open_collection(cid3); | |
2079 | ||
2080 | t = ObjectStore::Transaction(); | |
2081 | t.remove(cid3, hoid3); | |
2082 | t.remove_collection(cid3); | |
2083 | cerr << "Cleaning" << std::endl; | |
2084 | r = queue_transaction(store, ch3, std::move(t)); | |
2085 | ASSERT_EQ(r, 0); | |
2086 | } | |
7c673cae FG |
2087 | } |
2088 | ||
2089 | { | |
2090 | ObjectStore::Transaction t; | |
2091 | t.remove(cid, hoid); | |
2092 | t.remove(cid, hoid2); | |
2093 | t.remove_collection(cid); | |
2094 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2095 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2096 | ASSERT_EQ(r, 0); |
2097 | ||
2098 | struct store_statfs_t statfs; | |
2099 | r = store->statfs(&statfs); | |
2100 | ASSERT_EQ(r, 0); | |
2101 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
2102 | ASSERT_EQ( 0u, statfs.data_stored); |
2103 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
2104 | ASSERT_EQ( 0u, statfs.data_compressed); | |
2105 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
2106 | ||
2107 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
2108 | bool per_pool_omap; |
2109 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
2110 | ASSERT_EQ(r, 0); |
2111 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
2112 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
2113 | ASSERT_EQ( 0u, statfs_pool.data_compressed_original); | |
2114 | ASSERT_EQ( 0u, statfs_pool.data_compressed); | |
2115 | ASSERT_EQ( 0u, statfs_pool.data_compressed_allocated); | |
2116 | } | |
7c673cae FG |
2117 | } |
2118 | ||
2119 | TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) { | |
2120 | if(string(GetParam()) != "bluestore") | |
2121 | return; | |
20effc67 TL |
2122 | if (smr) { |
2123 | cout << "TODO: fix this for smr" << std::endl; | |
2124 | return; | |
2125 | } | |
f67539c2 | 2126 | SetVal(g_conf(), "bluestore_block_db_path", ""); |
7c673cae FG |
2127 | StartDeferred(0x10000); |
2128 | ||
7c673cae FG |
2129 | int r; |
2130 | coll_t cid; | |
2131 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2132 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2133 | { |
2134 | ObjectStore::Transaction t; | |
2135 | t.create_collection(cid, 0); | |
2136 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2137 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2138 | ASSERT_EQ(r, 0); |
2139 | } | |
2140 | { | |
11fdf7f2 | 2141 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
2142 | ASSERT_TRUE(!exists); |
2143 | ||
2144 | ObjectStore::Transaction t; | |
2145 | t.touch(cid, hoid); | |
2146 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 2147 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2148 | ASSERT_EQ(r, 0); |
2149 | ||
11fdf7f2 | 2150 | exists = store->exists(ch, hoid); |
7c673cae FG |
2151 | ASSERT_EQ(true, exists); |
2152 | } | |
2153 | { | |
2154 | struct store_statfs_t statfs; | |
2155 | int r = store->statfs(&statfs); | |
2156 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 2157 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); |
7c673cae | 2158 | ASSERT_EQ(0u, statfs.allocated); |
11fdf7f2 TL |
2159 | ASSERT_EQ(0u, statfs.data_stored); |
2160 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
7c673cae FG |
2161 | } |
2162 | std::string data; | |
2163 | data.resize(0x10000 * 3); | |
2164 | { | |
2165 | ObjectStore::Transaction t; | |
2166 | for(size_t i = 0;i < data.size(); i++) | |
2167 | data[i] = i / 256 + 1; | |
2168 | bufferlist bl, newdata; | |
2169 | bl.append(data); | |
2170 | t.write(cid, hoid, 0, bl.length(), bl); | |
2171 | t.zero(cid, hoid, 0x10000, 0x10000); | |
2172 | cerr << "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl; | |
11fdf7f2 | 2173 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2174 | ASSERT_EQ(r, 0); |
2175 | ||
2176 | struct store_statfs_t statfs; | |
2177 | int r = store->statfs(&statfs); | |
2178 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 2179 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae FG |
2180 | ASSERT_EQ(0x20000, statfs.allocated); |
2181 | ||
11fdf7f2 | 2182 | r = store->read(ch, hoid, 0, data.size(), newdata); |
7c673cae FG |
2183 | ASSERT_EQ(r, (int)data.size()); |
2184 | { | |
2185 | bufferlist expected; | |
2186 | expected.append(data.substr(0, 0x10000)); | |
2187 | expected.append(string(0x10000, 0)); | |
2188 | expected.append(data.substr(0x20000, 0x10000)); | |
2189 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2190 | } | |
2191 | newdata.clear(); | |
2192 | ||
11fdf7f2 | 2193 | r = store->read(ch, hoid, 1, data.size()-2, newdata); |
7c673cae FG |
2194 | ASSERT_EQ(r, (int)data.size()-2); |
2195 | { | |
2196 | bufferlist expected; | |
2197 | expected.append(data.substr(1, 0x10000-1)); | |
2198 | expected.append(string(0x10000, 0)); | |
2199 | expected.append(data.substr(0x20000, 0x10000 - 1)); | |
2200 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2201 | } | |
2202 | newdata.clear(); | |
2203 | } | |
2204 | //force fsck | |
11fdf7f2 | 2205 | ch.reset(); |
7c673cae | 2206 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2207 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2208 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2209 | ch = store->open_collection(cid); |
7c673cae FG |
2210 | |
2211 | { | |
2212 | ObjectStore::Transaction t; | |
2213 | std::string data2(3, 'b'); | |
2214 | bufferlist bl, newdata; | |
2215 | bl.append(data2); | |
2216 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
2217 | cerr << "Write 3 bytes after the hole" << std::endl; | |
11fdf7f2 | 2218 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2219 | ASSERT_EQ(r, 0); |
2220 | ||
2221 | struct store_statfs_t statfs; | |
2222 | int r = store->statfs(&statfs); | |
2223 | ASSERT_EQ(r, 0); | |
2224 | ASSERT_EQ(0x20000, statfs.allocated); | |
11fdf7f2 | 2225 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae | 2226 | |
11fdf7f2 | 2227 | r = store->read(ch, hoid, 0x20000-1, 21, newdata); |
7c673cae FG |
2228 | ASSERT_EQ(r, (int)21); |
2229 | { | |
2230 | bufferlist expected; | |
2231 | expected.append(string(0x1, 0)); | |
2232 | expected.append(string(data2)); | |
2233 | expected.append(data.substr(0x20003, 21-4)); | |
2234 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2235 | } | |
2236 | newdata.clear(); | |
2237 | } | |
2238 | //force fsck | |
11fdf7f2 | 2239 | ch.reset(); |
7c673cae | 2240 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2241 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2242 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2243 | ch = store->open_collection(cid); |
7c673cae FG |
2244 | |
2245 | { | |
2246 | ObjectStore::Transaction t; | |
2247 | std::string data2(3, 'a'); | |
2248 | bufferlist bl, newdata; | |
2249 | bl.append(data2); | |
2250 | t.write(cid, hoid, 0x10000+1, bl.length(), bl); | |
2251 | cerr << "Write 3 bytes to the hole" << std::endl; | |
11fdf7f2 | 2252 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2253 | ASSERT_EQ(r, 0); |
2254 | ||
2255 | struct store_statfs_t statfs; | |
2256 | int r = store->statfs(&statfs); | |
2257 | ASSERT_EQ(r, 0); | |
2258 | ASSERT_EQ(0x30000, statfs.allocated); | |
11fdf7f2 | 2259 | ASSERT_EQ(0x20003, statfs.data_stored); |
7c673cae | 2260 | |
11fdf7f2 | 2261 | r = store->read(ch, hoid, 0x10000-1, 0x10000+22, newdata); |
7c673cae FG |
2262 | ASSERT_EQ(r, (int)0x10000+22); |
2263 | { | |
2264 | bufferlist expected; | |
2265 | expected.append(data.substr(0x10000-1, 1)); | |
2266 | expected.append(string(0x1, 0)); | |
2267 | expected.append(data2); | |
2268 | expected.append(string(0x10000-4, 0)); | |
2269 | expected.append(string(0x3, 'b')); | |
2270 | expected.append(data.substr(0x20004, 21-3)); | |
2271 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2272 | } | |
2273 | newdata.clear(); | |
2274 | } | |
2275 | { | |
2276 | ObjectStore::Transaction t; | |
2277 | bufferlist bl, newdata; | |
2278 | bl.append(string(0x30000, 'c')); | |
2279 | t.write(cid, hoid, 0, 0x30000, bl); | |
2280 | t.zero(cid, hoid, 0, 0x10000); | |
2281 | t.zero(cid, hoid, 0x20000, 0x10000); | |
11fdf7f2 TL |
2282 | cerr << "Rewrite an object and create two holes at the beginning and the end" << std::endl; |
2283 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
2284 | ASSERT_EQ(r, 0); |
2285 | ||
2286 | struct store_statfs_t statfs; | |
2287 | int r = store->statfs(&statfs); | |
2288 | ASSERT_EQ(r, 0); | |
2289 | ASSERT_EQ(0x10000, statfs.allocated); | |
11fdf7f2 | 2290 | ASSERT_EQ(0x10000, statfs.data_stored); |
7c673cae | 2291 | |
11fdf7f2 | 2292 | r = store->read(ch, hoid, 0, 0x30000, newdata); |
7c673cae FG |
2293 | ASSERT_EQ(r, (int)0x30000); |
2294 | { | |
2295 | bufferlist expected; | |
2296 | expected.append(string(0x10000, 0)); | |
2297 | expected.append(string(0x10000, 'c')); | |
2298 | expected.append(string(0x10000, 0)); | |
2299 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2300 | } | |
2301 | newdata.clear(); | |
2302 | } | |
2303 | ||
2304 | //force fsck | |
11fdf7f2 | 2305 | ch.reset(); |
7c673cae | 2306 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2307 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2308 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2309 | ch = store->open_collection(cid); |
7c673cae FG |
2310 | |
2311 | { | |
2312 | ObjectStore::Transaction t; | |
2313 | t.remove(cid, hoid); | |
2314 | t.remove_collection(cid); | |
2315 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2316 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2317 | ASSERT_EQ(r, 0); |
2318 | ||
2319 | struct store_statfs_t statfs; | |
2320 | r = store->statfs(&statfs); | |
2321 | ASSERT_EQ(r, 0); | |
2322 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
2323 | ASSERT_EQ( 0u, statfs.data_stored); |
2324 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
2325 | ASSERT_EQ( 0u, statfs.data_compressed); | |
2326 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
7c673cae FG |
2327 | } |
2328 | } | |
2329 | #endif | |
2330 | ||
2331 | TEST_P(StoreTest, ManySmallWrite) { | |
7c673cae FG |
2332 | int r; |
2333 | coll_t cid; | |
2334 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2335 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2336 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2337 | { |
2338 | ObjectStore::Transaction t; | |
2339 | t.create_collection(cid, 0); | |
2340 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2341 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2342 | ASSERT_EQ(r, 0); |
2343 | } | |
2344 | bufferlist bl; | |
2345 | bufferptr bp(4096); | |
2346 | bp.zero(); | |
2347 | bl.append(bp); | |
2348 | for (int i=0; i<100; ++i) { | |
2349 | ObjectStore::Transaction t; | |
2350 | t.write(cid, a, i*4096, 4096, bl, 0); | |
11fdf7f2 | 2351 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2352 | ASSERT_EQ(r, 0); |
2353 | } | |
2354 | for (int i=0; i<100; ++i) { | |
2355 | ObjectStore::Transaction t; | |
2356 | t.write(cid, b, (rand() % 1024)*4096, 4096, bl, 0); | |
11fdf7f2 | 2357 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2358 | ASSERT_EQ(r, 0); |
2359 | } | |
2360 | { | |
2361 | ObjectStore::Transaction t; | |
2362 | t.remove(cid, a); | |
2363 | t.remove(cid, b); | |
2364 | t.remove_collection(cid); | |
2365 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2366 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2367 | ASSERT_EQ(r, 0); |
2368 | } | |
2369 | } | |
2370 | ||
2371 | TEST_P(StoreTest, MultiSmallWriteSameBlock) { | |
7c673cae FG |
2372 | int r; |
2373 | coll_t cid; | |
2374 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2375 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2376 | { |
2377 | ObjectStore::Transaction t; | |
2378 | t.create_collection(cid, 0); | |
2379 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2380 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2381 | ASSERT_EQ(r, 0); |
2382 | } | |
2383 | bufferlist bl; | |
2384 | bl.append("short"); | |
2385 | C_SaferCond c, d; | |
2386 | // touch same block in both same transaction, tls, and pipelined txns | |
2387 | { | |
2388 | ObjectStore::Transaction t, u; | |
2389 | t.write(cid, a, 0, 5, bl, 0); | |
2390 | t.write(cid, a, 5, 5, bl, 0); | |
2391 | t.write(cid, a, 4094, 5, bl, 0); | |
2392 | t.write(cid, a, 9000, 5, bl, 0); | |
2393 | u.write(cid, a, 10, 5, bl, 0); | |
2394 | u.write(cid, a, 7000, 5, bl, 0); | |
11fdf7f2 | 2395 | t.register_on_commit(&c); |
7c673cae | 2396 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2397 | store->queue_transactions(ch, v); |
7c673cae FG |
2398 | } |
2399 | { | |
2400 | ObjectStore::Transaction t, u; | |
2401 | t.write(cid, a, 40, 5, bl, 0); | |
2402 | t.write(cid, a, 45, 5, bl, 0); | |
2403 | t.write(cid, a, 4094, 5, bl, 0); | |
2404 | t.write(cid, a, 6000, 5, bl, 0); | |
2405 | u.write(cid, a, 610, 5, bl, 0); | |
2406 | u.write(cid, a, 11000, 5, bl, 0); | |
11fdf7f2 | 2407 | t.register_on_commit(&d); |
7c673cae | 2408 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2409 | store->queue_transactions(ch, v); |
7c673cae FG |
2410 | } |
2411 | c.wait(); | |
2412 | d.wait(); | |
2413 | { | |
2414 | bufferlist bl2; | |
11fdf7f2 | 2415 | r = store->read(ch, a, 0, 16000, bl2); |
7c673cae FG |
2416 | ASSERT_GE(r, 0); |
2417 | } | |
2418 | { | |
2419 | ObjectStore::Transaction t; | |
2420 | t.remove(cid, a); | |
2421 | t.remove_collection(cid); | |
2422 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2423 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2424 | ASSERT_EQ(r, 0); |
2425 | } | |
2426 | } | |
2427 | ||
2428 | TEST_P(StoreTest, SmallSkipFront) { | |
7c673cae FG |
2429 | int r; |
2430 | coll_t cid; | |
2431 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2432 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2433 | { |
2434 | ObjectStore::Transaction t; | |
2435 | t.create_collection(cid, 0); | |
2436 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2437 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2438 | ASSERT_EQ(r, 0); |
2439 | } | |
2440 | { | |
2441 | ObjectStore::Transaction t; | |
2442 | t.touch(cid, a); | |
2443 | t.truncate(cid, a, 3000); | |
11fdf7f2 | 2444 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2445 | ASSERT_EQ(r, 0); |
2446 | } | |
2447 | { | |
2448 | bufferlist bl; | |
2449 | bufferptr bp(4096); | |
2450 | memset(bp.c_str(), 1, 4096); | |
2451 | bl.append(bp); | |
2452 | ObjectStore::Transaction t; | |
2453 | t.write(cid, a, 4096, 4096, bl); | |
11fdf7f2 | 2454 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2455 | ASSERT_EQ(r, 0); |
2456 | } | |
2457 | { | |
2458 | bufferlist bl; | |
11fdf7f2 | 2459 | ASSERT_EQ(8192, store->read(ch, a, 0, 8192, bl)); |
7c673cae FG |
2460 | for (unsigned i=0; i<4096; ++i) |
2461 | ASSERT_EQ(0, bl[i]); | |
2462 | for (unsigned i=4096; i<8192; ++i) | |
2463 | ASSERT_EQ(1, bl[i]); | |
2464 | } | |
2465 | { | |
2466 | ObjectStore::Transaction t; | |
2467 | t.remove(cid, a); | |
2468 | t.remove_collection(cid); | |
2469 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2470 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2471 | ASSERT_EQ(r, 0); |
2472 | } | |
2473 | } | |
2474 | ||
2475 | TEST_P(StoreTest, AppendDeferredVsTailCache) { | |
7c673cae FG |
2476 | int r; |
2477 | coll_t cid; | |
2478 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
11fdf7f2 | 2479 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2480 | { |
2481 | ObjectStore::Transaction t; | |
2482 | t.create_collection(cid, 0); | |
2483 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2484 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2485 | ASSERT_EQ(r, 0); |
2486 | } | |
11fdf7f2 | 2487 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2488 | unsigned size = min_alloc / 3; |
2489 | bufferptr bpa(size); | |
2490 | memset(bpa.c_str(), 1, bpa.length()); | |
2491 | bufferlist bla; | |
2492 | bla.append(bpa); | |
2493 | { | |
2494 | ObjectStore::Transaction t; | |
2495 | t.write(cid, a, 0, bla.length(), bla, 0); | |
11fdf7f2 | 2496 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2497 | ASSERT_EQ(r, 0); |
2498 | } | |
2499 | ||
2500 | // force cached tail to clear ... | |
2501 | { | |
11fdf7f2 | 2502 | ch.reset(); |
7c673cae FG |
2503 | int r = store->umount(); |
2504 | ASSERT_EQ(0, r); | |
2505 | r = store->mount(); | |
2506 | ASSERT_EQ(0, r); | |
11fdf7f2 | 2507 | ch = store->open_collection(cid); |
7c673cae FG |
2508 | } |
2509 | ||
2510 | bufferptr bpb(size); | |
2511 | memset(bpb.c_str(), 2, bpb.length()); | |
2512 | bufferlist blb; | |
2513 | blb.append(bpb); | |
2514 | { | |
2515 | ObjectStore::Transaction t; | |
2516 | t.write(cid, a, bla.length(), blb.length(), blb, 0); | |
11fdf7f2 | 2517 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2518 | ASSERT_EQ(r, 0); |
2519 | } | |
2520 | bufferptr bpc(size); | |
2521 | memset(bpc.c_str(), 3, bpc.length()); | |
2522 | bufferlist blc; | |
2523 | blc.append(bpc); | |
2524 | { | |
2525 | ObjectStore::Transaction t; | |
2526 | t.write(cid, a, bla.length() + blb.length(), blc.length(), blc, 0); | |
11fdf7f2 | 2527 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2528 | ASSERT_EQ(r, 0); |
2529 | } | |
2530 | bufferlist final; | |
2531 | final.append(bla); | |
2532 | final.append(blb); | |
2533 | final.append(blc); | |
2534 | bufferlist actual; | |
2535 | { | |
2536 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2537 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2538 | ASSERT_TRUE(bl_eq(final, actual)); |
2539 | } | |
2540 | { | |
2541 | ObjectStore::Transaction t; | |
2542 | t.remove(cid, a); | |
2543 | t.remove_collection(cid); | |
2544 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2545 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2546 | ASSERT_EQ(r, 0); |
2547 | } | |
7c673cae FG |
2548 | } |
2549 | ||
2550 | TEST_P(StoreTest, AppendZeroTrailingSharedBlock) { | |
7c673cae FG |
2551 | int r; |
2552 | coll_t cid; | |
2553 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
2554 | ghobject_t b = a; | |
2555 | b.hobj.snap = 1; | |
11fdf7f2 | 2556 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2557 | { |
2558 | ObjectStore::Transaction t; | |
2559 | t.create_collection(cid, 0); | |
2560 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2561 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2562 | ASSERT_EQ(r, 0); |
2563 | } | |
11fdf7f2 | 2564 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2565 | unsigned size = min_alloc / 3; |
2566 | bufferptr bpa(size); | |
2567 | memset(bpa.c_str(), 1, bpa.length()); | |
2568 | bufferlist bla; | |
2569 | bla.append(bpa); | |
2570 | // make sure there is some trailing gunk in the last block | |
2571 | { | |
2572 | bufferlist bt; | |
2573 | bt.append(bla); | |
2574 | bt.append("BADBADBADBAD"); | |
2575 | ObjectStore::Transaction t; | |
2576 | t.write(cid, a, 0, bt.length(), bt, 0); | |
11fdf7f2 | 2577 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2578 | ASSERT_EQ(r, 0); |
2579 | } | |
2580 | { | |
2581 | ObjectStore::Transaction t; | |
2582 | t.truncate(cid, a, size); | |
11fdf7f2 | 2583 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2584 | ASSERT_EQ(r, 0); |
2585 | } | |
2586 | ||
2587 | // clone | |
2588 | { | |
2589 | ObjectStore::Transaction t; | |
2590 | t.clone(cid, a, b); | |
11fdf7f2 | 2591 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2592 | ASSERT_EQ(r, 0); |
2593 | } | |
2594 | ||
2595 | // append with implicit zeroing | |
2596 | bufferptr bpb(size); | |
2597 | memset(bpb.c_str(), 2, bpb.length()); | |
2598 | bufferlist blb; | |
2599 | blb.append(bpb); | |
2600 | { | |
2601 | ObjectStore::Transaction t; | |
2602 | t.write(cid, a, min_alloc * 3, blb.length(), blb, 0); | |
11fdf7f2 | 2603 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2604 | ASSERT_EQ(r, 0); |
2605 | } | |
2606 | bufferlist final; | |
2607 | final.append(bla); | |
2608 | bufferlist zeros; | |
2609 | zeros.append_zero(min_alloc * 3 - size); | |
2610 | final.append(zeros); | |
2611 | final.append(blb); | |
2612 | bufferlist actual; | |
2613 | { | |
2614 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2615 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2616 | final.hexdump(cout); |
2617 | actual.hexdump(cout); | |
2618 | ASSERT_TRUE(bl_eq(final, actual)); | |
2619 | } | |
2620 | { | |
2621 | ObjectStore::Transaction t; | |
2622 | t.remove(cid, a); | |
2623 | t.remove(cid, b); | |
2624 | t.remove_collection(cid); | |
2625 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2626 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2627 | ASSERT_EQ(r, 0); |
2628 | } | |
2629 | } | |
2630 | ||
2631 | TEST_P(StoreTest, SmallSequentialUnaligned) { | |
7c673cae FG |
2632 | int r; |
2633 | coll_t cid; | |
2634 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2635 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2636 | { |
2637 | ObjectStore::Transaction t; | |
2638 | t.create_collection(cid, 0); | |
2639 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2640 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2641 | ASSERT_EQ(r, 0); |
2642 | } | |
2643 | bufferlist bl; | |
2644 | int len = 1000; | |
2645 | bufferptr bp(len); | |
2646 | bp.zero(); | |
2647 | bl.append(bp); | |
2648 | for (int i=0; i<1000; ++i) { | |
2649 | ObjectStore::Transaction t; | |
2650 | t.write(cid, a, i*len, len, bl, 0); | |
11fdf7f2 | 2651 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2652 | ASSERT_EQ(r, 0); |
2653 | } | |
2654 | { | |
2655 | ObjectStore::Transaction t; | |
2656 | t.remove(cid, a); | |
2657 | t.remove_collection(cid); | |
2658 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2659 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2660 | ASSERT_EQ(r, 0); |
2661 | } | |
2662 | } | |
2663 | ||
2664 | TEST_P(StoreTest, ManyBigWrite) { | |
7c673cae FG |
2665 | int r; |
2666 | coll_t cid; | |
2667 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2668 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2669 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2670 | { |
2671 | ObjectStore::Transaction t; | |
2672 | t.create_collection(cid, 0); | |
2673 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2674 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2675 | ASSERT_EQ(r, 0); |
2676 | } | |
2677 | bufferlist bl; | |
2678 | bufferptr bp(4 * 1048576); | |
2679 | bp.zero(); | |
2680 | bl.append(bp); | |
2681 | for (int i=0; i<10; ++i) { | |
2682 | ObjectStore::Transaction t; | |
2683 | t.write(cid, a, i*4*1048586, 4*1048576, bl, 0); | |
11fdf7f2 | 2684 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2685 | ASSERT_EQ(r, 0); |
2686 | } | |
2687 | // aligned | |
2688 | for (int i=0; i<10; ++i) { | |
2689 | ObjectStore::Transaction t; | |
2690 | t.write(cid, b, (rand() % 256)*4*1048576, 4*1048576, bl, 0); | |
11fdf7f2 | 2691 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2692 | ASSERT_EQ(r, 0); |
2693 | } | |
2694 | // unaligned | |
2695 | for (int i=0; i<10; ++i) { | |
2696 | ObjectStore::Transaction t; | |
2697 | t.write(cid, b, (rand() % (256*4096))*1024, 4*1048576, bl, 0); | |
11fdf7f2 | 2698 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2699 | ASSERT_EQ(r, 0); |
2700 | } | |
2701 | // do some zeros | |
2702 | for (int i=0; i<10; ++i) { | |
2703 | ObjectStore::Transaction t; | |
2704 | t.zero(cid, b, (rand() % (256*4096))*1024, 16*1048576); | |
11fdf7f2 | 2705 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2706 | ASSERT_EQ(r, 0); |
2707 | } | |
2708 | { | |
2709 | ObjectStore::Transaction t; | |
2710 | t.remove(cid, a); | |
2711 | t.remove(cid, b); | |
2712 | t.remove_collection(cid); | |
2713 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2714 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2715 | ASSERT_EQ(r, 0); |
2716 | } | |
2717 | } | |
2718 | ||
2719 | TEST_P(StoreTest, BigWriteBigZero) { | |
7c673cae FG |
2720 | int r; |
2721 | coll_t cid; | |
2722 | ghobject_t a(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2723 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2724 | { |
2725 | ObjectStore::Transaction t; | |
2726 | t.create_collection(cid, 0); | |
11fdf7f2 | 2727 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2728 | ASSERT_EQ(r, 0); |
2729 | } | |
2730 | bufferlist bl; | |
2731 | bufferptr bp(1048576); | |
2732 | memset(bp.c_str(), 'b', bp.length()); | |
2733 | bl.append(bp); | |
2734 | bufferlist s; | |
2735 | bufferptr sp(4096); | |
2736 | memset(sp.c_str(), 's', sp.length()); | |
2737 | s.append(sp); | |
2738 | { | |
2739 | ObjectStore::Transaction t; | |
2740 | t.write(cid, a, 0, bl.length(), bl); | |
11fdf7f2 | 2741 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2742 | ASSERT_EQ(r, 0); |
2743 | } | |
2744 | { | |
2745 | ObjectStore::Transaction t; | |
2746 | t.zero(cid, a, bl.length() / 4, bl.length() / 2); | |
11fdf7f2 | 2747 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2748 | ASSERT_EQ(r, 0); |
2749 | } | |
2750 | { | |
2751 | ObjectStore::Transaction t; | |
2752 | t.write(cid, a, bl.length() / 2, s.length(), s); | |
11fdf7f2 | 2753 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2754 | ASSERT_EQ(r, 0); |
2755 | } | |
2756 | { | |
2757 | ObjectStore::Transaction t; | |
2758 | t.remove(cid, a); | |
2759 | t.remove_collection(cid); | |
11fdf7f2 | 2760 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2761 | ASSERT_EQ(r, 0); |
2762 | } | |
2763 | } | |
2764 | ||
2765 | TEST_P(StoreTest, MiscFragmentTests) { | |
7c673cae FG |
2766 | int r; |
2767 | coll_t cid; | |
2768 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2769 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2770 | { |
2771 | ObjectStore::Transaction t; | |
2772 | t.create_collection(cid, 0); | |
2773 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2774 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2775 | ASSERT_EQ(r, 0); |
2776 | } | |
2777 | bufferlist bl; | |
2778 | bufferptr bp(524288); | |
2779 | bp.zero(); | |
2780 | bl.append(bp); | |
2781 | { | |
2782 | ObjectStore::Transaction t; | |
2783 | t.write(cid, a, 0, 524288, bl, 0); | |
11fdf7f2 | 2784 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2785 | ASSERT_EQ(r, 0); |
2786 | } | |
2787 | { | |
2788 | ObjectStore::Transaction t; | |
2789 | t.write(cid, a, 1048576, 524288, bl, 0); | |
11fdf7f2 | 2790 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2791 | ASSERT_EQ(r, 0); |
2792 | } | |
2793 | { | |
2794 | bufferlist inbl; | |
11fdf7f2 | 2795 | int r = store->read(ch, a, 524288 + 131072, 1024, inbl); |
7c673cae FG |
2796 | ASSERT_EQ(r, 1024); |
2797 | ASSERT_EQ(inbl.length(), 1024u); | |
2798 | ASSERT_TRUE(inbl.is_zero()); | |
2799 | } | |
2800 | { | |
2801 | ObjectStore::Transaction t; | |
2802 | t.write(cid, a, 1048576 - 4096, 524288, bl, 0); | |
11fdf7f2 | 2803 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2804 | ASSERT_EQ(r, 0); |
2805 | } | |
2806 | { | |
2807 | ObjectStore::Transaction t; | |
2808 | t.remove(cid, a); | |
2809 | t.remove_collection(cid); | |
2810 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2811 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2812 | ASSERT_EQ(r, 0); |
2813 | } | |
2814 | ||
2815 | } | |
2816 | ||
b32b8144 | 2817 | TEST_P(StoreTest, ZeroVsObjectSize) { |
b32b8144 FG |
2818 | int r; |
2819 | coll_t cid; | |
2820 | struct stat stat; | |
2821 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2822 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2823 | { |
2824 | ObjectStore::Transaction t; | |
2825 | t.create_collection(cid, 0); | |
2826 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2827 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2828 | ASSERT_EQ(r, 0); |
2829 | } | |
2830 | bufferlist a; | |
2831 | a.append("stuff"); | |
2832 | { | |
2833 | ObjectStore::Transaction t; | |
2834 | t.write(cid, hoid, 0, 5, a); | |
11fdf7f2 | 2835 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2836 | ASSERT_EQ(r, 0); |
2837 | } | |
11fdf7f2 | 2838 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2839 | ASSERT_EQ(5, stat.st_size); |
2840 | { | |
2841 | ObjectStore::Transaction t; | |
2842 | t.zero(cid, hoid, 1, 2); | |
11fdf7f2 | 2843 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2844 | ASSERT_EQ(r, 0); |
2845 | } | |
11fdf7f2 | 2846 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2847 | ASSERT_EQ(5, stat.st_size); |
2848 | { | |
2849 | ObjectStore::Transaction t; | |
2850 | t.zero(cid, hoid, 3, 200); | |
11fdf7f2 | 2851 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2852 | ASSERT_EQ(r, 0); |
2853 | } | |
11fdf7f2 | 2854 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2855 | ASSERT_EQ(203, stat.st_size); |
2856 | { | |
2857 | ObjectStore::Transaction t; | |
2858 | t.zero(cid, hoid, 100000, 200); | |
11fdf7f2 | 2859 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2860 | ASSERT_EQ(r, 0); |
2861 | } | |
11fdf7f2 | 2862 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2863 | ASSERT_EQ(100200, stat.st_size); |
2864 | } | |
2865 | ||
7c673cae | 2866 | TEST_P(StoreTest, ZeroLengthWrite) { |
7c673cae FG |
2867 | int r; |
2868 | coll_t cid; | |
2869 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2870 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2871 | { |
2872 | ObjectStore::Transaction t; | |
2873 | t.create_collection(cid, 0); | |
2874 | t.touch(cid, hoid); | |
11fdf7f2 | 2875 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2876 | ASSERT_EQ(r, 0); |
2877 | } | |
2878 | { | |
2879 | ObjectStore::Transaction t; | |
2880 | bufferlist empty; | |
2881 | t.write(cid, hoid, 1048576, 0, empty); | |
11fdf7f2 | 2882 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2883 | ASSERT_EQ(r, 0); |
2884 | } | |
2885 | struct stat stat; | |
11fdf7f2 | 2886 | r = store->stat(ch, hoid, &stat); |
7c673cae FG |
2887 | ASSERT_EQ(0, r); |
2888 | ASSERT_EQ(0, stat.st_size); | |
2889 | ||
2890 | bufferlist newdata; | |
11fdf7f2 | 2891 | r = store->read(ch, hoid, 0, 1048576, newdata); |
7c673cae FG |
2892 | ASSERT_EQ(0, r); |
2893 | } | |
2894 | ||
b32b8144 | 2895 | TEST_P(StoreTest, ZeroLengthZero) { |
b32b8144 FG |
2896 | int r; |
2897 | coll_t cid; | |
2898 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2899 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2900 | { |
2901 | ObjectStore::Transaction t; | |
2902 | t.create_collection(cid, 0); | |
2903 | t.touch(cid, hoid); | |
11fdf7f2 | 2904 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2905 | ASSERT_EQ(0, r); |
2906 | } | |
2907 | { | |
2908 | ObjectStore::Transaction t; | |
2909 | t.zero(cid, hoid, 1048576, 0); | |
11fdf7f2 | 2910 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2911 | ASSERT_EQ(0, r); |
2912 | } | |
2913 | struct stat stat; | |
11fdf7f2 | 2914 | r = store->stat(ch, hoid, &stat); |
b32b8144 FG |
2915 | ASSERT_EQ(0, r); |
2916 | ASSERT_EQ(0, stat.st_size); | |
2917 | ||
2918 | bufferlist newdata; | |
11fdf7f2 | 2919 | r = store->read(ch, hoid, 0, 1048576, newdata); |
b32b8144 FG |
2920 | ASSERT_EQ(0, r); |
2921 | } | |
2922 | ||
7c673cae | 2923 | TEST_P(StoreTest, SimpleAttrTest) { |
7c673cae FG |
2924 | int r; |
2925 | coll_t cid; | |
2926 | ghobject_t hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP))); | |
2927 | bufferlist val, val2; | |
2928 | val.append("value"); | |
2929 | val.append("value2"); | |
2930 | { | |
11fdf7f2 TL |
2931 | auto ch = store->open_collection(cid); |
2932 | ASSERT_FALSE(ch); | |
7c673cae | 2933 | } |
11fdf7f2 | 2934 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2935 | { |
2936 | ObjectStore::Transaction t; | |
2937 | t.create_collection(cid, 0); | |
11fdf7f2 | 2938 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2939 | ASSERT_EQ(r, 0); |
2940 | } | |
2941 | { | |
2942 | bool empty; | |
11fdf7f2 | 2943 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2944 | ASSERT_EQ(0, r); |
2945 | ASSERT_TRUE(empty); | |
2946 | } | |
2947 | { | |
2948 | bufferptr bp; | |
11fdf7f2 | 2949 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2950 | ASSERT_EQ(-ENOENT, r); |
2951 | } | |
2952 | { | |
2953 | ObjectStore::Transaction t; | |
2954 | t.touch(cid, hoid); | |
2955 | t.setattr(cid, hoid, "foo", val); | |
2956 | t.setattr(cid, hoid, "bar", val2); | |
11fdf7f2 | 2957 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2958 | ASSERT_EQ(r, 0); |
2959 | } | |
2960 | { | |
2961 | bool empty; | |
11fdf7f2 | 2962 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2963 | ASSERT_EQ(0, r); |
2964 | ASSERT_TRUE(!empty); | |
2965 | } | |
2966 | { | |
2967 | bufferptr bp; | |
11fdf7f2 | 2968 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2969 | ASSERT_EQ(-ENODATA, r); |
2970 | ||
11fdf7f2 | 2971 | r = store->getattr(ch, hoid, "foo", bp); |
7c673cae FG |
2972 | ASSERT_EQ(0, r); |
2973 | bufferlist bl; | |
2974 | bl.append(bp); | |
2975 | ASSERT_TRUE(bl_eq(val, bl)); | |
2976 | ||
20effc67 | 2977 | map<string,bufferptr,less<>> bm; |
11fdf7f2 | 2978 | r = store->getattrs(ch, hoid, bm); |
7c673cae FG |
2979 | ASSERT_EQ(0, r); |
2980 | ||
2981 | } | |
2982 | { | |
2983 | ObjectStore::Transaction t; | |
2984 | t.remove(cid, hoid); | |
2985 | t.remove_collection(cid); | |
11fdf7f2 | 2986 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2987 | ASSERT_EQ(r, 0); |
2988 | } | |
2989 | } | |
2990 | ||
2991 | TEST_P(StoreTest, SimpleListTest) { | |
7c673cae FG |
2992 | int r; |
2993 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 2994 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2995 | { |
2996 | ObjectStore::Transaction t; | |
2997 | t.create_collection(cid, 0); | |
2998 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2999 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3000 | ASSERT_EQ(r, 0); |
3001 | } | |
3002 | set<ghobject_t> all; | |
3003 | { | |
3004 | ObjectStore::Transaction t; | |
3005 | for (int i=0; i<200; ++i) { | |
3006 | string name("object_"); | |
3007 | name += stringify(i); | |
3008 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
3009 | ghobject_t::NO_GEN, shard_id_t(1)); | |
3010 | hoid.hobj.pool = 1; | |
3011 | all.insert(hoid); | |
3012 | t.touch(cid, hoid); | |
3013 | cerr << "Creating object " << hoid << std::endl; | |
3014 | } | |
11fdf7f2 | 3015 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3016 | ASSERT_EQ(r, 0); |
3017 | } | |
3018 | { | |
3019 | set<ghobject_t> saw; | |
3020 | vector<ghobject_t> objects; | |
3021 | ghobject_t next, current; | |
3022 | while (!next.is_max()) { | |
f91f0fd5 TL |
3023 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 50, |
3024 | &objects, &next); | |
7c673cae FG |
3025 | ASSERT_EQ(r, 0); |
3026 | ASSERT_TRUE(sorted(objects)); | |
3027 | cout << " got " << objects.size() << " next " << next << std::endl; | |
3028 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
3029 | ++p) { | |
3030 | if (saw.count(*p)) { | |
3031 | cout << "got DUP " << *p << std::endl; | |
3032 | } else { | |
3033 | //cout << "got new " << *p << std::endl; | |
3034 | } | |
3035 | saw.insert(*p); | |
3036 | } | |
3037 | objects.clear(); | |
3038 | current = next; | |
3039 | } | |
3040 | ASSERT_EQ(saw.size(), all.size()); | |
3041 | ASSERT_EQ(saw, all); | |
3042 | } | |
3043 | { | |
3044 | ObjectStore::Transaction t; | |
3045 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
3046 | t.remove(cid, *p); | |
3047 | t.remove_collection(cid); | |
3048 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3049 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3050 | ASSERT_EQ(r, 0); |
3051 | } | |
3052 | } | |
3053 | ||
3054 | TEST_P(StoreTest, ListEndTest) { | |
7c673cae FG |
3055 | int r; |
3056 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 3057 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3058 | { |
3059 | ObjectStore::Transaction t; | |
3060 | t.create_collection(cid, 0); | |
3061 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3062 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3063 | ASSERT_EQ(r, 0); |
3064 | } | |
3065 | set<ghobject_t> all; | |
3066 | { | |
3067 | ObjectStore::Transaction t; | |
3068 | for (int i=0; i<200; ++i) { | |
3069 | string name("object_"); | |
3070 | name += stringify(i); | |
3071 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
3072 | ghobject_t::NO_GEN, shard_id_t(1)); | |
3073 | hoid.hobj.pool = 1; | |
3074 | all.insert(hoid); | |
3075 | t.touch(cid, hoid); | |
3076 | cerr << "Creating object " << hoid << std::endl; | |
3077 | } | |
11fdf7f2 | 3078 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3079 | ASSERT_EQ(r, 0); |
3080 | } | |
3081 | { | |
3082 | ghobject_t end(hobject_t(sobject_t("object_100", CEPH_NOSNAP)), | |
3083 | ghobject_t::NO_GEN, shard_id_t(1)); | |
3084 | end.hobj.pool = 1; | |
3085 | vector<ghobject_t> objects; | |
3086 | ghobject_t next; | |
f91f0fd5 | 3087 | int r = collection_list(store, ch, ghobject_t(), end, 500, &objects, &next); |
7c673cae FG |
3088 | ASSERT_EQ(r, 0); |
3089 | for (auto &p : objects) { | |
3090 | ASSERT_NE(p, end); | |
3091 | } | |
3092 | } | |
3093 | { | |
3094 | ObjectStore::Transaction t; | |
3095 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
3096 | t.remove(cid, *p); | |
3097 | t.remove_collection(cid); | |
3098 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3099 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3100 | ASSERT_EQ(r, 0); |
3101 | } | |
3102 | } | |
3103 | ||
a4b75251 TL |
3104 | TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_meta) { |
3105 | int r = 0; | |
3106 | coll_t cid; | |
3107 | auto ch = store->create_new_collection(cid); | |
3108 | { | |
3109 | ObjectStore::Transaction t; | |
3110 | t.create_collection(cid, 0); | |
3111 | r = queue_transaction(store, ch, std::move(t)); | |
3112 | ASSERT_EQ(r, 0); | |
3113 | } | |
3114 | { | |
3115 | ObjectStore::Transaction t; | |
3116 | ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), | |
3117 | "", UINT32_C(0xffffffff), -1, "nspace")); | |
3118 | t.touch(cid, hoid); | |
3119 | r = queue_transaction(store, ch, std::move(t)); | |
3120 | ASSERT_EQ(r, 0); | |
3121 | } | |
3122 | { | |
3123 | vector<ghobject_t> objects; | |
3124 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, | |
3125 | &objects, nullptr, true); | |
3126 | ASSERT_EQ(r, 0); | |
3127 | ASSERT_EQ(objects.size(), 1); | |
3128 | } | |
3129 | } | |
3130 | ||
3131 | TEST_P(StoreTest, List_0xfffffff_Hash_Test_in_PG) { | |
3132 | int r = 0; | |
3133 | const int64_t poolid = 1; | |
3134 | coll_t cid(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
3135 | auto ch = store->create_new_collection(cid); | |
3136 | { | |
3137 | ObjectStore::Transaction t; | |
3138 | t.create_collection(cid, 0); | |
3139 | r = queue_transaction(store, ch, std::move(t)); | |
3140 | ASSERT_EQ(r, 0); | |
3141 | } | |
3142 | { | |
3143 | ObjectStore::Transaction t; | |
3144 | ghobject_t hoid(hobject_t(sobject_t("obj", CEPH_NOSNAP), | |
3145 | "", UINT32_C(0xffffffff), poolid, "nspace")); | |
3146 | t.touch(cid, hoid); | |
3147 | r = queue_transaction(store, ch, std::move(t)); | |
3148 | ASSERT_EQ(r, 0); | |
3149 | } | |
3150 | { | |
3151 | vector<ghobject_t> objects; | |
3152 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, | |
3153 | &objects, nullptr, true); | |
3154 | ASSERT_EQ(r, 0); | |
3155 | ASSERT_EQ(objects.size(), 1); | |
3156 | } | |
3157 | } | |
3158 | ||
7c673cae FG |
3159 | TEST_P(StoreTest, Sort) { |
3160 | { | |
3161 | hobject_t a(sobject_t("a", CEPH_NOSNAP)); | |
3162 | hobject_t b = a; | |
3163 | ASSERT_EQ(a, b); | |
3164 | b.oid.name = "b"; | |
3165 | ASSERT_NE(a, b); | |
3166 | ASSERT_TRUE(a < b); | |
3167 | a.pool = 1; | |
3168 | b.pool = 2; | |
3169 | ASSERT_TRUE(a < b); | |
3170 | a.pool = 3; | |
3171 | ASSERT_TRUE(a > b); | |
3172 | } | |
3173 | { | |
3174 | ghobject_t a(hobject_t(sobject_t("a", CEPH_NOSNAP))); | |
3175 | ghobject_t b(hobject_t(sobject_t("b", CEPH_NOSNAP))); | |
3176 | a.hobj.pool = 1; | |
3177 | b.hobj.pool = 1; | |
3178 | ASSERT_TRUE(a < b); | |
3179 | a.hobj.pool = -3; | |
3180 | ASSERT_TRUE(a < b); | |
3181 | a.hobj.pool = 1; | |
3182 | b.hobj.pool = -3; | |
3183 | ASSERT_TRUE(a > b); | |
3184 | } | |
3185 | } | |
3186 | ||
3187 | TEST_P(StoreTest, MultipoolListTest) { | |
7c673cae FG |
3188 | int r; |
3189 | int poolid = 4373; | |
3190 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 3191 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3192 | { |
3193 | ObjectStore::Transaction t; | |
3194 | t.create_collection(cid, 0); | |
3195 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3196 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3197 | ASSERT_EQ(r, 0); |
3198 | } | |
3199 | set<ghobject_t> all, saw; | |
3200 | { | |
3201 | ObjectStore::Transaction t; | |
3202 | for (int i=0; i<200; ++i) { | |
3203 | string name("object_"); | |
3204 | name += stringify(i); | |
3205 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
3206 | if (rand() & 1) | |
3207 | hoid.hobj.pool = -2 - poolid; | |
3208 | else | |
3209 | hoid.hobj.pool = poolid; | |
3210 | all.insert(hoid); | |
3211 | t.touch(cid, hoid); | |
3212 | cerr << "Creating object " << hoid << std::endl; | |
3213 | } | |
11fdf7f2 | 3214 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3215 | ASSERT_EQ(r, 0); |
3216 | } | |
3217 | { | |
3218 | vector<ghobject_t> objects; | |
3219 | ghobject_t next, current; | |
3220 | while (!next.is_max()) { | |
f91f0fd5 TL |
3221 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 50, |
3222 | &objects, &next); | |
7c673cae FG |
3223 | ASSERT_EQ(r, 0); |
3224 | cout << " got " << objects.size() << " next " << next << std::endl; | |
3225 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
3226 | ++p) { | |
3227 | saw.insert(*p); | |
3228 | } | |
3229 | objects.clear(); | |
3230 | current = next; | |
3231 | } | |
3232 | ASSERT_EQ(saw, all); | |
3233 | } | |
3234 | { | |
3235 | ObjectStore::Transaction t; | |
3236 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
3237 | t.remove(cid, *p); | |
3238 | t.remove_collection(cid); | |
3239 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3240 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3241 | ASSERT_EQ(r, 0); |
3242 | } | |
3243 | } | |
3244 | ||
3245 | TEST_P(StoreTest, SimpleCloneTest) { | |
7c673cae FG |
3246 | int r; |
3247 | coll_t cid; | |
f91f0fd5 TL |
3248 | |
3249 | SetDeathTestStyle("threadsafe"); | |
3250 | ||
11fdf7f2 | 3251 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3252 | { |
3253 | ObjectStore::Transaction t; | |
3254 | t.create_collection(cid, 0); | |
3255 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3256 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3257 | ASSERT_EQ(r, 0); |
3258 | } | |
3259 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3260 | "key", 123, -1, "")); | |
3261 | bufferlist small, large, xlarge, newdata, attr; | |
3262 | small.append("small"); | |
3263 | large.append("large"); | |
3264 | xlarge.append("xlarge"); | |
3265 | { | |
3266 | ObjectStore::Transaction t; | |
3267 | t.touch(cid, hoid); | |
3268 | t.setattr(cid, hoid, "attr1", small); | |
3269 | t.setattr(cid, hoid, "attr2", large); | |
3270 | t.setattr(cid, hoid, "attr3", xlarge); | |
3271 | t.write(cid, hoid, 0, small.length(), small); | |
3272 | t.write(cid, hoid, 10, small.length(), small); | |
3273 | cerr << "Creating object and set attr " << hoid << std::endl; | |
11fdf7f2 | 3274 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3275 | ASSERT_EQ(r, 0); |
3276 | } | |
3277 | ||
3278 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3279 | "key", 123, -1, "")); | |
3280 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP))); | |
3281 | { | |
3282 | ObjectStore::Transaction t; | |
3283 | t.clone(cid, hoid, hoid2); | |
3284 | t.setattr(cid, hoid2, "attr2", small); | |
3285 | t.rmattr(cid, hoid2, "attr1"); | |
3286 | t.write(cid, hoid, 10, large.length(), large); | |
3287 | t.setattr(cid, hoid, "attr1", large); | |
3288 | t.setattr(cid, hoid, "attr2", small); | |
3289 | cerr << "Clone object and rm attr" << std::endl; | |
11fdf7f2 | 3290 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3291 | ASSERT_EQ(r, 0); |
3292 | ||
11fdf7f2 | 3293 | r = store->read(ch, hoid, 10, 5, newdata); |
7c673cae FG |
3294 | ASSERT_EQ(r, 5); |
3295 | ASSERT_TRUE(bl_eq(large, newdata)); | |
3296 | ||
3297 | newdata.clear(); | |
11fdf7f2 | 3298 | r = store->read(ch, hoid, 0, 5, newdata); |
7c673cae FG |
3299 | ASSERT_EQ(r, 5); |
3300 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3301 | ||
3302 | newdata.clear(); | |
11fdf7f2 | 3303 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3304 | ASSERT_EQ(r, 5); |
3305 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3306 | ||
11fdf7f2 | 3307 | r = store->getattr(ch, hoid2, "attr2", attr); |
7c673cae FG |
3308 | ASSERT_EQ(r, 0); |
3309 | ASSERT_TRUE(bl_eq(small, attr)); | |
3310 | ||
3311 | attr.clear(); | |
11fdf7f2 | 3312 | r = store->getattr(ch, hoid2, "attr3", attr); |
7c673cae FG |
3313 | ASSERT_EQ(r, 0); |
3314 | ASSERT_TRUE(bl_eq(xlarge, attr)); | |
3315 | ||
3316 | attr.clear(); | |
11fdf7f2 | 3317 | r = store->getattr(ch, hoid, "attr1", attr); |
7c673cae FG |
3318 | ASSERT_EQ(r, 0); |
3319 | ASSERT_TRUE(bl_eq(large, attr)); | |
3320 | } | |
3321 | { | |
3322 | ObjectStore::Transaction t; | |
3323 | t.remove(cid, hoid); | |
3324 | t.remove(cid, hoid2); | |
11fdf7f2 | 3325 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3326 | } |
3327 | { | |
3328 | bufferlist final; | |
3329 | bufferptr p(16384); | |
3330 | memset(p.c_str(), 1, p.length()); | |
3331 | bufferlist pl; | |
3332 | pl.append(p); | |
3333 | final.append(p); | |
3334 | ObjectStore::Transaction t; | |
3335 | t.write(cid, hoid, 0, pl.length(), pl); | |
3336 | t.clone(cid, hoid, hoid2); | |
3337 | bufferptr a(4096); | |
3338 | memset(a.c_str(), 2, a.length()); | |
3339 | bufferlist al; | |
3340 | al.append(a); | |
3341 | final.append(a); | |
3342 | t.write(cid, hoid, pl.length(), a.length(), al); | |
11fdf7f2 | 3343 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3344 | ASSERT_EQ(r, 0); |
3345 | bufferlist rl; | |
3346 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3347 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3348 | ASSERT_TRUE(bl_eq(rl, final)); |
3349 | } | |
3350 | { | |
3351 | ObjectStore::Transaction t; | |
3352 | t.remove(cid, hoid); | |
3353 | t.remove(cid, hoid2); | |
11fdf7f2 | 3354 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3355 | } |
3356 | { | |
3357 | bufferlist final; | |
3358 | bufferptr p(16384); | |
3359 | memset(p.c_str(), 111, p.length()); | |
3360 | bufferlist pl; | |
3361 | pl.append(p); | |
3362 | final.append(p); | |
3363 | ObjectStore::Transaction t; | |
3364 | t.write(cid, hoid, 0, pl.length(), pl); | |
3365 | t.clone(cid, hoid, hoid2); | |
3366 | bufferptr z(4096); | |
3367 | z.zero(); | |
3368 | final.append(z); | |
3369 | bufferptr a(4096); | |
3370 | memset(a.c_str(), 112, a.length()); | |
3371 | bufferlist al; | |
3372 | al.append(a); | |
3373 | final.append(a); | |
3374 | t.write(cid, hoid, pl.length() + z.length(), a.length(), al); | |
11fdf7f2 | 3375 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3376 | ASSERT_EQ(r, 0); |
3377 | bufferlist rl; | |
3378 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3379 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3380 | ASSERT_TRUE(bl_eq(rl, final)); |
3381 | } | |
3382 | { | |
3383 | ObjectStore::Transaction t; | |
3384 | t.remove(cid, hoid); | |
3385 | t.remove(cid, hoid2); | |
11fdf7f2 | 3386 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3387 | } |
3388 | { | |
3389 | bufferlist final; | |
3390 | bufferptr p(16000); | |
3391 | memset(p.c_str(), 5, p.length()); | |
3392 | bufferlist pl; | |
3393 | pl.append(p); | |
3394 | final.append(p); | |
3395 | ObjectStore::Transaction t; | |
3396 | t.write(cid, hoid, 0, pl.length(), pl); | |
3397 | t.clone(cid, hoid, hoid2); | |
3398 | bufferptr z(1000); | |
3399 | z.zero(); | |
3400 | final.append(z); | |
3401 | bufferptr a(8000); | |
3402 | memset(a.c_str(), 6, a.length()); | |
3403 | bufferlist al; | |
3404 | al.append(a); | |
3405 | final.append(a); | |
3406 | t.write(cid, hoid, 17000, a.length(), al); | |
11fdf7f2 | 3407 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3408 | bufferlist rl; |
3409 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3410 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3411 | /*cout << "expected:\n"; |
3412 | final.hexdump(cout); | |
3413 | cout << "got:\n"; | |
3414 | rl.hexdump(cout);*/ | |
3415 | ASSERT_TRUE(bl_eq(rl, final)); | |
3416 | } | |
3417 | { | |
3418 | ObjectStore::Transaction t; | |
3419 | t.remove(cid, hoid); | |
3420 | t.remove(cid, hoid2); | |
11fdf7f2 | 3421 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3422 | } |
3423 | { | |
3424 | bufferptr p(1048576); | |
3425 | memset(p.c_str(), 3, p.length()); | |
3426 | bufferlist pl; | |
3427 | pl.append(p); | |
3428 | ObjectStore::Transaction t; | |
3429 | t.write(cid, hoid, 0, pl.length(), pl); | |
3430 | t.clone(cid, hoid, hoid2); | |
3431 | bufferptr a(65536); | |
3432 | memset(a.c_str(), 4, a.length()); | |
3433 | bufferlist al; | |
3434 | al.append(a); | |
3435 | t.write(cid, hoid, a.length(), a.length(), al); | |
11fdf7f2 | 3436 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3437 | bufferlist rl; |
3438 | bufferlist final; | |
3439 | final.substr_of(pl, 0, al.length()); | |
3440 | final.append(al); | |
3441 | bufferlist end; | |
3442 | end.substr_of(pl, al.length()*2, pl.length() - al.length()*2); | |
3443 | final.append(end); | |
3444 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3445 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3446 | /*cout << "expected:\n"; |
3447 | final.hexdump(cout); | |
3448 | cout << "got:\n"; | |
3449 | rl.hexdump(cout);*/ | |
3450 | ASSERT_TRUE(bl_eq(rl, final)); | |
3451 | } | |
3452 | { | |
3453 | ObjectStore::Transaction t; | |
3454 | t.remove(cid, hoid); | |
3455 | t.remove(cid, hoid2); | |
11fdf7f2 | 3456 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3457 | } |
3458 | { | |
3459 | bufferptr p(65536); | |
3460 | memset(p.c_str(), 7, p.length()); | |
3461 | bufferlist pl; | |
3462 | pl.append(p); | |
3463 | ObjectStore::Transaction t; | |
3464 | t.write(cid, hoid, 0, pl.length(), pl); | |
3465 | t.clone(cid, hoid, hoid2); | |
3466 | bufferptr a(4096); | |
3467 | memset(a.c_str(), 8, a.length()); | |
3468 | bufferlist al; | |
3469 | al.append(a); | |
3470 | t.write(cid, hoid, 32768, a.length(), al); | |
11fdf7f2 | 3471 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3472 | bufferlist rl; |
3473 | bufferlist final; | |
3474 | final.substr_of(pl, 0, 32768); | |
3475 | final.append(al); | |
3476 | bufferlist end; | |
3477 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3478 | final.append(end); | |
3479 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3480 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3481 | /*cout << "expected:\n"; |
3482 | final.hexdump(cout); | |
3483 | cout << "got:\n"; | |
3484 | rl.hexdump(cout);*/ | |
3485 | ASSERT_TRUE(bl_eq(rl, final)); | |
3486 | } | |
3487 | { | |
3488 | ObjectStore::Transaction t; | |
3489 | t.remove(cid, hoid); | |
3490 | t.remove(cid, hoid2); | |
11fdf7f2 | 3491 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3492 | } |
3493 | { | |
3494 | bufferptr p(65536); | |
3495 | memset(p.c_str(), 9, p.length()); | |
3496 | bufferlist pl; | |
3497 | pl.append(p); | |
3498 | ObjectStore::Transaction t; | |
3499 | t.write(cid, hoid, 0, pl.length(), pl); | |
3500 | t.clone(cid, hoid, hoid2); | |
3501 | bufferptr a(4096); | |
3502 | memset(a.c_str(), 10, a.length()); | |
3503 | bufferlist al; | |
3504 | al.append(a); | |
3505 | t.write(cid, hoid, 33768, a.length(), al); | |
11fdf7f2 | 3506 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3507 | bufferlist rl; |
3508 | bufferlist final; | |
3509 | final.substr_of(pl, 0, 33768); | |
3510 | final.append(al); | |
3511 | bufferlist end; | |
3512 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3513 | final.append(end); | |
3514 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3515 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3516 | /*cout << "expected:\n"; |
3517 | final.hexdump(cout); | |
3518 | cout << "got:\n"; | |
3519 | rl.hexdump(cout);*/ | |
3520 | ASSERT_TRUE(bl_eq(rl, final)); | |
3521 | } | |
3522 | ||
f91f0fd5 | 3523 | { |
7c673cae | 3524 | //verify if non-empty collection is properly handled after store reload |
11fdf7f2 | 3525 | ch.reset(); |
7c673cae FG |
3526 | r = store->umount(); |
3527 | ASSERT_EQ(r, 0); | |
3528 | r = store->mount(); | |
3529 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3530 | ch = store->open_collection(cid); |
7c673cae FG |
3531 | |
3532 | ObjectStore::Transaction t; | |
3533 | t.remove_collection(cid); | |
3534 | cerr << "Invalid rm coll" << std::endl; | |
3535 | PrCtl unset_dumpable; | |
11fdf7f2 | 3536 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3537 | } |
3538 | { | |
3539 | ObjectStore::Transaction t; | |
3540 | t.touch(cid, hoid3); //new record in db | |
11fdf7f2 | 3541 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3542 | ASSERT_EQ(r, 0); |
3543 | } | |
f91f0fd5 | 3544 | { |
7c673cae FG |
3545 | ObjectStore::Transaction t; |
3546 | //verify if non-empty collection is properly handled when there are some pending removes and live records in db | |
3547 | cerr << "Invalid rm coll again" << std::endl; | |
11fdf7f2 | 3548 | ch.reset(); |
7c673cae FG |
3549 | r = store->umount(); |
3550 | ASSERT_EQ(r, 0); | |
3551 | r = store->mount(); | |
3552 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3553 | ch = store->open_collection(cid); |
7c673cae FG |
3554 | |
3555 | t.remove(cid, hoid); | |
3556 | t.remove(cid, hoid2); | |
3557 | t.remove_collection(cid); | |
3558 | PrCtl unset_dumpable; | |
11fdf7f2 | 3559 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3560 | } |
3561 | { | |
3562 | ObjectStore::Transaction t; | |
3563 | t.remove(cid, hoid); | |
3564 | t.remove(cid, hoid2); | |
3565 | t.remove(cid, hoid3); | |
3566 | t.remove_collection(cid); | |
3567 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3568 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3569 | ASSERT_EQ(r, 0); |
3570 | } | |
3571 | } | |
3572 | ||
3573 | TEST_P(StoreTest, OmapSimple) { | |
7c673cae FG |
3574 | int r; |
3575 | coll_t cid; | |
11fdf7f2 | 3576 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3577 | { |
3578 | ObjectStore::Transaction t; | |
3579 | t.create_collection(cid, 0); | |
3580 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3581 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3582 | ASSERT_EQ(r, 0); |
3583 | } | |
3584 | ghobject_t hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP), | |
3585 | "key", 123, -1, "")); | |
3586 | bufferlist small; | |
3587 | small.append("small"); | |
3588 | map<string,bufferlist> km; | |
3589 | km["foo"] = small; | |
3590 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3591 | bufferlist header; | |
3592 | header.append("this is a header"); | |
3593 | { | |
3594 | ObjectStore::Transaction t; | |
3595 | t.touch(cid, hoid); | |
3596 | t.omap_setkeys(cid, hoid, km); | |
3597 | t.omap_setheader(cid, hoid, header); | |
3598 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3599 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3600 | ASSERT_EQ(r, 0); |
3601 | } | |
3602 | // get header, keys | |
3603 | { | |
3604 | bufferlist h; | |
3605 | map<string,bufferlist> r; | |
11fdf7f2 | 3606 | store->omap_get(ch, hoid, &h, &r); |
7c673cae FG |
3607 | ASSERT_TRUE(bl_eq(header, h)); |
3608 | ASSERT_EQ(r.size(), km.size()); | |
3609 | cout << "r: " << r << std::endl; | |
3610 | } | |
3611 | // test iterator with seek_to_first | |
3612 | { | |
3613 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3614 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3615 | for (iter->seek_to_first(); iter->valid(); iter->next()) { | |
7c673cae FG |
3616 | r[iter->key()] = iter->value(); |
3617 | } | |
3618 | cout << "r: " << r << std::endl; | |
3619 | ASSERT_EQ(r.size(), km.size()); | |
3620 | } | |
3621 | // test iterator with initial lower_bound | |
3622 | { | |
3623 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3624 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3625 | for (iter->lower_bound(string()); iter->valid(); iter->next()) { | |
7c673cae FG |
3626 | r[iter->key()] = iter->value(); |
3627 | } | |
3628 | cout << "r: " << r << std::endl; | |
3629 | ASSERT_EQ(r.size(), km.size()); | |
3630 | } | |
3631 | { | |
3632 | ObjectStore::Transaction t; | |
3633 | t.remove(cid, hoid); | |
3634 | t.remove_collection(cid); | |
3635 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3636 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3637 | ASSERT_EQ(r, 0); |
3638 | } | |
3639 | } | |
3640 | ||
3641 | TEST_P(StoreTest, OmapCloneTest) { | |
7c673cae FG |
3642 | int r; |
3643 | coll_t cid; | |
11fdf7f2 | 3644 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3645 | { |
3646 | ObjectStore::Transaction t; | |
3647 | t.create_collection(cid, 0); | |
3648 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3649 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3650 | ASSERT_EQ(r, 0); |
3651 | } | |
3652 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3653 | "key", 123, -1, "")); | |
3654 | bufferlist small; | |
3655 | small.append("small"); | |
3656 | map<string,bufferlist> km; | |
3657 | km["foo"] = small; | |
3658 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3659 | bufferlist header; | |
3660 | header.append("this is a header"); | |
3661 | { | |
3662 | ObjectStore::Transaction t; | |
3663 | t.touch(cid, hoid); | |
3664 | t.omap_setkeys(cid, hoid, km); | |
3665 | t.omap_setheader(cid, hoid, header); | |
3666 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3667 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3668 | ASSERT_EQ(r, 0); |
3669 | } | |
3670 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3671 | "key", 123, -1, "")); | |
3672 | { | |
3673 | ObjectStore::Transaction t; | |
3674 | t.clone(cid, hoid, hoid2); | |
3675 | cerr << "Clone object" << std::endl; | |
11fdf7f2 | 3676 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3677 | ASSERT_EQ(r, 0); |
3678 | } | |
3679 | { | |
3680 | map<string,bufferlist> r; | |
3681 | bufferlist h; | |
11fdf7f2 | 3682 | store->omap_get(ch, hoid2, &h, &r); |
7c673cae FG |
3683 | ASSERT_TRUE(bl_eq(header, h)); |
3684 | ASSERT_EQ(r.size(), km.size()); | |
3685 | } | |
3686 | { | |
3687 | ObjectStore::Transaction t; | |
3688 | t.remove(cid, hoid); | |
3689 | t.remove(cid, hoid2); | |
3690 | t.remove_collection(cid); | |
3691 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3692 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3693 | ASSERT_EQ(r, 0); |
3694 | } | |
3695 | } | |
3696 | ||
3697 | TEST_P(StoreTest, SimpleCloneRangeTest) { | |
7c673cae FG |
3698 | int r; |
3699 | coll_t cid; | |
11fdf7f2 | 3700 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3701 | { |
3702 | ObjectStore::Transaction t; | |
3703 | t.create_collection(cid, 0); | |
3704 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3705 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3706 | ASSERT_EQ(r, 0); |
3707 | } | |
3708 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3709 | hoid.hobj.pool = -1; | |
3710 | bufferlist small, newdata; | |
3711 | small.append("small"); | |
3712 | { | |
3713 | ObjectStore::Transaction t; | |
3714 | t.write(cid, hoid, 10, 5, small); | |
3715 | cerr << "Creating object and write bl " << hoid << std::endl; | |
11fdf7f2 | 3716 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3717 | ASSERT_EQ(r, 0); |
3718 | } | |
3719 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
3720 | hoid2.hobj.pool = -1; | |
3721 | { | |
3722 | ObjectStore::Transaction t; | |
3723 | t.clone_range(cid, hoid, hoid2, 10, 5, 10); | |
3724 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3725 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 3726 | ASSERT_EQ(r, 0); |
11fdf7f2 | 3727 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3728 | ASSERT_EQ(r, 5); |
3729 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3730 | } | |
3731 | { | |
3732 | ObjectStore::Transaction t; | |
3733 | t.truncate(cid, hoid, 1024*1024); | |
3734 | t.clone_range(cid, hoid, hoid2, 0, 1024*1024, 0); | |
3735 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3736 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3737 | ASSERT_EQ(r, 0); |
3738 | struct stat stat, stat2; | |
11fdf7f2 TL |
3739 | r = store->stat(ch, hoid, &stat); |
3740 | r = store->stat(ch, hoid2, &stat2); | |
7c673cae FG |
3741 | ASSERT_EQ(stat.st_size, stat2.st_size); |
3742 | ASSERT_EQ(1024*1024, stat2.st_size); | |
3743 | } | |
3744 | { | |
3745 | ObjectStore::Transaction t; | |
3746 | t.remove(cid, hoid); | |
3747 | t.remove(cid, hoid2); | |
f67539c2 TL |
3748 | r = queue_transaction(store, ch, std::move(t)); |
3749 | ASSERT_EQ(r, 0); | |
3750 | } | |
3751 | } | |
3752 | ||
3753 | #if defined(WITH_BLUESTORE) | |
3754 | TEST_P(StoreTest, BlueStoreUnshareBlobTest) { | |
3755 | if (string(GetParam()) != "bluestore") | |
3756 | return; | |
20effc67 TL |
3757 | if (smr) { |
3758 | cout << "SKIP: non-deterministic behavior with smr" << std::endl; | |
3759 | return; | |
3760 | } | |
f67539c2 TL |
3761 | int r; |
3762 | coll_t cid; | |
3763 | auto ch = store->create_new_collection(cid); | |
3764 | { | |
3765 | ObjectStore::Transaction t; | |
3766 | t.create_collection(cid, 0); | |
3767 | cerr << "Creating collection " << cid << std::endl; | |
3768 | r = queue_transaction(store, ch, std::move(t)); | |
3769 | ASSERT_EQ(r, 0); | |
3770 | } | |
3771 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3772 | hoid.hobj.pool = -1; | |
3773 | ghobject_t hoid2(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3774 | hoid2.hobj.pool = -1; | |
3775 | hoid2.generation = 2; | |
3776 | { | |
3777 | // check if blob is unshared properly | |
3778 | bufferlist data, newdata; | |
3779 | data.append(string(8192, 'a')); | |
3780 | ||
3781 | ObjectStore::Transaction t; | |
3782 | t.write(cid, hoid, 0, data.length(), data); | |
3783 | cerr << "Creating object and write 8K " << hoid << std::endl; | |
3784 | r = queue_transaction(store, ch, std::move(t)); | |
3785 | ASSERT_EQ(r, 0); | |
3786 | ||
3787 | ObjectStore::Transaction t2; | |
3788 | t2.clone_range(cid, hoid, hoid2, 0, 4096, 0); | |
3789 | cerr << "Clone range object" << std::endl; | |
3790 | r = queue_transaction(store, ch, std::move(t2)); | |
3791 | ASSERT_EQ(r, 0); | |
3792 | ||
3793 | data.clear(); | |
3794 | data.append(string(4096, 'b')); | |
3795 | ||
3796 | ObjectStore::Transaction t3; | |
3797 | t3.write(cid, hoid, 0, data.length(), data); | |
3798 | cerr << "Writing 4k to source object " << hoid << std::endl; | |
3799 | r = queue_transaction(store, ch, std::move(t3)); | |
3800 | ASSERT_EQ(r, 0); | |
3801 | ||
3802 | { | |
3803 | // this trims hoid one out of onode cache | |
3804 | EXPECT_EQ(store->umount(), 0); | |
3805 | EXPECT_EQ(store->mount(), 0); | |
3806 | ch = store->open_collection(cid); | |
3807 | } | |
3808 | ||
3809 | ObjectStore::Transaction t4; | |
3810 | t4.remove(cid, hoid2); | |
3811 | cerr << "Deleting dest object" << hoid2 << std::endl; | |
3812 | r = queue_transaction(store, ch, std::move(t4)); | |
3813 | ASSERT_EQ(r, 0); | |
3814 | ||
20effc67 TL |
3815 | { |
3816 | // this ensures remove operation submitted to kv store | |
3817 | EXPECT_EQ(store->umount(), 0); | |
3818 | EXPECT_EQ(store->mount(), 0); | |
3819 | ch = store->open_collection(cid); | |
3820 | } | |
3821 | ||
f67539c2 TL |
3822 | bufferlist resdata; |
3823 | r = store->read(ch, hoid, 0, 0x2000, resdata); | |
3824 | ASSERT_EQ(r, 0x2000); | |
3825 | ||
3826 | { | |
3827 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
3828 | auto* kv = bstore->get_kv(); | |
3829 | ||
3830 | // to be inline with BlueStore.cc | |
3831 | const string PREFIX_SHARED_BLOB = "X"; | |
3832 | ||
3833 | size_t cnt = 0; | |
3834 | auto it = kv->get_iterator(PREFIX_SHARED_BLOB); | |
3835 | ceph_assert(it); | |
3836 | for (it->lower_bound(string()); it->valid(); it->next()) { | |
3837 | ++cnt; | |
3838 | } | |
3839 | ASSERT_EQ(cnt, 0); | |
3840 | } | |
3841 | } | |
3842 | { | |
3843 | ObjectStore::Transaction t; | |
3844 | t.remove(cid, hoid); | |
7c673cae FG |
3845 | t.remove_collection(cid); |
3846 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3847 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3848 | ASSERT_EQ(r, 0); |
3849 | } | |
3850 | } | |
3851 | ||
f67539c2 TL |
3852 | TEST_P(StoreTest, BlueStoreUnshareBlobBugTest) { |
3853 | if (string(GetParam()) != "bluestore") | |
3854 | return; | |
3855 | int r; | |
3856 | coll_t cid; | |
3857 | auto ch = store->create_new_collection(cid); | |
3858 | { | |
3859 | ObjectStore::Transaction t; | |
3860 | t.create_collection(cid, 0); | |
3861 | cerr << "Creating collection " << cid << std::endl; | |
3862 | r = queue_transaction(store, ch, std::move(t)); | |
3863 | ASSERT_EQ(r, 0); | |
3864 | } | |
3865 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3866 | hoid.hobj.pool = -1; | |
3867 | ghobject_t hoid2(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3868 | hoid2.hobj.pool = -1; | |
3869 | hoid2.generation = 2; | |
3870 | { | |
3871 | // check if blob is unshared properly | |
3872 | bufferlist data, newdata; | |
3873 | data.append(string(8192, 'a')); | |
3874 | ||
3875 | ObjectStore::Transaction t; | |
3876 | t.write(cid, hoid, 0, data.length(), data); | |
3877 | cerr << "Creating object and write 8K " << hoid << std::endl; | |
3878 | r = queue_transaction(store, ch, std::move(t)); | |
3879 | ASSERT_EQ(r, 0); | |
3880 | ||
3881 | ObjectStore::Transaction t2; | |
3882 | t2.clone_range(cid, hoid, hoid2, 0, 4096, 0); | |
3883 | cerr << "Clone range object" << std::endl; | |
3884 | r = queue_transaction(store, ch, std::move(t2)); | |
3885 | ASSERT_EQ(r, 0); | |
3886 | ||
3887 | data.clear(); | |
3888 | data.append(string(4096, 'b')); | |
3889 | ||
3890 | ObjectStore::Transaction t3; | |
3891 | t3.write(cid, hoid, 0, data.length(), data); | |
3892 | cerr << "Writing 4k to source object " << hoid << std::endl; | |
3893 | r = queue_transaction(store, ch, std::move(t3)); | |
3894 | ASSERT_EQ(r, 0); | |
3895 | ||
3896 | { | |
3897 | // this trims hoid one out of onode cache | |
3898 | EXPECT_EQ(store->umount(), 0); | |
3899 | EXPECT_EQ(store->mount(), 0); | |
3900 | ch = store->open_collection(cid); | |
3901 | } | |
3902 | ||
3903 | ObjectStore::Transaction t4; | |
3904 | t4.write(cid, hoid2, 0, data.length(), data); | |
3905 | cerr << "Writing 4k to second object " << hoid2 << std::endl; | |
3906 | r = queue_transaction(store, ch, std::move(t4)); | |
3907 | ASSERT_EQ(r, 0); | |
3908 | ||
3909 | bufferlist resdata; | |
3910 | r = store->read(ch, hoid, 0, 0x2000, resdata); | |
3911 | ASSERT_EQ(r, 0x2000); | |
3912 | ||
3913 | { | |
3914 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
3915 | auto* kv = bstore->get_kv(); | |
3916 | ||
3917 | // to be inline with BlueStore.cc | |
3918 | const string PREFIX_SHARED_BLOB = "X"; | |
3919 | ||
3920 | size_t cnt = 0; | |
3921 | auto it = kv->get_iterator(PREFIX_SHARED_BLOB); | |
3922 | ceph_assert(it); | |
3923 | for (it->lower_bound(string()); it->valid(); it->next()) { | |
3924 | ++cnt; | |
3925 | } | |
3926 | // This shows a bug in unsharing a blob, | |
3927 | // after writing to 0x0~1000 to hoid2 share blob at hoid should be | |
3928 | //unshared but it doesn't in the current implementation | |
3929 | ASSERT_EQ(cnt, 1); | |
3930 | } | |
3931 | } | |
3932 | { | |
3933 | ObjectStore::Transaction t; | |
3934 | t.remove(cid, hoid); | |
3935 | t.remove(cid, hoid2); | |
3936 | t.remove_collection(cid); | |
3937 | cerr << "Cleaning" << std::endl; | |
3938 | r = queue_transaction(store, ch, std::move(t)); | |
3939 | ASSERT_EQ(r, 0); | |
3940 | } | |
3941 | } | |
3942 | #endif | |
7c673cae FG |
3943 | |
3944 | TEST_P(StoreTest, SimpleObjectLongnameTest) { | |
7c673cae FG |
3945 | int r; |
3946 | coll_t cid; | |
11fdf7f2 | 3947 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3948 | { |
3949 | ObjectStore::Transaction t; | |
3950 | t.create_collection(cid, 0); | |
3951 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3952 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3953 | ASSERT_EQ(r, 0); |
3954 | } | |
3955 | ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP))); | |
3956 | { | |
3957 | ObjectStore::Transaction t; | |
3958 | t.touch(cid, hoid); | |
3959 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3960 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3961 | ASSERT_EQ(r, 0); |
3962 | } | |
3963 | { | |
3964 | ObjectStore::Transaction t; | |
3965 | t.remove(cid, hoid); | |
3966 | t.remove_collection(cid); | |
3967 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3968 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3969 | ASSERT_EQ(r, 0); |
3970 | } | |
3971 | } | |
3972 | ||
3973 | ghobject_t generate_long_name(unsigned i) | |
3974 | { | |
3975 | stringstream name; | |
3976 | name << "object id " << i << " "; | |
3977 | for (unsigned j = 0; j < 500; ++j) name << 'a'; | |
3978 | ghobject_t hoid(hobject_t(sobject_t(name.str(), CEPH_NOSNAP))); | |
3979 | hoid.hobj.set_hash(i % 2); | |
3980 | return hoid; | |
3981 | } | |
3982 | ||
3983 | TEST_P(StoreTest, LongnameSplitTest) { | |
7c673cae FG |
3984 | int r; |
3985 | coll_t cid; | |
11fdf7f2 | 3986 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3987 | { |
3988 | ObjectStore::Transaction t; | |
3989 | t.create_collection(cid, 0); | |
3990 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3991 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3992 | ASSERT_EQ(0, r); |
3993 | } | |
3994 | for (unsigned i = 0; i < 320; ++i) { | |
3995 | ObjectStore::Transaction t; | |
3996 | ghobject_t hoid = generate_long_name(i); | |
3997 | t.touch(cid, hoid); | |
3998 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3999 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4000 | ASSERT_EQ(0, r); |
4001 | } | |
4002 | ||
4003 | ghobject_t test_obj = generate_long_name(319); | |
4004 | ghobject_t test_obj_2 = test_obj; | |
4005 | test_obj_2.generation = 0; | |
4006 | { | |
4007 | ObjectStore::Transaction t; | |
4008 | // should cause a split | |
4009 | t.collection_move_rename( | |
4010 | cid, test_obj, | |
4011 | cid, test_obj_2); | |
11fdf7f2 | 4012 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4013 | ASSERT_EQ(0, r); |
4014 | } | |
4015 | ||
4016 | for (unsigned i = 0; i < 319; ++i) { | |
4017 | ObjectStore::Transaction t; | |
4018 | ghobject_t hoid = generate_long_name(i); | |
4019 | t.remove(cid, hoid); | |
4020 | cerr << "Removing object " << hoid << std::endl; | |
11fdf7f2 | 4021 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4022 | ASSERT_EQ(0, r); |
4023 | } | |
4024 | { | |
4025 | ObjectStore::Transaction t; | |
4026 | t.remove(cid, test_obj_2); | |
4027 | t.remove_collection(cid); | |
4028 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 4029 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4030 | ASSERT_EQ(0, r); |
4031 | } | |
4032 | ||
4033 | } | |
4034 | ||
4035 | TEST_P(StoreTest, ManyObjectTest) { | |
7c673cae FG |
4036 | int NUM_OBJS = 2000; |
4037 | int r = 0; | |
4038 | coll_t cid; | |
4039 | string base = ""; | |
4040 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
4041 | set<ghobject_t> created; | |
11fdf7f2 | 4042 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
4043 | { |
4044 | ObjectStore::Transaction t; | |
4045 | t.create_collection(cid, 0); | |
11fdf7f2 | 4046 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4047 | ASSERT_EQ(r, 0); |
4048 | } | |
4049 | for (int i = 0; i < NUM_OBJS; ++i) { | |
4050 | if (!(i % 5)) { | |
4051 | cerr << "Object " << i << std::endl; | |
4052 | } | |
4053 | ObjectStore::Transaction t; | |
4054 | char buf[100]; | |
4055 | snprintf(buf, sizeof(buf), "%d", i); | |
4056 | ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP))); | |
4057 | t.touch(cid, hoid); | |
4058 | created.insert(hoid); | |
11fdf7f2 | 4059 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4060 | ASSERT_EQ(r, 0); |
4061 | } | |
4062 | ||
4063 | for (set<ghobject_t>::iterator i = created.begin(); | |
4064 | i != created.end(); | |
4065 | ++i) { | |
4066 | struct stat buf; | |
11fdf7f2 | 4067 | ASSERT_TRUE(!store->stat(ch, *i, &buf)); |
7c673cae FG |
4068 | } |
4069 | ||
4070 | set<ghobject_t> listed, listed2; | |
4071 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
4072 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
4073 | &objects, 0); | |
7c673cae FG |
4074 | ASSERT_EQ(r, 0); |
4075 | ||
4076 | cerr << "objects.size() is " << objects.size() << std::endl; | |
4077 | for (vector<ghobject_t> ::iterator i = objects.begin(); | |
4078 | i != objects.end(); | |
4079 | ++i) { | |
4080 | listed.insert(*i); | |
4081 | ASSERT_TRUE(created.count(*i)); | |
4082 | } | |
4083 | ASSERT_TRUE(listed.size() == created.size()); | |
4084 | ||
4085 | ghobject_t start, next; | |
4086 | objects.clear(); | |
f91f0fd5 TL |
4087 | r = collection_list( |
4088 | store, | |
11fdf7f2 | 4089 | ch, |
7c673cae FG |
4090 | ghobject_t::get_max(), |
4091 | ghobject_t::get_max(), | |
4092 | 50, | |
4093 | &objects, | |
4094 | &next | |
4095 | ); | |
4096 | ASSERT_EQ(r, 0); | |
4097 | ASSERT_TRUE(objects.empty()); | |
4098 | ||
4099 | objects.clear(); | |
4100 | listed.clear(); | |
4101 | ghobject_t start2, next2; | |
4102 | while (1) { | |
f91f0fd5 TL |
4103 | r = collection_list(store, ch, start, ghobject_t::get_max(), 50, &objects, |
4104 | &next); | |
7c673cae FG |
4105 | ASSERT_TRUE(sorted(objects)); |
4106 | ASSERT_EQ(r, 0); | |
4107 | listed.insert(objects.begin(), objects.end()); | |
4108 | if (objects.size() < 50) { | |
4109 | ASSERT_TRUE(next.is_max()); | |
4110 | break; | |
4111 | } | |
4112 | objects.clear(); | |
4113 | ||
4114 | start = next; | |
4115 | } | |
4116 | cerr << "listed.size() is " << listed.size() << std::endl; | |
4117 | ASSERT_TRUE(listed.size() == created.size()); | |
4118 | if (listed2.size()) { | |
4119 | ASSERT_EQ(listed.size(), listed2.size()); | |
4120 | } | |
4121 | for (set<ghobject_t>::iterator i = listed.begin(); | |
4122 | i != listed.end(); | |
4123 | ++i) { | |
4124 | ASSERT_TRUE(created.count(*i)); | |
4125 | } | |
4126 | ||
4127 | for (set<ghobject_t>::iterator i = created.begin(); | |
4128 | i != created.end(); | |
4129 | ++i) { | |
4130 | ObjectStore::Transaction t; | |
4131 | t.remove(cid, *i); | |
11fdf7f2 | 4132 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4133 | ASSERT_EQ(r, 0); |
4134 | } | |
4135 | cerr << "cleaning up" << std::endl; | |
4136 | { | |
4137 | ObjectStore::Transaction t; | |
4138 | t.remove_collection(cid); | |
11fdf7f2 | 4139 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4140 | ASSERT_EQ(r, 0); |
4141 | } | |
4142 | } | |
4143 | ||
4144 | ||
4145 | class ObjectGenerator { | |
4146 | public: | |
4147 | virtual ghobject_t create_object(gen_type *gen) = 0; | |
4148 | virtual ~ObjectGenerator() {} | |
4149 | }; | |
4150 | ||
4151 | class MixedGenerator : public ObjectGenerator { | |
4152 | public: | |
4153 | unsigned seq; | |
4154 | int64_t poolid; | |
4155 | explicit MixedGenerator(int64_t p) : seq(0), poolid(p) {} | |
4156 | ghobject_t create_object(gen_type *gen) override { | |
4157 | char buf[100]; | |
4158 | snprintf(buf, sizeof(buf), "OBJ_%u", seq); | |
4159 | string name(buf); | |
4160 | if (seq % 2) { | |
4161 | for (unsigned i = 0; i < 300; ++i) { | |
4162 | name.push_back('a'); | |
4163 | } | |
4164 | } | |
4165 | ++seq; | |
4166 | return ghobject_t( | |
4167 | hobject_t( | |
4168 | name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), | |
4169 | (((seq / 1024) % 2) * 0xF00 ) + | |
4170 | (seq & 0xFF), | |
4171 | poolid, "")); | |
4172 | } | |
4173 | }; | |
4174 | ||
4175 | class SyntheticWorkloadState { | |
4176 | struct Object { | |
4177 | bufferlist data; | |
4178 | map<string, bufferlist> attrs; | |
4179 | }; | |
4180 | public: | |
4181 | static const unsigned max_in_flight = 16; | |
4182 | static const unsigned max_objects = 3000; | |
4183 | static const unsigned max_attr_size = 5; | |
4184 | static const unsigned max_attr_name_len = 100; | |
4185 | static const unsigned max_attr_value_len = 1024 * 64; | |
4186 | coll_t cid; | |
4187 | unsigned write_alignment; | |
4188 | unsigned max_object_len, max_write_len; | |
4189 | unsigned in_flight; | |
4190 | map<ghobject_t, Object> contents; | |
4191 | set<ghobject_t> available_objects; | |
f6b5b4d7 | 4192 | set<ghobject_t>::iterator next_available_object; |
7c673cae FG |
4193 | set<ghobject_t> in_flight_objects; |
4194 | ObjectGenerator *object_gen; | |
4195 | gen_type *rng; | |
4196 | ObjectStore *store; | |
11fdf7f2 | 4197 | ObjectStore::CollectionHandle ch; |
7c673cae | 4198 | |
9f95a23c TL |
4199 | ceph::mutex lock = ceph::make_mutex("State lock"); |
4200 | ceph::condition_variable cond; | |
7c673cae FG |
4201 | |
4202 | struct EnterExit { | |
4203 | const char *msg; | |
4204 | explicit EnterExit(const char *m) : msg(m) { | |
4205 | //cout << pthread_self() << " enter " << msg << std::endl; | |
4206 | } | |
4207 | ~EnterExit() { | |
4208 | //cout << pthread_self() << " exit " << msg << std::endl; | |
4209 | } | |
4210 | }; | |
4211 | ||
4212 | class C_SyntheticOnReadable : public Context { | |
4213 | public: | |
4214 | SyntheticWorkloadState *state; | |
4215 | ghobject_t hoid; | |
4216 | C_SyntheticOnReadable(SyntheticWorkloadState *state, ghobject_t hoid) | |
4217 | : state(state), hoid(hoid) {} | |
4218 | ||
4219 | void finish(int r) override { | |
9f95a23c | 4220 | std::lock_guard locker{state->lock}; |
7c673cae FG |
4221 | EnterExit ee("onreadable finish"); |
4222 | ASSERT_TRUE(state->in_flight_objects.count(hoid)); | |
4223 | ASSERT_EQ(r, 0); | |
4224 | state->in_flight_objects.erase(hoid); | |
4225 | if (state->contents.count(hoid)) | |
4226 | state->available_objects.insert(hoid); | |
4227 | --(state->in_flight); | |
9f95a23c | 4228 | state->cond.notify_all(); |
7c673cae FG |
4229 | |
4230 | bufferlist r2; | |
11fdf7f2 TL |
4231 | r = state->store->read(state->ch, hoid, 0, state->contents[hoid].data.length(), r2); |
4232 | ceph_assert(bl_eq(state->contents[hoid].data, r2)); | |
9f95a23c | 4233 | state->cond.notify_all(); |
7c673cae FG |
4234 | } |
4235 | }; | |
4236 | ||
4237 | class C_SyntheticOnStash : public Context { | |
4238 | public: | |
4239 | SyntheticWorkloadState *state; | |
4240 | ghobject_t oid, noid; | |
4241 | ||
4242 | C_SyntheticOnStash(SyntheticWorkloadState *state, | |
4243 | ghobject_t oid, ghobject_t noid) | |
4244 | : state(state), oid(oid), noid(noid) {} | |
4245 | ||
4246 | void finish(int r) override { | |
9f95a23c | 4247 | std::lock_guard locker{state->lock}; |
7c673cae FG |
4248 | EnterExit ee("stash finish"); |
4249 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
4250 | ASSERT_EQ(r, 0); | |
4251 | state->in_flight_objects.erase(oid); | |
4252 | if (state->contents.count(noid)) | |
4253 | state->available_objects.insert(noid); | |
4254 | --(state->in_flight); | |
4255 | bufferlist r2; | |
4256 | r = state->store->read( | |
11fdf7f2 | 4257 | state->ch, noid, 0, |
7c673cae | 4258 | state->contents[noid].data.length(), r2); |
11fdf7f2 | 4259 | ceph_assert(bl_eq(state->contents[noid].data, r2)); |
9f95a23c | 4260 | state->cond.notify_all(); |
7c673cae FG |
4261 | } |
4262 | }; | |
4263 | ||
4264 | class C_SyntheticOnClone : public Context { | |
4265 | public: | |
4266 | SyntheticWorkloadState *state; | |
4267 | ghobject_t oid, noid; | |
4268 | ||
4269 | C_SyntheticOnClone(SyntheticWorkloadState *state, | |
4270 | ghobject_t oid, ghobject_t noid) | |
4271 | : state(state), oid(oid), noid(noid) {} | |
4272 | ||
4273 | void finish(int r) override { | |
9f95a23c | 4274 | std::lock_guard locker{state->lock}; |
7c673cae FG |
4275 | EnterExit ee("clone finish"); |
4276 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
4277 | ASSERT_EQ(r, 0); | |
4278 | state->in_flight_objects.erase(oid); | |
4279 | if (state->contents.count(oid)) | |
4280 | state->available_objects.insert(oid); | |
4281 | if (state->contents.count(noid)) | |
4282 | state->available_objects.insert(noid); | |
4283 | --(state->in_flight); | |
4284 | bufferlist r2; | |
11fdf7f2 TL |
4285 | r = state->store->read(state->ch, noid, 0, state->contents[noid].data.length(), r2); |
4286 | ceph_assert(bl_eq(state->contents[noid].data, r2)); | |
9f95a23c | 4287 | state->cond.notify_all(); |
7c673cae FG |
4288 | } |
4289 | }; | |
4290 | ||
4291 | static void filled_byte_array(bufferlist& bl, size_t size) | |
4292 | { | |
4293 | static const char alphanum[] = "0123456789" | |
4294 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
4295 | "abcdefghijklmnopqrstuvwxyz"; | |
4296 | if (!size) { | |
4297 | return; | |
4298 | } | |
4299 | bufferptr bp(size); | |
4300 | for (unsigned int i = 0; i < size - 1; i++) { | |
4301 | // severely limit entropy so we can compress... | |
4302 | bp[i] = alphanum[rand() % 10]; //(sizeof(alphanum) - 1)]; | |
4303 | } | |
4304 | bp[size - 1] = '\0'; | |
4305 | ||
4306 | bl.append(bp); | |
4307 | } | |
4308 | ||
4309 | SyntheticWorkloadState(ObjectStore *store, | |
4310 | ObjectGenerator *gen, | |
4311 | gen_type *rng, | |
7c673cae FG |
4312 | coll_t cid, |
4313 | unsigned max_size, | |
4314 | unsigned max_write, | |
4315 | unsigned alignment) | |
4316 | : cid(cid), write_alignment(alignment), max_object_len(max_size), | |
f6b5b4d7 TL |
4317 | max_write_len(max_write), in_flight(0), |
4318 | next_available_object(available_objects.end()), | |
4319 | object_gen(gen), rng(rng), store(store) {} | |
7c673cae FG |
4320 | |
4321 | int init() { | |
4322 | ObjectStore::Transaction t; | |
11fdf7f2 | 4323 | ch = store->create_new_collection(cid); |
7c673cae | 4324 | t.create_collection(cid, 0); |
11fdf7f2 | 4325 | return queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4326 | } |
4327 | void shutdown() { | |
f6b5b4d7 | 4328 | ghobject_t next; |
7c673cae FG |
4329 | while (1) { |
4330 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
4331 | int r = collection_list(store, ch, next, ghobject_t::get_max(), 10, |
4332 | &objects, &next); | |
11fdf7f2 | 4333 | ceph_assert(r >= 0); |
f6b5b4d7 TL |
4334 | if (objects.size() == 0) |
4335 | break; | |
7c673cae | 4336 | ObjectStore::Transaction t; |
f6b5b4d7 | 4337 | std::map<std::string, ceph::buffer::list> attrset; |
7c673cae | 4338 | for (vector<ghobject_t>::iterator p = objects.begin(); |
f6b5b4d7 TL |
4339 | p != objects.end(); ++p) { |
4340 | t.remove(cid, *p); | |
7c673cae | 4341 | } |
11fdf7f2 | 4342 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4343 | } |
4344 | ObjectStore::Transaction t; | |
4345 | t.remove_collection(cid); | |
11fdf7f2 | 4346 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4347 | } |
4348 | void statfs(store_statfs_t& stat) { | |
4349 | store->statfs(&stat); | |
4350 | } | |
4351 | ||
9f95a23c TL |
4352 | ghobject_t get_uniform_random_object(std::unique_lock<ceph::mutex>& locker) { |
4353 | cond.wait(locker, [this] { | |
4354 | return in_flight < max_in_flight && !available_objects.empty(); | |
4355 | }); | |
7c673cae FG |
4356 | boost::uniform_int<> choose(0, available_objects.size() - 1); |
4357 | int index = choose(*rng); | |
4358 | set<ghobject_t>::iterator i = available_objects.begin(); | |
4359 | for ( ; index > 0; --index, ++i) ; | |
4360 | ghobject_t ret = *i; | |
4361 | return ret; | |
4362 | } | |
4363 | ||
f6b5b4d7 TL |
4364 | ghobject_t get_next_object(std::unique_lock<ceph::mutex>& locker) { |
4365 | cond.wait(locker, [this] { | |
4366 | return in_flight < max_in_flight && !available_objects.empty(); | |
4367 | }); | |
4368 | ||
4369 | if (next_available_object == available_objects.end()) { | |
4370 | next_available_object = available_objects.begin(); | |
4371 | } | |
4372 | ||
4373 | ghobject_t ret = *next_available_object; | |
4374 | ++next_available_object; | |
4375 | return ret; | |
4376 | } | |
4377 | ||
9f95a23c TL |
4378 | void wait_for_ready(std::unique_lock<ceph::mutex>& locker) { |
4379 | cond.wait(locker, [this] { return in_flight < max_in_flight; }); | |
7c673cae FG |
4380 | } |
4381 | ||
4382 | void wait_for_done() { | |
9f95a23c TL |
4383 | std::unique_lock locker{lock}; |
4384 | cond.wait(locker, [this] { return in_flight == 0; }); | |
7c673cae FG |
4385 | } |
4386 | ||
4387 | bool can_create() { | |
4388 | return (available_objects.size() + in_flight_objects.size()) < max_objects; | |
4389 | } | |
4390 | ||
4391 | bool can_unlink() { | |
4392 | return (available_objects.size() + in_flight_objects.size()) > 0; | |
4393 | } | |
4394 | ||
4395 | unsigned get_random_alloc_hints() { | |
4396 | unsigned f = 0; | |
4397 | { | |
4398 | boost::uniform_int<> u(0, 3); | |
4399 | switch (u(*rng)) { | |
4400 | case 1: | |
4401 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE; | |
4402 | break; | |
4403 | case 2: | |
4404 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE; | |
4405 | break; | |
4406 | } | |
4407 | } | |
4408 | { | |
4409 | boost::uniform_int<> u(0, 3); | |
4410 | switch (u(*rng)) { | |
4411 | case 1: | |
4412 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ; | |
4413 | break; | |
4414 | case 2: | |
4415 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ; | |
4416 | break; | |
4417 | } | |
4418 | } | |
4419 | { | |
4420 | // append_only, immutable | |
4421 | boost::uniform_int<> u(0, 4); | |
4422 | f |= u(*rng) << 4; | |
4423 | } | |
4424 | { | |
4425 | boost::uniform_int<> u(0, 3); | |
4426 | switch (u(*rng)) { | |
4427 | case 1: | |
4428 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED; | |
4429 | break; | |
4430 | case 2: | |
4431 | f |= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED; | |
4432 | break; | |
4433 | } | |
4434 | } | |
4435 | { | |
4436 | boost::uniform_int<> u(0, 3); | |
4437 | switch (u(*rng)) { | |
4438 | case 1: | |
4439 | f |= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE; | |
4440 | break; | |
4441 | case 2: | |
4442 | f |= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE; | |
4443 | break; | |
4444 | } | |
4445 | } | |
4446 | return f; | |
4447 | } | |
4448 | ||
4449 | int touch() { | |
9f95a23c | 4450 | std::unique_lock locker{lock}; |
7c673cae FG |
4451 | EnterExit ee("touch"); |
4452 | if (!can_create()) | |
4453 | return -ENOSPC; | |
9f95a23c | 4454 | wait_for_ready(locker); |
7c673cae FG |
4455 | ghobject_t new_obj = object_gen->create_object(rng); |
4456 | available_objects.erase(new_obj); | |
4457 | ObjectStore::Transaction t; | |
4458 | t.touch(cid, new_obj); | |
4459 | boost::uniform_int<> u(17, 22); | |
4460 | boost::uniform_int<> v(12, 17); | |
4461 | t.set_alloc_hint(cid, new_obj, | |
4462 | 1ull << u(*rng), | |
4463 | 1ull << v(*rng), | |
4464 | get_random_alloc_hints()); | |
4465 | ++in_flight; | |
4466 | in_flight_objects.insert(new_obj); | |
4467 | if (!contents.count(new_obj)) | |
4468 | contents[new_obj] = Object(); | |
11fdf7f2 TL |
4469 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4470 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4471 | return status; |
4472 | } | |
4473 | ||
4474 | int stash() { | |
9f95a23c | 4475 | std::unique_lock locker{lock}; |
7c673cae FG |
4476 | EnterExit ee("stash"); |
4477 | if (!can_unlink()) | |
4478 | return -ENOENT; | |
4479 | if (!can_create()) | |
4480 | return -ENOSPC; | |
9f95a23c | 4481 | wait_for_ready(locker); |
7c673cae FG |
4482 | |
4483 | ghobject_t old_obj; | |
4484 | int max = 20; | |
4485 | do { | |
9f95a23c | 4486 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4487 | } while (--max && !contents[old_obj].data.length()); |
4488 | available_objects.erase(old_obj); | |
4489 | ghobject_t new_obj = old_obj; | |
4490 | new_obj.generation++; | |
4491 | available_objects.erase(new_obj); | |
4492 | ||
4493 | ObjectStore::Transaction t; | |
4494 | t.collection_move_rename(cid, old_obj, cid, new_obj); | |
4495 | ++in_flight; | |
4496 | in_flight_objects.insert(old_obj); | |
4497 | ||
4498 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4499 | contents[new_obj].data = contents[old_obj].data; | |
4500 | contents.erase(old_obj); | |
11fdf7f2 TL |
4501 | t.register_on_applied(new C_SyntheticOnStash(this, old_obj, new_obj)); |
4502 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4503 | return status; |
4504 | } | |
4505 | ||
4506 | int clone() { | |
9f95a23c | 4507 | std::unique_lock locker{lock}; |
7c673cae FG |
4508 | EnterExit ee("clone"); |
4509 | if (!can_unlink()) | |
4510 | return -ENOENT; | |
4511 | if (!can_create()) | |
4512 | return -ENOSPC; | |
9f95a23c | 4513 | wait_for_ready(locker); |
7c673cae FG |
4514 | |
4515 | ghobject_t old_obj; | |
4516 | int max = 20; | |
4517 | do { | |
9f95a23c | 4518 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4519 | } while (--max && !contents[old_obj].data.length()); |
4520 | available_objects.erase(old_obj); | |
4521 | ghobject_t new_obj = object_gen->create_object(rng); | |
4522 | // make the hash match | |
4523 | new_obj.hobj.set_hash(old_obj.hobj.get_hash()); | |
4524 | available_objects.erase(new_obj); | |
4525 | ||
4526 | ObjectStore::Transaction t; | |
4527 | t.clone(cid, old_obj, new_obj); | |
4528 | ++in_flight; | |
4529 | in_flight_objects.insert(old_obj); | |
4530 | ||
4531 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4532 | contents[new_obj].data = contents[old_obj].data; | |
4533 | ||
11fdf7f2 TL |
4534 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4535 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4536 | return status; |
4537 | } | |
4538 | ||
4539 | int clone_range() { | |
9f95a23c | 4540 | std::unique_lock locker{lock}; |
7c673cae FG |
4541 | EnterExit ee("clone_range"); |
4542 | if (!can_unlink()) | |
4543 | return -ENOENT; | |
4544 | if (!can_create()) | |
4545 | return -ENOSPC; | |
9f95a23c | 4546 | wait_for_ready(locker); |
7c673cae FG |
4547 | |
4548 | ghobject_t old_obj; | |
4549 | int max = 20; | |
4550 | do { | |
9f95a23c | 4551 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4552 | } while (--max && !contents[old_obj].data.length()); |
4553 | bufferlist &srcdata = contents[old_obj].data; | |
4554 | if (srcdata.length() == 0) { | |
4555 | return 0; | |
4556 | } | |
4557 | available_objects.erase(old_obj); | |
9f95a23c | 4558 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4559 | available_objects.erase(new_obj); |
4560 | ||
4561 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4562 | boost::uniform_int<> u2(0, max_write_len); | |
4563 | uint64_t srcoff = u1(*rng); | |
4564 | // make src and dst offsets match, since that's what the osd does | |
4565 | uint64_t dstoff = srcoff; //u1(*rng); | |
4566 | uint64_t len = u2(*rng); | |
4567 | if (write_alignment) { | |
11fdf7f2 TL |
4568 | srcoff = round_up_to(srcoff, write_alignment); |
4569 | dstoff = round_up_to(dstoff, write_alignment); | |
4570 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4571 | } |
4572 | ||
4573 | if (srcoff > srcdata.length() - 1) { | |
4574 | srcoff = srcdata.length() - 1; | |
4575 | } | |
4576 | if (srcoff + len > srcdata.length()) { | |
4577 | len = srcdata.length() - srcoff; | |
4578 | } | |
4579 | if (0) | |
4580 | cout << __func__ << " from " << srcoff << "~" << len | |
4581 | << " (size " << srcdata.length() << ") to " | |
4582 | << dstoff << "~" << len << std::endl; | |
4583 | ||
4584 | ObjectStore::Transaction t; | |
4585 | t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff); | |
4586 | ++in_flight; | |
4587 | in_flight_objects.insert(old_obj); | |
4588 | ||
4589 | bufferlist bl; | |
4590 | if (srcoff < srcdata.length()) { | |
4591 | if (srcoff + len > srcdata.length()) { | |
4592 | bl.substr_of(srcdata, srcoff, srcdata.length() - srcoff); | |
4593 | } else { | |
4594 | bl.substr_of(srcdata, srcoff, len); | |
4595 | } | |
4596 | } | |
4597 | ||
4598 | bufferlist& dstdata = contents[new_obj].data; | |
4599 | if (dstdata.length() <= dstoff) { | |
4600 | if (bl.length() > 0) { | |
4601 | dstdata.append_zero(dstoff - dstdata.length()); | |
4602 | dstdata.append(bl); | |
4603 | } | |
4604 | } else { | |
4605 | bufferlist value; | |
11fdf7f2 | 4606 | ceph_assert(dstdata.length() > dstoff); |
9f95a23c | 4607 | dstdata.cbegin().copy(dstoff, value); |
7c673cae FG |
4608 | value.append(bl); |
4609 | if (value.length() < dstdata.length()) | |
9f95a23c TL |
4610 | dstdata.cbegin(value.length()).copy( |
4611 | dstdata.length() - value.length(), value); | |
7c673cae FG |
4612 | value.swap(dstdata); |
4613 | } | |
4614 | ||
11fdf7f2 TL |
4615 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4616 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4617 | return status; |
4618 | } | |
4619 | ||
4620 | ||
4621 | int write() { | |
9f95a23c | 4622 | std::unique_lock locker{lock}; |
7c673cae FG |
4623 | EnterExit ee("write"); |
4624 | if (!can_unlink()) | |
4625 | return -ENOENT; | |
9f95a23c | 4626 | wait_for_ready(locker); |
7c673cae | 4627 | |
9f95a23c | 4628 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4629 | available_objects.erase(new_obj); |
4630 | ObjectStore::Transaction t; | |
4631 | ||
4632 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4633 | boost::uniform_int<> u2(0, max_write_len); | |
4634 | uint64_t offset = u1(*rng); | |
4635 | uint64_t len = u2(*rng); | |
4636 | bufferlist bl; | |
4637 | if (write_alignment) { | |
11fdf7f2 TL |
4638 | offset = round_up_to(offset, write_alignment); |
4639 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4640 | } |
4641 | ||
4642 | filled_byte_array(bl, len); | |
4643 | ||
4644 | bufferlist& data = contents[new_obj].data; | |
4645 | if (data.length() <= offset) { | |
4646 | if (len > 0) { | |
4647 | data.append_zero(offset-data.length()); | |
4648 | data.append(bl); | |
4649 | } | |
4650 | } else { | |
4651 | bufferlist value; | |
11fdf7f2 | 4652 | ceph_assert(data.length() > offset); |
9f95a23c | 4653 | data.cbegin().copy(offset, value); |
7c673cae FG |
4654 | value.append(bl); |
4655 | if (value.length() < data.length()) | |
9f95a23c TL |
4656 | data.cbegin(value.length()).copy( |
4657 | data.length()-value.length(), value); | |
7c673cae FG |
4658 | value.swap(data); |
4659 | } | |
4660 | ||
4661 | t.write(cid, new_obj, offset, len, bl); | |
4662 | ++in_flight; | |
4663 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4664 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4665 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4666 | return status; |
4667 | } | |
4668 | ||
4669 | int truncate() { | |
9f95a23c | 4670 | std::unique_lock locker{lock}; |
7c673cae FG |
4671 | EnterExit ee("truncate"); |
4672 | if (!can_unlink()) | |
4673 | return -ENOENT; | |
9f95a23c | 4674 | wait_for_ready(locker); |
7c673cae | 4675 | |
9f95a23c | 4676 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4677 | available_objects.erase(obj); |
4678 | ObjectStore::Transaction t; | |
4679 | ||
4680 | boost::uniform_int<> choose(0, max_object_len); | |
4681 | size_t len = choose(*rng); | |
4682 | if (write_alignment) { | |
11fdf7f2 | 4683 | len = round_up_to(len, write_alignment); |
7c673cae FG |
4684 | } |
4685 | ||
4686 | t.truncate(cid, obj, len); | |
4687 | ++in_flight; | |
4688 | in_flight_objects.insert(obj); | |
4689 | bufferlist& data = contents[obj].data; | |
4690 | if (data.length() <= len) { | |
4691 | data.append_zero(len - data.length()); | |
4692 | } else { | |
4693 | bufferlist bl; | |
9f95a23c | 4694 | data.cbegin().copy(len, bl); |
7c673cae FG |
4695 | bl.swap(data); |
4696 | } | |
4697 | ||
11fdf7f2 TL |
4698 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4699 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4700 | return status; |
4701 | } | |
4702 | ||
4703 | int zero() { | |
9f95a23c | 4704 | std::unique_lock locker{lock}; |
7c673cae FG |
4705 | EnterExit ee("zero"); |
4706 | if (!can_unlink()) | |
4707 | return -ENOENT; | |
9f95a23c | 4708 | wait_for_ready(locker); |
7c673cae | 4709 | |
9f95a23c | 4710 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4711 | available_objects.erase(new_obj); |
4712 | ObjectStore::Transaction t; | |
4713 | ||
4714 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4715 | boost::uniform_int<> u2(0, max_write_len); | |
4716 | uint64_t offset = u1(*rng); | |
4717 | uint64_t len = u2(*rng); | |
4718 | if (write_alignment) { | |
11fdf7f2 TL |
4719 | offset = round_up_to(offset, write_alignment); |
4720 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4721 | } |
4722 | ||
b32b8144 FG |
4723 | if (len > 0) { |
4724 | auto& data = contents[new_obj].data; | |
4725 | if (data.length() < offset + len) { | |
4726 | data.append_zero(offset+len-data.length()); | |
4727 | } | |
4728 | bufferlist n; | |
4729 | n.substr_of(data, 0, offset); | |
4730 | n.append_zero(len); | |
4731 | if (data.length() > offset + len) | |
9f95a23c | 4732 | data.cbegin(offset + len).copy(data.length() - offset - len, n); |
b32b8144 | 4733 | data.swap(n); |
7c673cae | 4734 | } |
7c673cae FG |
4735 | |
4736 | t.zero(cid, new_obj, offset, len); | |
4737 | ++in_flight; | |
4738 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4739 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4740 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4741 | return status; |
4742 | } | |
4743 | ||
4744 | void read() { | |
4745 | EnterExit ee("read"); | |
4746 | boost::uniform_int<> u1(0, max_object_len/2); | |
4747 | boost::uniform_int<> u2(0, max_object_len); | |
4748 | uint64_t offset = u1(*rng); | |
4749 | uint64_t len = u2(*rng); | |
4750 | if (offset > len) | |
4751 | swap(offset, len); | |
4752 | ||
4753 | ghobject_t obj; | |
4754 | bufferlist expected; | |
4755 | int r; | |
4756 | { | |
9f95a23c | 4757 | std::unique_lock locker{lock}; |
7c673cae FG |
4758 | EnterExit ee("read locked"); |
4759 | if (!can_unlink()) | |
4760 | return ; | |
9f95a23c | 4761 | wait_for_ready(locker); |
7c673cae | 4762 | |
9f95a23c | 4763 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4764 | expected = contents[obj].data; |
4765 | } | |
4766 | bufferlist bl, result; | |
4767 | if (0) cout << " obj " << obj | |
4768 | << " size " << expected.length() | |
4769 | << " offset " << offset | |
4770 | << " len " << len << std::endl; | |
11fdf7f2 | 4771 | r = store->read(ch, obj, offset, len, result); |
7c673cae FG |
4772 | if (offset >= expected.length()) { |
4773 | ASSERT_EQ(r, 0); | |
4774 | } else { | |
4775 | size_t max_len = expected.length() - offset; | |
4776 | if (len > max_len) | |
4777 | len = max_len; | |
11fdf7f2 | 4778 | ceph_assert(len == result.length()); |
7c673cae | 4779 | ASSERT_EQ(len, result.length()); |
9f95a23c | 4780 | expected.cbegin(offset).copy(len, bl); |
7c673cae FG |
4781 | ASSERT_EQ(r, (int)len); |
4782 | ASSERT_TRUE(bl_eq(bl, result)); | |
4783 | } | |
4784 | } | |
4785 | ||
4786 | int setattrs() { | |
9f95a23c | 4787 | std::unique_lock locker{lock}; |
7c673cae FG |
4788 | EnterExit ee("setattrs"); |
4789 | if (!can_unlink()) | |
4790 | return -ENOENT; | |
9f95a23c | 4791 | wait_for_ready(locker); |
7c673cae | 4792 | |
9f95a23c | 4793 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4794 | available_objects.erase(obj); |
4795 | ObjectStore::Transaction t; | |
4796 | ||
4797 | boost::uniform_int<> u0(1, max_attr_size); | |
4798 | boost::uniform_int<> u1(4, max_attr_name_len); | |
4799 | boost::uniform_int<> u2(4, max_attr_value_len); | |
4800 | boost::uniform_int<> u3(0, 100); | |
4801 | uint64_t size = u0(*rng); | |
4802 | uint64_t name_len; | |
20effc67 | 4803 | map<string, bufferlist, less<>> attrs; |
7c673cae FG |
4804 | set<string> keys; |
4805 | for (map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4806 | it != contents[obj].attrs.end(); ++it) | |
4807 | keys.insert(it->first); | |
4808 | ||
4809 | while (size--) { | |
4810 | bufferlist name, value; | |
4811 | uint64_t get_exist = u3(*rng); | |
4812 | uint64_t value_len = u2(*rng); | |
4813 | filled_byte_array(value, value_len); | |
4814 | if (get_exist < 50 && keys.size()) { | |
4815 | set<string>::iterator k = keys.begin(); | |
4816 | attrs[*k] = value; | |
4817 | contents[obj].attrs[*k] = value; | |
4818 | keys.erase(k); | |
4819 | } else { | |
4820 | name_len = u1(*rng); | |
4821 | filled_byte_array(name, name_len); | |
4822 | attrs[name.c_str()] = value; | |
4823 | contents[obj].attrs[name.c_str()] = value; | |
4824 | } | |
4825 | } | |
4826 | t.setattrs(cid, obj, attrs); | |
4827 | ++in_flight; | |
4828 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4829 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4830 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4831 | return status; |
4832 | } | |
4833 | ||
f6b5b4d7 TL |
4834 | int set_fixed_attrs(size_t entries, size_t key_size, size_t val_size) { |
4835 | std::unique_lock locker{ lock }; | |
4836 | EnterExit ee("setattrs"); | |
4837 | if (!can_unlink()) | |
4838 | return -ENOENT; | |
4839 | wait_for_ready(locker); | |
4840 | ||
4841 | ghobject_t obj = get_next_object(locker); | |
4842 | available_objects.erase(obj); | |
4843 | ObjectStore::Transaction t; | |
4844 | ||
20effc67 | 4845 | map<string, bufferlist, less<>> attrs; |
f6b5b4d7 TL |
4846 | set<string> keys; |
4847 | ||
4848 | while (entries--) { | |
4849 | bufferlist name, value; | |
4850 | filled_byte_array(value, val_size); | |
4851 | filled_byte_array(name, key_size); | |
4852 | attrs[name.c_str()] = value; | |
4853 | contents[obj].attrs[name.c_str()] = value; | |
4854 | } | |
4855 | t.setattrs(cid, obj, attrs); | |
4856 | ++in_flight; | |
4857 | in_flight_objects.insert(obj); | |
4858 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); | |
4859 | int status = store->queue_transaction(ch, std::move(t)); | |
4860 | return status; | |
4861 | } | |
4862 | ||
7c673cae FG |
4863 | void getattrs() { |
4864 | EnterExit ee("getattrs"); | |
4865 | ghobject_t obj; | |
4866 | map<string, bufferlist> expected; | |
4867 | { | |
9f95a23c | 4868 | std::unique_lock locker{lock}; |
7c673cae FG |
4869 | EnterExit ee("getattrs locked"); |
4870 | if (!can_unlink()) | |
4871 | return ; | |
9f95a23c | 4872 | wait_for_ready(locker); |
7c673cae FG |
4873 | |
4874 | int retry = 10; | |
4875 | do { | |
9f95a23c | 4876 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4877 | if (!--retry) |
4878 | return ; | |
4879 | } while (contents[obj].attrs.empty()); | |
4880 | expected = contents[obj].attrs; | |
4881 | } | |
20effc67 | 4882 | map<string, bufferlist, less<>> attrs; |
11fdf7f2 | 4883 | int r = store->getattrs(ch, obj, attrs); |
7c673cae FG |
4884 | ASSERT_TRUE(r == 0); |
4885 | ASSERT_TRUE(attrs.size() == expected.size()); | |
4886 | for (map<string, bufferlist>::iterator it = expected.begin(); | |
4887 | it != expected.end(); ++it) { | |
4888 | ASSERT_TRUE(bl_eq(attrs[it->first], it->second)); | |
4889 | } | |
4890 | } | |
4891 | ||
4892 | void getattr() { | |
4893 | EnterExit ee("getattr"); | |
4894 | ghobject_t obj; | |
4895 | int r; | |
4896 | int retry; | |
4897 | map<string, bufferlist> expected; | |
4898 | { | |
9f95a23c | 4899 | std::unique_lock locker{lock}; |
7c673cae FG |
4900 | EnterExit ee("getattr locked"); |
4901 | if (!can_unlink()) | |
4902 | return ; | |
9f95a23c | 4903 | wait_for_ready(locker); |
7c673cae FG |
4904 | |
4905 | retry = 10; | |
4906 | do { | |
9f95a23c | 4907 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4908 | if (!--retry) |
4909 | return ; | |
4910 | } while (contents[obj].attrs.empty()); | |
4911 | expected = contents[obj].attrs; | |
4912 | } | |
4913 | boost::uniform_int<> u(0, expected.size()-1); | |
4914 | retry = u(*rng); | |
4915 | map<string, bufferlist>::iterator it = expected.begin(); | |
4916 | while (retry) { | |
4917 | retry--; | |
4918 | ++it; | |
4919 | } | |
4920 | ||
4921 | bufferlist bl; | |
11fdf7f2 | 4922 | r = store->getattr(ch, obj, it->first, bl); |
7c673cae FG |
4923 | ASSERT_EQ(r, 0); |
4924 | ASSERT_TRUE(bl_eq(it->second, bl)); | |
4925 | } | |
4926 | ||
4927 | int rmattr() { | |
9f95a23c | 4928 | std::unique_lock locker{lock}; |
7c673cae FG |
4929 | EnterExit ee("rmattr"); |
4930 | if (!can_unlink()) | |
4931 | return -ENOENT; | |
9f95a23c | 4932 | wait_for_ready(locker); |
7c673cae FG |
4933 | |
4934 | ghobject_t obj; | |
4935 | int retry = 10; | |
4936 | do { | |
9f95a23c | 4937 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4938 | if (!--retry) |
4939 | return 0; | |
4940 | } while (contents[obj].attrs.empty()); | |
4941 | ||
4942 | boost::uniform_int<> u(0, contents[obj].attrs.size()-1); | |
4943 | retry = u(*rng); | |
4944 | map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4945 | while (retry) { | |
4946 | retry--; | |
4947 | ++it; | |
4948 | } | |
4949 | ||
4950 | available_objects.erase(obj); | |
4951 | ObjectStore::Transaction t; | |
4952 | t.rmattr(cid, obj, it->first); | |
4953 | ||
4954 | contents[obj].attrs.erase(it->first); | |
4955 | ++in_flight; | |
4956 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4957 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4958 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4959 | return status; |
4960 | } | |
4961 | ||
4962 | void fsck(bool deep) { | |
9f95a23c | 4963 | std::unique_lock locker{lock}; |
7c673cae | 4964 | EnterExit ee("fsck"); |
9f95a23c | 4965 | cond.wait(locker, [this] { return in_flight == 0; }); |
11fdf7f2 | 4966 | ch.reset(); |
7c673cae FG |
4967 | store->umount(); |
4968 | int r = store->fsck(deep); | |
11fdf7f2 | 4969 | ceph_assert(r == 0 || r == -EOPNOTSUPP); |
7c673cae | 4970 | store->mount(); |
11fdf7f2 | 4971 | ch = store->open_collection(cid); |
7c673cae FG |
4972 | } |
4973 | ||
4974 | void scan() { | |
9f95a23c | 4975 | std::unique_lock locker{lock}; |
7c673cae | 4976 | EnterExit ee("scan"); |
9f95a23c | 4977 | cond.wait(locker, [this] { return in_flight == 0; }); |
7c673cae FG |
4978 | vector<ghobject_t> objects; |
4979 | set<ghobject_t> objects_set, objects_set2; | |
4980 | ghobject_t next, current; | |
4981 | while (1) { | |
4982 | //cerr << "scanning..." << std::endl; | |
f91f0fd5 TL |
4983 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 100, |
4984 | &objects, &next); | |
7c673cae FG |
4985 | ASSERT_EQ(r, 0); |
4986 | ASSERT_TRUE(sorted(objects)); | |
4987 | objects_set.insert(objects.begin(), objects.end()); | |
4988 | objects.clear(); | |
4989 | if (next.is_max()) break; | |
4990 | current = next; | |
4991 | } | |
4992 | if (objects_set.size() != available_objects.size()) { | |
4993 | for (set<ghobject_t>::iterator p = objects_set.begin(); | |
4994 | p != objects_set.end(); | |
4995 | ++p) | |
4996 | if (available_objects.count(*p) == 0) { | |
4997 | cerr << "+ " << *p << std::endl; | |
4998 | ceph_abort(); | |
4999 | } | |
5000 | for (set<ghobject_t>::iterator p = available_objects.begin(); | |
5001 | p != available_objects.end(); | |
5002 | ++p) | |
5003 | if (objects_set.count(*p) == 0) | |
5004 | cerr << "- " << *p << std::endl; | |
5005 | //cerr << " objects_set: " << objects_set << std::endl; | |
5006 | //cerr << " available_set: " << available_objects << std::endl; | |
11fdf7f2 | 5007 | ceph_abort_msg("badness"); |
7c673cae FG |
5008 | } |
5009 | ||
5010 | ASSERT_EQ(objects_set.size(), available_objects.size()); | |
5011 | for (set<ghobject_t>::iterator i = objects_set.begin(); | |
5012 | i != objects_set.end(); | |
5013 | ++i) { | |
5014 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
5015 | } | |
5016 | ||
f91f0fd5 TL |
5017 | int r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), |
5018 | INT_MAX, &objects, 0); | |
7c673cae FG |
5019 | ASSERT_EQ(r, 0); |
5020 | objects_set2.insert(objects.begin(), objects.end()); | |
5021 | ASSERT_EQ(objects_set2.size(), available_objects.size()); | |
5022 | for (set<ghobject_t>::iterator i = objects_set2.begin(); | |
5023 | i != objects_set2.end(); | |
5024 | ++i) { | |
5025 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
5026 | if (available_objects.count(*i) == 0) { | |
5027 | cerr << "+ " << *i << std::endl; | |
5028 | } | |
5029 | } | |
5030 | } | |
5031 | ||
5032 | void stat() { | |
5033 | EnterExit ee("stat"); | |
5034 | ghobject_t hoid; | |
5035 | uint64_t expected; | |
5036 | { | |
9f95a23c | 5037 | std::unique_lock locker{lock}; |
7c673cae FG |
5038 | EnterExit ee("stat lock1"); |
5039 | if (!can_unlink()) | |
5040 | return ; | |
9f95a23c | 5041 | hoid = get_uniform_random_object(locker); |
7c673cae FG |
5042 | in_flight_objects.insert(hoid); |
5043 | available_objects.erase(hoid); | |
5044 | ++in_flight; | |
5045 | expected = contents[hoid].data.length(); | |
5046 | } | |
5047 | struct stat buf; | |
11fdf7f2 | 5048 | int r = store->stat(ch, hoid, &buf); |
7c673cae | 5049 | ASSERT_EQ(0, r); |
11fdf7f2 | 5050 | ceph_assert((uint64_t)buf.st_size == expected); |
7c673cae FG |
5051 | ASSERT_TRUE((uint64_t)buf.st_size == expected); |
5052 | { | |
9f95a23c | 5053 | std::lock_guard locker{lock}; |
7c673cae FG |
5054 | EnterExit ee("stat lock2"); |
5055 | --in_flight; | |
9f95a23c | 5056 | cond.notify_all(); |
7c673cae FG |
5057 | in_flight_objects.erase(hoid); |
5058 | available_objects.insert(hoid); | |
5059 | } | |
5060 | } | |
5061 | ||
5062 | int unlink() { | |
9f95a23c | 5063 | std::unique_lock locker{lock}; |
7c673cae FG |
5064 | EnterExit ee("unlink"); |
5065 | if (!can_unlink()) | |
5066 | return -ENOENT; | |
9f95a23c | 5067 | ghobject_t to_remove = get_uniform_random_object(locker); |
7c673cae FG |
5068 | ObjectStore::Transaction t; |
5069 | t.remove(cid, to_remove); | |
5070 | ++in_flight; | |
5071 | available_objects.erase(to_remove); | |
5072 | in_flight_objects.insert(to_remove); | |
5073 | contents.erase(to_remove); | |
11fdf7f2 TL |
5074 | t.register_on_applied(new C_SyntheticOnReadable(this, to_remove)); |
5075 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
5076 | return status; |
5077 | } | |
5078 | ||
5079 | void print_internal_state() { | |
9f95a23c | 5080 | std::lock_guard locker{lock}; |
7c673cae FG |
5081 | cerr << "available_objects: " << available_objects.size() |
5082 | << " in_flight_objects: " << in_flight_objects.size() | |
5083 | << " total objects: " << in_flight_objects.size() + available_objects.size() | |
5084 | << " in_flight " << in_flight << std::endl; | |
5085 | } | |
5086 | }; | |
5087 | ||
5088 | ||
11fdf7f2 | 5089 | void StoreTest::doSyntheticTest( |
7c673cae FG |
5090 | int num_ops, |
5091 | uint64_t max_obj, uint64_t max_wr, uint64_t align) | |
5092 | { | |
7c673cae FG |
5093 | MixedGenerator gen(555); |
5094 | gen_type rng(time(NULL)); | |
5095 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
5096 | ||
11fdf7f2 TL |
5097 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); |
5098 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
5099 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 5100 | |
11fdf7f2 | 5101 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, |
7c673cae FG |
5102 | max_obj, max_wr, align); |
5103 | test_obj.init(); | |
5104 | for (int i = 0; i < num_ops/10; ++i) { | |
5105 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
5106 | test_obj.touch(); | |
5107 | } | |
5108 | for (int i = 0; i < num_ops; ++i) { | |
5109 | if (!(i % 1000)) { | |
5110 | cerr << "Op " << i << std::endl; | |
5111 | test_obj.print_internal_state(); | |
5112 | } | |
5113 | boost::uniform_int<> true_false(0, 999); | |
5114 | int val = true_false(rng); | |
5115 | if (val > 998) { | |
5116 | test_obj.fsck(true); | |
5117 | } else if (val > 997) { | |
5118 | test_obj.fsck(false); | |
5119 | } else if (val > 970) { | |
5120 | test_obj.scan(); | |
5121 | } else if (val > 950) { | |
5122 | test_obj.stat(); | |
5123 | } else if (val > 850) { | |
5124 | test_obj.zero(); | |
5125 | } else if (val > 800) { | |
5126 | test_obj.unlink(); | |
5127 | } else if (val > 550) { | |
5128 | test_obj.write(); | |
5129 | } else if (val > 500) { | |
5130 | test_obj.clone(); | |
5131 | } else if (val > 450) { | |
5132 | test_obj.clone_range(); | |
5133 | } else if (val > 300) { | |
5134 | test_obj.stash(); | |
5135 | } else if (val > 100) { | |
5136 | test_obj.read(); | |
5137 | } else { | |
5138 | test_obj.truncate(); | |
5139 | } | |
5140 | } | |
5141 | test_obj.wait_for_done(); | |
5142 | test_obj.shutdown(); | |
7c673cae FG |
5143 | } |
5144 | ||
5145 | TEST_P(StoreTest, Synthetic) { | |
11fdf7f2 | 5146 | doSyntheticTest(10000, 400*1024, 40*1024, 0); |
7c673cae FG |
5147 | } |
5148 | ||
9f95a23c | 5149 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
5150 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixSharding) { |
5151 | if (string(GetParam()) != "bluestore") | |
5152 | return; | |
5153 | ||
5154 | const char *m[][10] = { | |
5155 | { "bluestore_min_alloc_size", "4096", 0 }, // must be the first! | |
5156 | { "num_ops", "50000", 0 }, | |
5157 | { "max_write", "65536", 0 }, | |
5158 | { "max_size", "262144", 0 }, | |
5159 | { "alignment", "4096", 0 }, | |
5160 | { "bluestore_max_blob_size", "65536", 0 }, | |
5161 | { "bluestore_extent_map_shard_min_size", "60", 0 }, | |
5162 | { "bluestore_extent_map_shard_max_size", "300", 0 }, | |
5163 | { "bluestore_extent_map_shard_target_size", "150", 0 }, | |
5164 | { "bluestore_default_buffered_read", "true", 0 }, | |
5165 | { "bluestore_default_buffered_write", "true", 0 }, | |
5166 | { 0 }, | |
5167 | }; | |
11fdf7f2 | 5168 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5169 | } |
5170 | ||
5171 | TEST_P(StoreTestSpecificAUSize, ZipperPatternSharded) { | |
5172 | if(string(GetParam()) != "bluestore") | |
5173 | return; | |
5174 | StartDeferred(4096); | |
5175 | ||
5176 | int r; | |
7c673cae FG |
5177 | coll_t cid; |
5178 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 5179 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5180 | { |
5181 | ObjectStore::Transaction t; | |
5182 | t.create_collection(cid, 0); | |
5183 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 5184 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5185 | ASSERT_EQ(r, 0); |
5186 | } | |
5187 | bufferlist bl; | |
5188 | int len = 4096; | |
5189 | bufferptr bp(len); | |
5190 | bp.zero(); | |
5191 | bl.append(bp); | |
5192 | for (int i=0; i<1000; ++i) { | |
5193 | ObjectStore::Transaction t; | |
5194 | t.write(cid, a, i*2*len, len, bl, 0); | |
11fdf7f2 | 5195 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5196 | ASSERT_EQ(r, 0); |
5197 | } | |
5198 | for (int i=0; i<1000; ++i) { | |
5199 | ObjectStore::Transaction t; | |
5200 | t.write(cid, a, i*2*len + 1, len, bl, 0); | |
11fdf7f2 | 5201 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5202 | ASSERT_EQ(r, 0); |
5203 | } | |
5204 | { | |
5205 | ObjectStore::Transaction t; | |
5206 | t.remove(cid, a); | |
5207 | t.remove_collection(cid); | |
5208 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 5209 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5210 | ASSERT_EQ(r, 0); |
5211 | } | |
5212 | } | |
5213 | ||
5214 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumAlgorithm) { | |
5215 | if (string(GetParam()) != "bluestore") | |
5216 | return; | |
5217 | ||
5218 | const char *m[][10] = { | |
5219 | { "bluestore_min_alloc_size", "65536", 0 }, // must be the first! | |
5220 | { "max_write", "65536", 0 }, | |
5221 | { "max_size", "1048576", 0 }, | |
5222 | { "alignment", "16", 0 }, | |
5223 | { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", | |
5224 | "xxhash64", "none", 0 }, | |
5225 | { "bluestore_default_buffered_write", "false", 0 }, | |
5226 | { 0 }, | |
5227 | }; | |
11fdf7f2 | 5228 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5229 | } |
5230 | ||
5231 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumVsCompression) { | |
5232 | if (string(GetParam()) != "bluestore") | |
5233 | return; | |
5234 | ||
5235 | const char *m[][10] = { | |
5236 | { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first! | |
5237 | { "max_write", "131072", 0 }, | |
5238 | { "max_size", "262144", 0 }, | |
5239 | { "alignment", "512", 0 }, | |
5240 | { "bluestore_compression_mode", "force", 0}, | |
5241 | { "bluestore_compression_algorithm", "snappy", "zlib", 0 }, | |
5242 | { "bluestore_csum_type", "crc32c", 0 }, | |
5243 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
5244 | { "bluestore_default_buffered_write", "true", "false", 0 }, | |
5245 | { "bluestore_sync_submit_transaction", "false", 0 }, | |
5246 | { 0 }, | |
5247 | }; | |
11fdf7f2 | 5248 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5249 | } |
5250 | ||
5251 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompression) { | |
5252 | if (string(GetParam()) != "bluestore") | |
5253 | return; | |
5254 | ||
5255 | const char *m[][10] = { | |
5256 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
5257 | { "max_write", "1048576", 0 }, | |
5258 | { "max_size", "4194304", 0 }, | |
5259 | { "alignment", "65536", 0 }, | |
5260 | { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0}, | |
5261 | { "bluestore_default_buffered_write", "false", 0 }, | |
5262 | { "bluestore_sync_submit_transaction", "true", 0 }, | |
5263 | { 0 }, | |
5264 | }; | |
11fdf7f2 | 5265 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5266 | } |
5267 | ||
5268 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompressionAlgorithm) { | |
5269 | if (string(GetParam()) != "bluestore") | |
5270 | return; | |
5271 | ||
5272 | const char *m[][10] = { | |
5273 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
5274 | { "max_write", "1048576", 0 }, | |
5275 | { "max_size", "4194304", 0 }, | |
5276 | { "alignment", "65536", 0 }, | |
5277 | { "bluestore_compression_algorithm", "zlib", "snappy", 0 }, | |
5278 | { "bluestore_compression_mode", "force", 0 }, | |
5279 | { "bluestore_default_buffered_write", "false", 0 }, | |
5280 | { 0 }, | |
5281 | }; | |
11fdf7f2 | 5282 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5283 | } |
5284 | ||
5285 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixNoCsum) { | |
5286 | if (string(GetParam()) != "bluestore") | |
5287 | return; | |
5288 | ||
5289 | const char *m[][10] = { | |
5290 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
5291 | { "max_write", "65536", 0 }, | |
5292 | { "max_size", "1048576", 0 }, | |
5293 | { "alignment", "512", 0 }, | |
5294 | { "bluestore_max_blob_size", "262144", 0 }, | |
5295 | { "bluestore_compression_mode", "force", "none", 0}, | |
5296 | { "bluestore_csum_type", "none", 0}, | |
5297 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
5298 | { "bluestore_default_buffered_write", "true", 0 }, | |
5299 | { "bluestore_sync_submit_transaction", "true", "false", 0 }, | |
5300 | { 0 }, | |
5301 | }; | |
11fdf7f2 | 5302 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
5303 | } |
5304 | ||
5305 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixPreferDeferred) { | |
5306 | if (string(GetParam()) != "bluestore") | |
5307 | return; | |
5308 | ||
5309 | const char *m[][10] = { | |
5310 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
5311 | { "max_write", "65536", 0 }, | |
5312 | { "max_size", "1048576", 0 }, | |
5313 | { "alignment", "512", 0 }, | |
5314 | { "bluestore_max_blob_size", "262144", 0 }, | |
5315 | { "bluestore_compression_mode", "force", "none", 0}, | |
5316 | { "bluestore_prefer_deferred_size", "32768", "0", 0}, | |
5317 | { 0 }, | |
5318 | }; | |
11fdf7f2 | 5319 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae | 5320 | } |
11fdf7f2 | 5321 | #endif // WITH_BLUESTORE |
7c673cae FG |
5322 | |
5323 | TEST_P(StoreTest, AttrSynthetic) { | |
7c673cae FG |
5324 | MixedGenerator gen(447); |
5325 | gen_type rng(time(NULL)); | |
5326 | coll_t cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD)); | |
5327 | ||
11fdf7f2 | 5328 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, 40*1024, 4*1024, 0); |
7c673cae FG |
5329 | test_obj.init(); |
5330 | for (int i = 0; i < 500; ++i) { | |
5331 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; | |
5332 | test_obj.touch(); | |
5333 | } | |
5334 | for (int i = 0; i < 1000; ++i) { | |
5335 | if (!(i % 100)) { | |
5336 | cerr << "Op " << i << std::endl; | |
5337 | test_obj.print_internal_state(); | |
5338 | } | |
5339 | boost::uniform_int<> true_false(0, 99); | |
5340 | int val = true_false(rng); | |
5341 | if (val > 97) { | |
5342 | test_obj.scan(); | |
5343 | } else if (val > 93) { | |
5344 | test_obj.stat(); | |
5345 | } else if (val > 75) { | |
5346 | test_obj.rmattr(); | |
5347 | } else if (val > 47) { | |
5348 | test_obj.setattrs(); | |
5349 | } else if (val > 45) { | |
5350 | test_obj.clone(); | |
5351 | } else if (val > 37) { | |
5352 | test_obj.stash(); | |
5353 | } else if (val > 30) { | |
5354 | test_obj.getattrs(); | |
5355 | } else { | |
5356 | test_obj.getattr(); | |
5357 | } | |
5358 | } | |
5359 | test_obj.wait_for_done(); | |
5360 | test_obj.shutdown(); | |
5361 | } | |
5362 | ||
5363 | TEST_P(StoreTest, HashCollisionTest) { | |
7c673cae FG |
5364 | int64_t poolid = 11; |
5365 | coll_t cid(spg_t(pg_t(0,poolid),shard_id_t::NO_SHARD)); | |
5366 | int r; | |
11fdf7f2 | 5367 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5368 | { |
5369 | ObjectStore::Transaction t; | |
5370 | t.create_collection(cid, 0); | |
11fdf7f2 | 5371 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5372 | ASSERT_EQ(r, 0); |
5373 | } | |
5374 | string base = ""; | |
5375 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
5376 | set<ghobject_t> created; | |
5377 | for (int n = 0; n < 10; ++n) { | |
5378 | char nbuf[100]; | |
5379 | sprintf(nbuf, "n%d", n); | |
5380 | for (int i = 0; i < 1000; ++i) { | |
5381 | char buf[100]; | |
5382 | sprintf(buf, "%d", i); | |
5383 | if (!(i % 100)) { | |
5384 | cerr << "Object n" << n << " "<< i << std::endl; | |
5385 | } | |
5386 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, poolid, string(nbuf))); | |
5387 | { | |
5388 | ObjectStore::Transaction t; | |
5389 | t.touch(cid, hoid); | |
11fdf7f2 | 5390 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5391 | ASSERT_EQ(r, 0); |
5392 | } | |
5393 | created.insert(hoid); | |
5394 | } | |
5395 | } | |
5396 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
5397 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5398 | &objects, 0); | |
7c673cae FG |
5399 | ASSERT_EQ(r, 0); |
5400 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5401 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5402 | ASSERT_TRUE(listed.size() == created.size()); | |
5403 | objects.clear(); | |
5404 | listed.clear(); | |
5405 | ghobject_t current, next; | |
5406 | while (1) { | |
f91f0fd5 TL |
5407 | r = collection_list(store, ch, current, ghobject_t::get_max(), 60, &objects, |
5408 | &next); | |
7c673cae FG |
5409 | ASSERT_EQ(r, 0); |
5410 | ASSERT_TRUE(sorted(objects)); | |
5411 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5412 | i != objects.end(); | |
5413 | ++i) { | |
5414 | if (listed.count(*i)) | |
5415 | cerr << *i << " repeated" << std::endl; | |
5416 | listed.insert(*i); | |
5417 | } | |
5418 | if (objects.size() < 50) { | |
5419 | ASSERT_TRUE(next.is_max()); | |
5420 | break; | |
5421 | } | |
5422 | objects.clear(); | |
5423 | current = next; | |
5424 | } | |
5425 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5426 | ASSERT_TRUE(listed.size() == created.size()); | |
5427 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5428 | i != listed.end(); | |
5429 | ++i) { | |
5430 | ASSERT_TRUE(created.count(*i)); | |
5431 | } | |
5432 | ||
5433 | for (set<ghobject_t>::iterator i = created.begin(); | |
5434 | i != created.end(); | |
5435 | ++i) { | |
5436 | ObjectStore::Transaction t; | |
5437 | t.remove(cid, *i); | |
11fdf7f2 | 5438 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5439 | ASSERT_EQ(r, 0); |
5440 | } | |
5441 | ObjectStore::Transaction t; | |
5442 | t.remove_collection(cid); | |
11fdf7f2 | 5443 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5444 | ASSERT_EQ(r, 0); |
5445 | } | |
5446 | ||
f91f0fd5 TL |
5447 | TEST_P(StoreTest, HashCollisionSorting) { |
5448 | bool disable_legacy = (string(GetParam()) == "bluestore"); | |
5449 | ||
5450 | char buf121664318_1[] = {18, -119, -121, -111, 0}; | |
5451 | char buf121664318_2[] = {19, 127, -121, 32, 0}; | |
5452 | char buf121664318_3[] = {19, -118, 15, 19, 0}; | |
5453 | char buf121664318_4[] = {28, 27, -116, -113, 0}; | |
5454 | char buf121664318_5[] = {28, 27, -115, -124, 0}; | |
5455 | ||
5456 | char buf121666222_1[] = {18, -119, -120, -111, 0}; | |
5457 | char buf121666222_2[] = {19, 127, -120, 32, 0}; | |
5458 | char buf121666222_3[] = {19, -118, 15, 30, 0}; | |
5459 | char buf121666222_4[] = {29, 17, -126, -113, 0}; | |
5460 | char buf121666222_5[] = {29, 17, -125, -124, 0}; | |
5461 | ||
5462 | std::map<uint32_t, std::vector<std::string>> object_names = { | |
5463 | {121664318, {{buf121664318_1}, | |
5464 | {buf121664318_2}, | |
5465 | {buf121664318_3}, | |
5466 | {buf121664318_4}, | |
5467 | {buf121664318_5}}}, | |
5468 | {121666222, {{buf121666222_1}, | |
5469 | {buf121666222_2}, | |
5470 | {buf121666222_3}, | |
5471 | {buf121666222_4}, | |
5472 | {buf121666222_5}}}}; | |
5473 | ||
5474 | int64_t poolid = 111; | |
5475 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
5476 | auto ch = store->create_new_collection(cid); | |
5477 | { | |
5478 | ObjectStore::Transaction t; | |
5479 | t.create_collection(cid, 0); | |
5480 | int r = queue_transaction(store, ch, std::move(t)); | |
5481 | ASSERT_EQ(r, 0); | |
5482 | } | |
5483 | ||
5484 | std::set<ghobject_t> created; | |
5485 | for (auto &[hash, names] : object_names) { | |
5486 | for (auto &name : names) { | |
5487 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP), | |
5488 | string(), | |
5489 | hash, | |
5490 | poolid, | |
5491 | string())); | |
5492 | ASSERT_EQ(hash, hoid.hobj.get_hash()); | |
5493 | ObjectStore::Transaction t; | |
5494 | t.touch(cid, hoid); | |
5495 | int r = queue_transaction(store, ch, std::move(t)); | |
5496 | ASSERT_EQ(r, 0); | |
5497 | created.insert(hoid); | |
5498 | } | |
5499 | } | |
5500 | ||
5501 | vector<ghobject_t> objects; | |
5502 | int r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), | |
5503 | INT_MAX, &objects, 0, disable_legacy); | |
5504 | ASSERT_EQ(r, 0); | |
5505 | ASSERT_EQ(created.size(), objects.size()); | |
5506 | auto it = objects.begin(); | |
5507 | for (auto &hoid : created) { | |
5508 | ASSERT_EQ(hoid, *it); | |
5509 | it++; | |
5510 | } | |
5511 | ||
5512 | for (auto i = created.begin(); i != created.end(); i++) { | |
5513 | auto j = i; | |
5514 | for (j++; j != created.end(); j++) { | |
5515 | std::set<ghobject_t> created_sub(i, j); | |
5516 | objects.clear(); | |
5517 | ghobject_t next; | |
5518 | r = collection_list(store, ch, *i, ghobject_t::get_max(), | |
5519 | created_sub.size(), &objects, &next, disable_legacy); | |
5520 | ASSERT_EQ(r, 0); | |
5521 | ASSERT_EQ(created_sub.size(), objects.size()); | |
5522 | it = objects.begin(); | |
5523 | for (auto &hoid : created_sub) { | |
5524 | ASSERT_EQ(hoid, *it); | |
5525 | it++; | |
5526 | } | |
5527 | if (j == created.end()) { | |
5528 | ASSERT_TRUE(next.is_max()); | |
5529 | } else { | |
5530 | ASSERT_EQ(*j, next); | |
5531 | } | |
5532 | } | |
5533 | } | |
5534 | ||
5535 | for (auto i = created.begin(); i != created.end(); i++) { | |
5536 | auto j = i; | |
5537 | for (j++; j != created.end(); j++) { | |
5538 | std::set<ghobject_t> created_sub(i, j); | |
5539 | objects.clear(); | |
5540 | ghobject_t next; | |
5541 | r = collection_list(store, ch, *i, *j, INT_MAX, &objects, &next, | |
5542 | disable_legacy); | |
5543 | ASSERT_EQ(r, 0); | |
5544 | ASSERT_EQ(created_sub.size(), objects.size()); | |
5545 | it = objects.begin(); | |
5546 | for (auto &hoid : created_sub) { | |
5547 | ASSERT_EQ(hoid, *it); | |
5548 | it++; | |
5549 | } | |
5550 | if (j == created.end()) { | |
5551 | ASSERT_TRUE(next.is_max()); | |
5552 | } else { | |
5553 | ASSERT_EQ(*j, next); | |
5554 | } | |
5555 | } | |
5556 | } | |
5557 | } | |
5558 | ||
7c673cae | 5559 | TEST_P(StoreTest, ScrubTest) { |
7c673cae FG |
5560 | int64_t poolid = 111; |
5561 | coll_t cid(spg_t(pg_t(0, poolid),shard_id_t(1))); | |
5562 | int r; | |
11fdf7f2 | 5563 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5564 | { |
5565 | ObjectStore::Transaction t; | |
5566 | t.create_collection(cid, 0); | |
11fdf7f2 | 5567 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5568 | ASSERT_EQ(r, 0); |
5569 | } | |
5570 | string base = "aaaaa"; | |
5571 | set<ghobject_t> created; | |
5572 | for (int i = 0; i < 1000; ++i) { | |
5573 | char buf[100]; | |
5574 | sprintf(buf, "%d", i); | |
5575 | if (!(i % 5)) { | |
5576 | cerr << "Object " << i << std::endl; | |
5577 | } | |
5578 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, i, | |
5579 | poolid, ""), | |
5580 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5581 | { | |
5582 | ObjectStore::Transaction t; | |
5583 | t.touch(cid, hoid); | |
11fdf7f2 | 5584 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5585 | ASSERT_EQ(r, 0); |
5586 | } | |
5587 | created.insert(hoid); | |
5588 | } | |
5589 | ||
5590 | // Add same hobject_t but different generation | |
5591 | { | |
5592 | ghobject_t hoid1(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), | |
5593 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5594 | ghobject_t hoid2(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)1, shard_id_t(1)); | |
5595 | ghobject_t hoid3(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)2, shard_id_t(1)); | |
5596 | ObjectStore::Transaction t; | |
5597 | t.touch(cid, hoid1); | |
5598 | t.touch(cid, hoid2); | |
5599 | t.touch(cid, hoid3); | |
11fdf7f2 | 5600 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5601 | created.insert(hoid1); |
5602 | created.insert(hoid2); | |
5603 | created.insert(hoid3); | |
5604 | ASSERT_EQ(r, 0); | |
5605 | } | |
5606 | ||
5607 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
5608 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5609 | &objects, 0); | |
7c673cae FG |
5610 | ASSERT_EQ(r, 0); |
5611 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5612 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5613 | ASSERT_TRUE(listed.size() == created.size()); | |
5614 | objects.clear(); | |
5615 | listed.clear(); | |
5616 | ghobject_t current, next; | |
5617 | while (1) { | |
f91f0fd5 TL |
5618 | r = collection_list(store, ch, current, ghobject_t::get_max(), 60, &objects, |
5619 | &next); | |
7c673cae FG |
5620 | ASSERT_EQ(r, 0); |
5621 | ASSERT_TRUE(sorted(objects)); | |
5622 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5623 | i != objects.end(); ++i) { | |
5624 | if (listed.count(*i)) | |
5625 | cerr << *i << " repeated" << std::endl; | |
5626 | listed.insert(*i); | |
5627 | } | |
5628 | if (objects.size() < 50) { | |
5629 | ASSERT_TRUE(next.is_max()); | |
5630 | break; | |
5631 | } | |
5632 | objects.clear(); | |
5633 | current = next.get_boundary(); | |
5634 | } | |
5635 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5636 | ASSERT_TRUE(listed.size() == created.size()); | |
5637 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5638 | i != listed.end(); | |
5639 | ++i) { | |
5640 | ASSERT_TRUE(created.count(*i)); | |
5641 | } | |
5642 | ||
5643 | for (set<ghobject_t>::iterator i = created.begin(); | |
5644 | i != created.end(); | |
5645 | ++i) { | |
5646 | ObjectStore::Transaction t; | |
5647 | t.remove(cid, *i); | |
11fdf7f2 | 5648 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5649 | ASSERT_EQ(r, 0); |
5650 | } | |
5651 | ObjectStore::Transaction t; | |
5652 | t.remove_collection(cid); | |
11fdf7f2 | 5653 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5654 | ASSERT_EQ(r, 0); |
5655 | } | |
5656 | ||
5657 | ||
5658 | TEST_P(StoreTest, OMapTest) { | |
7c673cae FG |
5659 | coll_t cid; |
5660 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 5661 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5662 | int r; |
5663 | { | |
5664 | ObjectStore::Transaction t; | |
5665 | t.create_collection(cid, 0); | |
11fdf7f2 | 5666 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5667 | ASSERT_EQ(r, 0); |
5668 | } | |
5669 | ||
5670 | map<string, bufferlist> attrs; | |
5671 | { | |
5672 | ObjectStore::Transaction t; | |
5673 | t.touch(cid, hoid); | |
5674 | t.omap_clear(cid, hoid); | |
5675 | map<string, bufferlist> start_set; | |
5676 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5677 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5678 | ASSERT_EQ(r, 0); |
7c673cae FG |
5679 | } |
5680 | ||
5681 | for (int i = 0; i < 100; i++) { | |
5682 | if (!(i%5)) { | |
5683 | std::cout << "On iteration " << i << std::endl; | |
5684 | } | |
5685 | ObjectStore::Transaction t; | |
5686 | bufferlist bl; | |
5687 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5688 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5689 | ASSERT_EQ(r, 0); |
5690 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5691 | j != attrs.end(); | |
5692 | ++j) { | |
5693 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5694 | if (!correct) { | |
5695 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5696 | if (cur_attrs.count(j->first) > 0) { | |
5697 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5698 | } | |
5699 | } | |
5700 | ASSERT_EQ(correct, true); | |
5701 | } | |
5702 | ASSERT_EQ(attrs.size(), cur_attrs.size()); | |
5703 | ||
5704 | char buf[100]; | |
5705 | snprintf(buf, sizeof(buf), "%d", i); | |
5706 | bl.clear(); | |
5707 | bufferptr bp(buf, strlen(buf) + 1); | |
5708 | bl.append(bp); | |
5709 | map<string, bufferlist> to_add; | |
5710 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5711 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5712 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5713 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5714 | ASSERT_EQ(r, 0); |
7c673cae FG |
5715 | } |
5716 | ||
5717 | int i = 0; | |
5718 | while (attrs.size()) { | |
5719 | if (!(i%5)) { | |
5720 | std::cout << "removal: On iteration " << i << std::endl; | |
5721 | } | |
5722 | ObjectStore::Transaction t; | |
5723 | bufferlist bl; | |
5724 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5725 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5726 | ASSERT_EQ(r, 0); |
5727 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5728 | j != attrs.end(); | |
5729 | ++j) { | |
5730 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5731 | if (!correct) { | |
5732 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5733 | if (cur_attrs.count(j->first) > 0) { | |
5734 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5735 | } | |
5736 | } | |
5737 | ASSERT_EQ(correct, true); | |
5738 | } | |
5739 | ||
5740 | string to_remove = attrs.begin()->first; | |
9f95a23c | 5741 | t.omap_rmkey(cid, hoid, to_remove); |
11fdf7f2 | 5742 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5743 | ASSERT_EQ(r, 0); |
7c673cae FG |
5744 | |
5745 | attrs.erase(to_remove); | |
5746 | ||
5747 | ++i; | |
5748 | } | |
5749 | ||
5750 | { | |
5751 | bufferlist bl1; | |
5752 | bl1.append("omap_header"); | |
5753 | ObjectStore::Transaction t; | |
5754 | t.omap_setheader(cid, hoid, bl1); | |
11fdf7f2 | 5755 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5756 | ASSERT_EQ(r, 0); |
7c673cae FG |
5757 | t = ObjectStore::Transaction(); |
5758 | ||
5759 | bufferlist bl2; | |
5760 | bl2.append("value"); | |
5761 | map<string, bufferlist> to_add; | |
5762 | to_add.insert(pair<string, bufferlist>("key", bl2)); | |
5763 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5764 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5765 | ASSERT_EQ(r, 0); |
7c673cae FG |
5766 | |
5767 | bufferlist bl3; | |
5768 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5769 | r = store->omap_get(ch, hoid, &bl3, &cur_attrs); |
7c673cae FG |
5770 | ASSERT_EQ(r, 0); |
5771 | ASSERT_EQ(cur_attrs.size(), size_t(1)); | |
5772 | ASSERT_TRUE(bl_eq(bl1, bl3)); | |
5773 | ||
5774 | set<string> keys; | |
11fdf7f2 | 5775 | r = store->omap_get_keys(ch, hoid, &keys); |
7c673cae FG |
5776 | ASSERT_EQ(r, 0); |
5777 | ASSERT_EQ(keys.size(), size_t(1)); | |
5778 | } | |
5779 | ||
5780 | // test omap_clear, omap_rmkey_range | |
5781 | { | |
5782 | { | |
5783 | map<string,bufferlist> to_set; | |
5784 | for (int n=0; n<10; ++n) { | |
5785 | to_set[stringify(n)].append("foo"); | |
5786 | } | |
5787 | bufferlist h; | |
5788 | h.append("header"); | |
5789 | ObjectStore::Transaction t; | |
5790 | t.remove(cid, hoid); | |
5791 | t.touch(cid, hoid); | |
5792 | t.omap_setheader(cid, hoid, h); | |
5793 | t.omap_setkeys(cid, hoid, to_set); | |
11fdf7f2 | 5794 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5795 | ASSERT_EQ(r, 0); |
7c673cae FG |
5796 | } |
5797 | { | |
5798 | ObjectStore::Transaction t; | |
5799 | t.omap_rmkeyrange(cid, hoid, "3", "7"); | |
11fdf7f2 | 5800 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5801 | ASSERT_EQ(r, 0); |
7c673cae FG |
5802 | } |
5803 | { | |
5804 | bufferlist hdr; | |
5805 | map<string,bufferlist> m; | |
11fdf7f2 | 5806 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5807 | ASSERT_EQ(6u, hdr.length()); |
5808 | ASSERT_TRUE(m.count("2")); | |
5809 | ASSERT_TRUE(!m.count("3")); | |
5810 | ASSERT_TRUE(!m.count("6")); | |
5811 | ASSERT_TRUE(m.count("7")); | |
5812 | ASSERT_TRUE(m.count("8")); | |
5813 | //cout << m << std::endl; | |
5814 | ASSERT_EQ(6u, m.size()); | |
5815 | } | |
5816 | { | |
5817 | ObjectStore::Transaction t; | |
5818 | t.omap_clear(cid, hoid); | |
11fdf7f2 | 5819 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5820 | ASSERT_EQ(r, 0); |
7c673cae FG |
5821 | } |
5822 | { | |
5823 | bufferlist hdr; | |
5824 | map<string,bufferlist> m; | |
11fdf7f2 | 5825 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5826 | ASSERT_EQ(0u, hdr.length()); |
5827 | ASSERT_EQ(0u, m.size()); | |
5828 | } | |
5829 | } | |
5830 | ||
5831 | ObjectStore::Transaction t; | |
5832 | t.remove(cid, hoid); | |
5833 | t.remove_collection(cid); | |
11fdf7f2 | 5834 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5835 | ASSERT_EQ(r, 0); |
5836 | } | |
5837 | ||
5838 | TEST_P(StoreTest, OMapIterator) { | |
7c673cae FG |
5839 | coll_t cid; |
5840 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5841 | int count = 0; | |
11fdf7f2 | 5842 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5843 | int r; |
5844 | { | |
5845 | ObjectStore::Transaction t; | |
5846 | t.create_collection(cid, 0); | |
11fdf7f2 | 5847 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5848 | ASSERT_EQ(r, 0); |
5849 | } | |
5850 | ||
5851 | map<string, bufferlist> attrs; | |
5852 | { | |
5853 | ObjectStore::Transaction t; | |
5854 | t.touch(cid, hoid); | |
5855 | t.omap_clear(cid, hoid); | |
5856 | map<string, bufferlist> start_set; | |
5857 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5858 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5859 | ASSERT_EQ(r, 0); |
7c673cae FG |
5860 | } |
5861 | ObjectMap::ObjectMapIterator iter; | |
5862 | bool correct; | |
5863 | //basic iteration | |
5864 | for (int i = 0; i < 100; i++) { | |
5865 | if (!(i%5)) { | |
5866 | std::cout << "On iteration " << i << std::endl; | |
5867 | } | |
5868 | bufferlist bl; | |
5869 | ||
5870 | // FileStore may deadlock two active iterators over the same data | |
5871 | iter = ObjectMap::ObjectMapIterator(); | |
5872 | ||
11fdf7f2 | 5873 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5874 | for (iter->seek_to_first(), count=0; iter->valid(); iter->next(), count++) { |
5875 | string key = iter->key(); | |
5876 | bufferlist value = iter->value(); | |
5877 | correct = attrs.count(key) && (string(value.c_str()) == string(attrs[key].c_str())); | |
5878 | if (!correct) { | |
5879 | if (attrs.count(key) > 0) { | |
5880 | std::cout << "key " << key << "in omap , " << value.c_str() << " : " << attrs[key].c_str() << std::endl; | |
5881 | } | |
5882 | else | |
5883 | std::cout << "key " << key << "should not exists in omap" << std::endl; | |
5884 | } | |
5885 | ASSERT_EQ(correct, true); | |
5886 | } | |
5887 | ASSERT_EQ((int)attrs.size(), count); | |
5888 | ||
11fdf7f2 | 5889 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5890 | iter = ObjectMap::ObjectMapIterator(); |
5891 | ||
5892 | char buf[100]; | |
5893 | snprintf(buf, sizeof(buf), "%d", i); | |
5894 | bl.clear(); | |
5895 | bufferptr bp(buf, strlen(buf) + 1); | |
5896 | bl.append(bp); | |
5897 | map<string, bufferlist> to_add; | |
5898 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5899 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5900 | ObjectStore::Transaction t; | |
5901 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5902 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5903 | ASSERT_EQ(r, 0); |
7c673cae FG |
5904 | } |
5905 | ||
11fdf7f2 | 5906 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5907 | //lower bound |
5908 | string bound_key = "key-5"; | |
5909 | iter->lower_bound(bound_key); | |
5910 | correct = bound_key <= iter->key(); | |
5911 | if (!correct) { | |
5912 | std::cout << "lower bound, bound key is " << bound_key << " < iter key is " << iter->key() << std::endl; | |
5913 | } | |
5914 | ASSERT_EQ(correct, true); | |
5915 | //upper bound | |
5916 | iter->upper_bound(bound_key); | |
5917 | correct = iter->key() > bound_key; | |
5918 | if (!correct) { | |
5919 | std::cout << "upper bound, bound key is " << bound_key << " >= iter key is " << iter->key() << std::endl; | |
5920 | } | |
5921 | ASSERT_EQ(correct, true); | |
5922 | ||
11fdf7f2 | 5923 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5924 | iter = ObjectMap::ObjectMapIterator(); |
5925 | { | |
5926 | ObjectStore::Transaction t; | |
5927 | t.remove(cid, hoid); | |
5928 | t.remove_collection(cid); | |
11fdf7f2 | 5929 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5930 | ASSERT_EQ(r, 0); |
5931 | } | |
5932 | } | |
5933 | ||
5934 | TEST_P(StoreTest, XattrTest) { | |
7c673cae FG |
5935 | coll_t cid; |
5936 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5937 | bufferlist big; | |
5938 | for (unsigned i = 0; i < 10000; ++i) { | |
5939 | big.append('\0'); | |
5940 | } | |
5941 | bufferlist small; | |
5942 | for (unsigned i = 0; i < 10; ++i) { | |
5943 | small.append('\0'); | |
5944 | } | |
5945 | int r; | |
11fdf7f2 | 5946 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5947 | { |
5948 | ObjectStore::Transaction t; | |
5949 | t.create_collection(cid, 0); | |
5950 | t.touch(cid, hoid); | |
11fdf7f2 | 5951 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5952 | ASSERT_EQ(r, 0); |
5953 | } | |
5954 | ||
5955 | map<string, bufferlist> attrs; | |
5956 | { | |
5957 | ObjectStore::Transaction t; | |
5958 | t.setattr(cid, hoid, "attr1", small); | |
5959 | attrs["attr1"] = small; | |
5960 | t.setattr(cid, hoid, "attr2", big); | |
5961 | attrs["attr2"] = big; | |
5962 | t.setattr(cid, hoid, "attr3", small); | |
5963 | attrs["attr3"] = small; | |
5964 | t.setattr(cid, hoid, "attr1", small); | |
5965 | attrs["attr1"] = small; | |
5966 | t.setattr(cid, hoid, "attr4", big); | |
5967 | attrs["attr4"] = big; | |
5968 | t.setattr(cid, hoid, "attr3", big); | |
5969 | attrs["attr3"] = big; | |
11fdf7f2 | 5970 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5971 | ASSERT_EQ(r, 0); |
5972 | } | |
5973 | ||
20effc67 | 5974 | map<string, bufferptr, less<>> aset; |
11fdf7f2 | 5975 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5976 | ASSERT_EQ(aset.size(), attrs.size()); |
5977 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5978 | i != aset.end(); | |
5979 | ++i) { | |
5980 | bufferlist bl; | |
5981 | bl.push_back(i->second); | |
5982 | ASSERT_TRUE(attrs[i->first] == bl); | |
5983 | } | |
5984 | ||
5985 | { | |
5986 | ObjectStore::Transaction t; | |
5987 | t.rmattr(cid, hoid, "attr2"); | |
5988 | attrs.erase("attr2"); | |
11fdf7f2 | 5989 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5990 | ASSERT_EQ(r, 0); |
5991 | } | |
5992 | ||
5993 | aset.clear(); | |
11fdf7f2 | 5994 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5995 | ASSERT_EQ(aset.size(), attrs.size()); |
5996 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5997 | i != aset.end(); | |
5998 | ++i) { | |
5999 | bufferlist bl; | |
6000 | bl.push_back(i->second); | |
6001 | ASSERT_TRUE(attrs[i->first] == bl); | |
6002 | } | |
6003 | ||
6004 | bufferptr bp; | |
11fdf7f2 | 6005 | r = store->getattr(ch, hoid, "attr2", bp); |
7c673cae FG |
6006 | ASSERT_EQ(r, -ENODATA); |
6007 | ||
11fdf7f2 | 6008 | r = store->getattr(ch, hoid, "attr3", bp); |
7c673cae FG |
6009 | ASSERT_EQ(r, 0); |
6010 | bufferlist bl2; | |
6011 | bl2.push_back(bp); | |
6012 | ASSERT_TRUE(bl2 == attrs["attr3"]); | |
6013 | ||
6014 | ObjectStore::Transaction t; | |
6015 | t.remove(cid, hoid); | |
6016 | t.remove_collection(cid); | |
11fdf7f2 | 6017 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6018 | ASSERT_EQ(r, 0); |
6019 | } | |
6020 | ||
6021 | void colsplittest( | |
6022 | ObjectStore *store, | |
6023 | unsigned num_objects, | |
6024 | unsigned common_suffix_size, | |
6025 | bool clones | |
6026 | ) { | |
7c673cae FG |
6027 | coll_t cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD)); |
6028 | coll_t tid(spg_t(pg_t(1<<common_suffix_size,52),shard_id_t::NO_SHARD)); | |
11fdf7f2 TL |
6029 | auto ch = store->create_new_collection(cid); |
6030 | auto tch = store->create_new_collection(tid); | |
7c673cae FG |
6031 | int r = 0; |
6032 | { | |
6033 | ObjectStore::Transaction t; | |
6034 | t.create_collection(cid, common_suffix_size); | |
11fdf7f2 | 6035 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6036 | ASSERT_EQ(r, 0); |
6037 | } | |
6038 | bufferlist small; | |
6039 | small.append("small"); | |
6040 | { | |
6041 | ObjectStore::Transaction t; | |
6042 | for (uint32_t i = 0; i < (2 - (int)clones)*num_objects; ++i) { | |
6043 | stringstream objname; | |
6044 | objname << "obj" << i; | |
6045 | ghobject_t a(hobject_t( | |
6046 | objname.str(), | |
6047 | "", | |
6048 | CEPH_NOSNAP, | |
6049 | i<<common_suffix_size, | |
6050 | 52, "")); | |
6051 | t.write(cid, a, 0, small.length(), small, | |
6052 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6053 | if (clones) { | |
6054 | objname << "-clone"; | |
6055 | ghobject_t b(hobject_t( | |
6056 | objname.str(), | |
6057 | "", | |
6058 | CEPH_NOSNAP, | |
6059 | i<<common_suffix_size, | |
6060 | 52, "")); | |
6061 | t.clone(cid, a, b); | |
6062 | } | |
6063 | if (i % 100) { | |
11fdf7f2 | 6064 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6065 | ASSERT_EQ(r, 0); |
6066 | t = ObjectStore::Transaction(); | |
6067 | } | |
6068 | } | |
11fdf7f2 | 6069 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6070 | ASSERT_EQ(r, 0); |
6071 | } | |
6072 | { | |
6073 | ObjectStore::Transaction t; | |
6074 | t.create_collection(tid, common_suffix_size + 1); | |
6075 | t.split_collection(cid, common_suffix_size+1, 1<<common_suffix_size, tid); | |
11fdf7f2 | 6076 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6077 | ASSERT_EQ(r, 0); |
6078 | } | |
11fdf7f2 | 6079 | ch->flush(); |
7c673cae | 6080 | |
11fdf7f2 | 6081 | // check |
7c673cae | 6082 | vector<ghobject_t> objects; |
f91f0fd5 TL |
6083 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6084 | &objects, 0); | |
7c673cae FG |
6085 | ASSERT_EQ(r, 0); |
6086 | ASSERT_EQ(objects.size(), num_objects); | |
7c673cae FG |
6087 | for (vector<ghobject_t>::iterator i = objects.begin(); |
6088 | i != objects.end(); | |
6089 | ++i) { | |
6090 | ASSERT_EQ(!!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
6091 | } |
6092 | ||
6093 | objects.clear(); | |
f91f0fd5 TL |
6094 | r = collection_list(store, tch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6095 | &objects, 0); | |
7c673cae FG |
6096 | ASSERT_EQ(r, 0); |
6097 | ASSERT_EQ(objects.size(), num_objects); | |
6098 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
6099 | i != objects.end(); | |
6100 | ++i) { | |
6101 | ASSERT_EQ(!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
6102 | } |
6103 | ||
11fdf7f2 TL |
6104 | // merge them again! |
6105 | { | |
6106 | ObjectStore::Transaction t; | |
6107 | t.merge_collection(tid, cid, common_suffix_size); | |
6108 | r = queue_transaction(store, ch, std::move(t)); | |
6109 | ASSERT_EQ(r, 0); | |
6110 | } | |
6111 | ||
6112 | // check and clean up | |
6113 | ObjectStore::Transaction t; | |
6114 | { | |
6115 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
6116 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6117 | &objects, 0); | |
11fdf7f2 TL |
6118 | ASSERT_EQ(r, 0); |
6119 | ASSERT_EQ(objects.size(), num_objects * 2); // both halves | |
6120 | unsigned size = 0; | |
6121 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
6122 | i != objects.end(); | |
6123 | ++i) { | |
6124 | t.remove(cid, *i); | |
6125 | if (++size > 100) { | |
6126 | size = 0; | |
6127 | r = queue_transaction(store, ch, std::move(t)); | |
6128 | ASSERT_EQ(r, 0); | |
6129 | t = ObjectStore::Transaction(); | |
6130 | } | |
6131 | } | |
6132 | } | |
7c673cae | 6133 | t.remove_collection(cid); |
11fdf7f2 | 6134 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 6135 | ASSERT_EQ(r, 0); |
11fdf7f2 TL |
6136 | |
6137 | ch->flush(); | |
6138 | ASSERT_TRUE(!store->collection_exists(tid)); | |
7c673cae FG |
6139 | } |
6140 | ||
11fdf7f2 TL |
6141 | TEST_P(StoreTest, ColSplitTest0) { |
6142 | colsplittest(store.get(), 10, 5, false); | |
6143 | } | |
7c673cae FG |
6144 | TEST_P(StoreTest, ColSplitTest1) { |
6145 | colsplittest(store.get(), 10000, 11, false); | |
6146 | } | |
6147 | TEST_P(StoreTest, ColSplitTest1Clones) { | |
6148 | colsplittest(store.get(), 10000, 11, true); | |
6149 | } | |
6150 | TEST_P(StoreTest, ColSplitTest2) { | |
6151 | colsplittest(store.get(), 100, 7, false); | |
6152 | } | |
6153 | TEST_P(StoreTest, ColSplitTest2Clones) { | |
6154 | colsplittest(store.get(), 100, 7, true); | |
6155 | } | |
6156 | ||
6157 | #if 0 | |
6158 | TEST_P(StoreTest, ColSplitTest3) { | |
6159 | colsplittest(store.get(), 100000, 25); | |
6160 | } | |
6161 | #endif | |
6162 | ||
11fdf7f2 TL |
6163 | void test_merge_skewed(ObjectStore *store, |
6164 | unsigned base, unsigned bits, | |
6165 | unsigned anum, unsigned bnum) | |
6166 | { | |
6167 | cout << __func__ << " 0x" << std::hex << base << std::dec | |
6168 | << " bits " << bits | |
6169 | << " anum " << anum << " bnum " << bnum << std::endl; | |
6170 | /* | |
6171 | make merge source pgs have radically different # of objects in them, | |
6172 | which should trigger different splitting in filestore, and verify that | |
6173 | post-merge all objects are accessible. | |
6174 | */ | |
6175 | int r; | |
6176 | coll_t a(spg_t(pg_t(base, 0), shard_id_t::NO_SHARD)); | |
6177 | coll_t b(spg_t(pg_t(base | (1<<bits), 0), shard_id_t::NO_SHARD)); | |
6178 | ||
6179 | auto cha = store->create_new_collection(a); | |
6180 | auto chb = store->create_new_collection(b); | |
6181 | { | |
6182 | ObjectStore::Transaction t; | |
6183 | t.create_collection(a, bits + 1); | |
6184 | r = queue_transaction(store, cha, std::move(t)); | |
6185 | ASSERT_EQ(r, 0); | |
6186 | } | |
6187 | { | |
6188 | ObjectStore::Transaction t; | |
6189 | t.create_collection(b, bits + 1); | |
6190 | r = queue_transaction(store, chb, std::move(t)); | |
6191 | ASSERT_EQ(r, 0); | |
6192 | } | |
6193 | ||
6194 | bufferlist small; | |
6195 | small.append("small"); | |
6196 | string suffix = "ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooaaaaaaaaaa"; | |
6197 | set<ghobject_t> aobjects, bobjects; | |
6198 | { | |
6199 | // fill a | |
6200 | ObjectStore::Transaction t; | |
6201 | for (unsigned i = 0; i < 1000; ++i) { | |
6202 | string objname = "a" + stringify(i) + suffix; | |
6203 | ghobject_t o(hobject_t( | |
6204 | objname, | |
6205 | "", | |
6206 | CEPH_NOSNAP, | |
6207 | i<<(bits+1) | base, | |
6208 | 52, "")); | |
6209 | aobjects.insert(o); | |
6210 | t.write(a, o, 0, small.length(), small, 0); | |
6211 | if (i % 100) { | |
6212 | r = queue_transaction(store, cha, std::move(t)); | |
6213 | ASSERT_EQ(r, 0); | |
6214 | t = ObjectStore::Transaction(); | |
6215 | } | |
6216 | } | |
6217 | r = queue_transaction(store, cha, std::move(t)); | |
6218 | ASSERT_EQ(r, 0); | |
6219 | } | |
6220 | { | |
6221 | // fill b | |
6222 | ObjectStore::Transaction t; | |
6223 | for (unsigned i = 0; i < 10; ++i) { | |
6224 | string objname = "b" + stringify(i) + suffix; | |
6225 | ghobject_t o(hobject_t( | |
6226 | objname, | |
6227 | "", | |
6228 | CEPH_NOSNAP, | |
6229 | (i<<(base+1)) | base | (1<<bits), | |
6230 | 52, "")); | |
6231 | bobjects.insert(o); | |
6232 | t.write(b, o, 0, small.length(), small, 0); | |
6233 | if (i % 100) { | |
6234 | r = queue_transaction(store, chb, std::move(t)); | |
6235 | ASSERT_EQ(r, 0); | |
6236 | t = ObjectStore::Transaction(); | |
6237 | } | |
6238 | } | |
6239 | r = queue_transaction(store, chb, std::move(t)); | |
6240 | ASSERT_EQ(r, 0); | |
6241 | } | |
6242 | ||
6243 | // merge b->a | |
6244 | { | |
6245 | ObjectStore::Transaction t; | |
6246 | t.merge_collection(b, a, bits); | |
6247 | r = queue_transaction(store, cha, std::move(t)); | |
6248 | ASSERT_EQ(r, 0); | |
6249 | } | |
6250 | ||
6251 | // verify | |
6252 | { | |
6253 | vector<ghobject_t> got; | |
f91f0fd5 TL |
6254 | collection_list(store, cha, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6255 | &got, 0); | |
11fdf7f2 TL |
6256 | set<ghobject_t> gotset; |
6257 | for (auto& o : got) { | |
6258 | ASSERT_TRUE(aobjects.count(o) || bobjects.count(o)); | |
6259 | gotset.insert(o); | |
6260 | } | |
6261 | // check both listing and stat-ability (different code paths!) | |
6262 | struct stat st; | |
6263 | for (auto& o : aobjects) { | |
6264 | ASSERT_TRUE(gotset.count(o)); | |
6265 | int r = store->stat(cha, o, &st, false); | |
6266 | ASSERT_EQ(r, 0); | |
6267 | } | |
6268 | for (auto& o : bobjects) { | |
6269 | ASSERT_TRUE(gotset.count(o)); | |
6270 | int r = store->stat(cha, o, &st, false); | |
6271 | ASSERT_EQ(r, 0); | |
6272 | } | |
6273 | } | |
6274 | ||
6275 | // clean up | |
6276 | { | |
6277 | ObjectStore::Transaction t; | |
6278 | for (auto &o : aobjects) { | |
6279 | t.remove(a, o); | |
6280 | } | |
6281 | r = queue_transaction(store, cha, std::move(t)); | |
6282 | ASSERT_EQ(r, 0); | |
6283 | } | |
6284 | { | |
6285 | ObjectStore::Transaction t; | |
6286 | for (auto &o : bobjects) { | |
6287 | t.remove(a, o); | |
6288 | } | |
6289 | t.remove_collection(a); | |
6290 | r = queue_transaction(store, cha, std::move(t)); | |
6291 | ASSERT_EQ(r, 0); | |
6292 | } | |
6293 | } | |
6294 | ||
6295 | TEST_P(StoreTest, MergeSkewed) { | |
6296 | if (string(GetParam()) != "filestore") | |
6297 | return; | |
6298 | ||
6299 | // this is sufficient to exercise merges with different hashing levels | |
6300 | test_merge_skewed(store.get(), 0xf, 4, 10, 10000); | |
6301 | test_merge_skewed(store.get(), 0xf, 4, 10000, 10); | |
6302 | ||
6303 | /* | |
6304 | // this covers a zillion variations that all boil down to the same thing | |
6305 | for (unsigned base = 3; base < 0x1000; base *= 5) { | |
6306 | unsigned bits; | |
6307 | unsigned t = base; | |
6308 | for (bits = 0; t; t >>= 1) { | |
6309 | ++bits; | |
6310 | } | |
6311 | for (unsigned b = bits; b < bits + 10; b += 3) { | |
6312 | for (auto anum : { 10, 1000, 10000 }) { | |
6313 | for (auto bnum : { 10, 1000, 10000 }) { | |
6314 | if (anum == bnum) { | |
6315 | continue; | |
6316 | } | |
6317 | test_merge_skewed(store.get(), base, b, anum, bnum); | |
6318 | } | |
6319 | } | |
6320 | } | |
6321 | } | |
6322 | */ | |
6323 | } | |
6324 | ||
6325 | ||
7c673cae FG |
6326 | /** |
6327 | * This test tests adding two different groups | |
6328 | * of objects, each with 1 common prefix and 1 | |
6329 | * different prefix. We then remove half | |
6330 | * in order to verify that the merging correctly | |
6331 | * stops at the common prefix subdir. See bug | |
6332 | * #5273 */ | |
6333 | TEST_P(StoreTest, TwoHash) { | |
7c673cae FG |
6334 | coll_t cid; |
6335 | int r; | |
11fdf7f2 | 6336 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6337 | { |
6338 | ObjectStore::Transaction t; | |
6339 | t.create_collection(cid, 0); | |
11fdf7f2 | 6340 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6341 | ASSERT_EQ(r, 0); |
6342 | } | |
6343 | std::cout << "Making objects" << std::endl; | |
6344 | for (int i = 0; i < 360; ++i) { | |
6345 | ObjectStore::Transaction t; | |
6346 | ghobject_t o; | |
6347 | o.hobj.pool = -1; | |
6348 | if (i < 8) { | |
6349 | o.hobj.set_hash((i << 16) | 0xA1); | |
6350 | t.touch(cid, o); | |
6351 | } | |
6352 | o.hobj.set_hash((i << 16) | 0xB1); | |
6353 | t.touch(cid, o); | |
11fdf7f2 | 6354 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6355 | ASSERT_EQ(r, 0); |
6356 | } | |
6357 | std::cout << "Removing half" << std::endl; | |
6358 | for (int i = 1; i < 8; ++i) { | |
6359 | ObjectStore::Transaction t; | |
6360 | ghobject_t o; | |
6361 | o.hobj.pool = -1; | |
6362 | o.hobj.set_hash((i << 16) | 0xA1); | |
6363 | t.remove(cid, o); | |
11fdf7f2 | 6364 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6365 | ASSERT_EQ(r, 0); |
6366 | } | |
6367 | std::cout << "Checking" << std::endl; | |
6368 | for (int i = 1; i < 8; ++i) { | |
6369 | ObjectStore::Transaction t; | |
6370 | ghobject_t o; | |
6371 | o.hobj.set_hash((i << 16) | 0xA1); | |
6372 | o.hobj.pool = -1; | |
11fdf7f2 | 6373 | bool exists = store->exists(ch, o); |
7c673cae FG |
6374 | ASSERT_EQ(exists, false); |
6375 | } | |
6376 | { | |
6377 | ghobject_t o; | |
6378 | o.hobj.set_hash(0xA1); | |
6379 | o.hobj.pool = -1; | |
11fdf7f2 | 6380 | bool exists = store->exists(ch, o); |
7c673cae FG |
6381 | ASSERT_EQ(exists, true); |
6382 | } | |
6383 | std::cout << "Cleanup" << std::endl; | |
6384 | for (int i = 0; i < 360; ++i) { | |
6385 | ObjectStore::Transaction t; | |
6386 | ghobject_t o; | |
6387 | o.hobj.set_hash((i << 16) | 0xA1); | |
6388 | o.hobj.pool = -1; | |
6389 | t.remove(cid, o); | |
6390 | o.hobj.set_hash((i << 16) | 0xB1); | |
6391 | t.remove(cid, o); | |
11fdf7f2 | 6392 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6393 | ASSERT_EQ(r, 0); |
6394 | } | |
6395 | ObjectStore::Transaction t; | |
6396 | t.remove_collection(cid); | |
11fdf7f2 | 6397 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6398 | ASSERT_EQ(r, 0); |
6399 | } | |
6400 | ||
6401 | TEST_P(StoreTest, Rename) { | |
7c673cae FG |
6402 | coll_t cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD)); |
6403 | ghobject_t srcoid(hobject_t("src_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6404 | ghobject_t dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6405 | bufferlist a, b; | |
6406 | a.append("foo"); | |
6407 | b.append("bar"); | |
11fdf7f2 | 6408 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6409 | int r; |
6410 | { | |
6411 | ObjectStore::Transaction t; | |
6412 | t.create_collection(cid, 0); | |
6413 | t.write(cid, srcoid, 0, a.length(), a); | |
11fdf7f2 | 6414 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6415 | ASSERT_EQ(r, 0); |
6416 | } | |
11fdf7f2 | 6417 | ASSERT_TRUE(store->exists(ch, srcoid)); |
7c673cae FG |
6418 | { |
6419 | ObjectStore::Transaction t; | |
6420 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
6421 | t.write(cid, srcoid, 0, b.length(), b); | |
6422 | t.setattr(cid, srcoid, "attr", b); | |
11fdf7f2 | 6423 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6424 | ASSERT_EQ(r, 0); |
6425 | } | |
11fdf7f2 TL |
6426 | ASSERT_TRUE(store->exists(ch, srcoid)); |
6427 | ASSERT_TRUE(store->exists(ch, dstoid)); | |
7c673cae FG |
6428 | { |
6429 | bufferlist bl; | |
11fdf7f2 | 6430 | store->read(ch, srcoid, 0, 3, bl); |
7c673cae | 6431 | ASSERT_TRUE(bl_eq(b, bl)); |
11fdf7f2 | 6432 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
6433 | ASSERT_TRUE(bl_eq(a, bl)); |
6434 | } | |
6435 | { | |
6436 | ObjectStore::Transaction t; | |
6437 | t.remove(cid, dstoid); | |
6438 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
11fdf7f2 | 6439 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6440 | ASSERT_EQ(r, 0); |
6441 | } | |
11fdf7f2 TL |
6442 | ASSERT_TRUE(store->exists(ch, dstoid)); |
6443 | ASSERT_FALSE(store->exists(ch, srcoid)); | |
7c673cae FG |
6444 | { |
6445 | bufferlist bl; | |
11fdf7f2 | 6446 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
6447 | ASSERT_TRUE(bl_eq(b, bl)); |
6448 | } | |
6449 | { | |
6450 | ObjectStore::Transaction t; | |
6451 | t.remove(cid, dstoid); | |
6452 | t.remove_collection(cid); | |
11fdf7f2 | 6453 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6454 | ASSERT_EQ(r, 0); |
6455 | } | |
6456 | } | |
6457 | ||
6458 | TEST_P(StoreTest, MoveRename) { | |
7c673cae FG |
6459 | coll_t cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD)); |
6460 | ghobject_t temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6461 | ghobject_t oid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 6462 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6463 | int r; |
6464 | { | |
6465 | ObjectStore::Transaction t; | |
6466 | t.create_collection(cid, 0); | |
6467 | t.touch(cid, oid); | |
11fdf7f2 | 6468 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6469 | ASSERT_EQ(r, 0); |
6470 | } | |
11fdf7f2 | 6471 | ASSERT_TRUE(store->exists(ch, oid)); |
7c673cae FG |
6472 | bufferlist data, attr; |
6473 | map<string, bufferlist> omap; | |
6474 | data.append("data payload"); | |
6475 | attr.append("attr value"); | |
6476 | omap["omap_key"].append("omap value"); | |
6477 | { | |
6478 | ObjectStore::Transaction t; | |
6479 | t.touch(cid, temp_oid); | |
6480 | t.write(cid, temp_oid, 0, data.length(), data); | |
6481 | t.setattr(cid, temp_oid, "attr", attr); | |
6482 | t.omap_setkeys(cid, temp_oid, omap); | |
11fdf7f2 | 6483 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6484 | ASSERT_EQ(r, 0); |
6485 | } | |
11fdf7f2 | 6486 | ASSERT_TRUE(store->exists(ch, temp_oid)); |
7c673cae FG |
6487 | { |
6488 | ObjectStore::Transaction t; | |
6489 | t.remove(cid, oid); | |
6490 | t.collection_move_rename(cid, temp_oid, cid, oid); | |
11fdf7f2 | 6491 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6492 | ASSERT_EQ(r, 0); |
6493 | } | |
11fdf7f2 TL |
6494 | ASSERT_TRUE(store->exists(ch, oid)); |
6495 | ASSERT_FALSE(store->exists(ch, temp_oid)); | |
7c673cae FG |
6496 | { |
6497 | bufferlist newdata; | |
11fdf7f2 | 6498 | r = store->read(ch, oid, 0, 1000, newdata); |
7c673cae FG |
6499 | ASSERT_GE(r, 0); |
6500 | ASSERT_TRUE(bl_eq(data, newdata)); | |
6501 | bufferlist newattr; | |
11fdf7f2 | 6502 | r = store->getattr(ch, oid, "attr", newattr); |
7c673cae FG |
6503 | ASSERT_EQ(r, 0); |
6504 | ASSERT_TRUE(bl_eq(attr, newattr)); | |
6505 | set<string> keys; | |
6506 | keys.insert("omap_key"); | |
6507 | map<string, bufferlist> newomap; | |
11fdf7f2 | 6508 | r = store->omap_get_values(ch, oid, keys, &newomap); |
7c673cae FG |
6509 | ASSERT_GE(r, 0); |
6510 | ASSERT_EQ(1u, newomap.size()); | |
6511 | ASSERT_TRUE(newomap.count("omap_key")); | |
6512 | ASSERT_TRUE(bl_eq(omap["omap_key"], newomap["omap_key"])); | |
6513 | } | |
6514 | { | |
6515 | ObjectStore::Transaction t; | |
6516 | t.remove(cid, oid); | |
6517 | t.remove_collection(cid); | |
11fdf7f2 | 6518 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6519 | ASSERT_EQ(r, 0); |
6520 | } | |
6521 | } | |
6522 | ||
6523 | TEST_P(StoreTest, BigRGWObjectName) { | |
7c673cae FG |
6524 | coll_t cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD)); |
6525 | ghobject_t oid( | |
6526 | hobject_t( | |
6527 | "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", | |
6528 | "", | |
6529 | CEPH_NOSNAP, | |
6530 | 0x81920472, | |
6531 | 12, | |
6532 | ""), | |
6533 | 15, | |
6534 | shard_id_t::NO_SHARD); | |
6535 | ghobject_t oid2(oid); | |
6536 | oid2.generation = 17; | |
6537 | ghobject_t oidhead(oid); | |
6538 | oidhead.generation = ghobject_t::NO_GEN; | |
6539 | ||
11fdf7f2 TL |
6540 | auto ch = store->create_new_collection(cid); |
6541 | ||
7c673cae FG |
6542 | int r; |
6543 | { | |
6544 | ObjectStore::Transaction t; | |
6545 | t.create_collection(cid, 0); | |
6546 | t.touch(cid, oidhead); | |
6547 | t.collection_move_rename(cid, oidhead, cid, oid); | |
6548 | t.touch(cid, oidhead); | |
6549 | t.collection_move_rename(cid, oidhead, cid, oid2); | |
11fdf7f2 | 6550 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6551 | ASSERT_EQ(r, 0); |
6552 | } | |
6553 | ||
6554 | { | |
6555 | ObjectStore::Transaction t; | |
6556 | t.remove(cid, oid); | |
11fdf7f2 | 6557 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6558 | ASSERT_EQ(r, 0); |
6559 | } | |
6560 | ||
6561 | { | |
6562 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
6563 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6564 | &objects, 0); | |
7c673cae FG |
6565 | ASSERT_EQ(r, 0); |
6566 | ASSERT_EQ(objects.size(), 1u); | |
6567 | ASSERT_EQ(objects[0], oid2); | |
6568 | } | |
6569 | ||
11fdf7f2 | 6570 | ASSERT_FALSE(store->exists(ch, oid)); |
7c673cae FG |
6571 | |
6572 | { | |
6573 | ObjectStore::Transaction t; | |
6574 | t.remove(cid, oid2); | |
6575 | t.remove_collection(cid); | |
11fdf7f2 | 6576 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6577 | ASSERT_EQ(r, 0); |
6578 | ||
6579 | } | |
6580 | } | |
6581 | ||
6582 | TEST_P(StoreTest, SetAllocHint) { | |
7c673cae FG |
6583 | coll_t cid; |
6584 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 6585 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6586 | int r; |
6587 | { | |
6588 | ObjectStore::Transaction t; | |
6589 | t.create_collection(cid, 0); | |
6590 | t.touch(cid, hoid); | |
11fdf7f2 | 6591 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6592 | ASSERT_EQ(r, 0); |
6593 | } | |
6594 | { | |
6595 | ObjectStore::Transaction t; | |
6596 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6597 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6598 | ASSERT_EQ(r, 0); |
6599 | } | |
6600 | { | |
6601 | ObjectStore::Transaction t; | |
6602 | t.remove(cid, hoid); | |
11fdf7f2 | 6603 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6604 | ASSERT_EQ(r, 0); |
6605 | } | |
6606 | { | |
6607 | ObjectStore::Transaction t; | |
6608 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6609 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6610 | ASSERT_EQ(r, 0); |
6611 | } | |
6612 | { | |
6613 | ObjectStore::Transaction t; | |
6614 | t.remove_collection(cid); | |
11fdf7f2 | 6615 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6616 | ASSERT_EQ(r, 0); |
6617 | } | |
6618 | } | |
6619 | ||
6620 | TEST_P(StoreTest, TryMoveRename) { | |
7c673cae FG |
6621 | coll_t cid; |
6622 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6623 | ghobject_t hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP, 0, -1, "")); | |
11fdf7f2 | 6624 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6625 | int r; |
6626 | { | |
6627 | ObjectStore::Transaction t; | |
6628 | t.create_collection(cid, 0); | |
11fdf7f2 | 6629 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6630 | ASSERT_EQ(r, 0); |
6631 | } | |
6632 | { | |
6633 | ObjectStore::Transaction t; | |
6634 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6635 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6636 | ASSERT_EQ(r, 0); |
6637 | } | |
6638 | { | |
6639 | ObjectStore::Transaction t; | |
6640 | t.touch(cid, hoid); | |
11fdf7f2 | 6641 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6642 | ASSERT_EQ(r, 0); |
6643 | } | |
6644 | { | |
6645 | ObjectStore::Transaction t; | |
6646 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6647 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6648 | ASSERT_EQ(r, 0); |
6649 | } | |
6650 | struct stat st; | |
11fdf7f2 TL |
6651 | ASSERT_EQ(store->stat(ch, hoid, &st), -ENOENT); |
6652 | ASSERT_EQ(store->stat(ch, hoid2, &st), 0); | |
7c673cae FG |
6653 | } |
6654 | ||
11fdf7f2 | 6655 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6656 | TEST_P(StoreTest, BluestoreOnOffCSumTest) { |
6657 | if (string(GetParam()) != "bluestore") | |
6658 | return; | |
11fdf7f2 TL |
6659 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6660 | g_conf().apply_changes(nullptr); | |
7c673cae | 6661 | |
7c673cae FG |
6662 | int r; |
6663 | coll_t cid; | |
6664 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6665 | { | |
11fdf7f2 TL |
6666 | auto ch = store->open_collection(cid); |
6667 | ASSERT_FALSE(ch); | |
7c673cae | 6668 | } |
11fdf7f2 | 6669 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6670 | { |
6671 | ObjectStore::Transaction t; | |
6672 | t.create_collection(cid, 0); | |
6673 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 6674 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6675 | ASSERT_EQ(r, 0); |
6676 | } | |
6677 | { | |
6678 | //write with csum enabled followed by read with csum disabled | |
6679 | size_t block_size = 64*1024; | |
6680 | ObjectStore::Transaction t; | |
6681 | bufferlist bl, orig; | |
6682 | bl.append(std::string(block_size, 'a')); | |
6683 | orig = bl; | |
6684 | t.remove(cid, hoid); | |
6685 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6686 | t.write(cid, hoid, 0, bl.length(), bl); | |
6687 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6688 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6689 | ASSERT_EQ(r, 0); |
6690 | ||
11fdf7f2 TL |
6691 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6692 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6693 | |
6694 | bufferlist in; | |
11fdf7f2 | 6695 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6696 | ASSERT_EQ((int)block_size, r); |
6697 | ASSERT_TRUE(bl_eq(orig, in)); | |
6698 | ||
6699 | } | |
6700 | { | |
6701 | //write with csum disabled followed by read with csum enabled | |
6702 | ||
6703 | size_t block_size = 64*1024; | |
6704 | ObjectStore::Transaction t; | |
6705 | bufferlist bl, orig; | |
6706 | bl.append(std::string(block_size, 'a')); | |
6707 | orig = bl; | |
6708 | t.remove(cid, hoid); | |
6709 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6710 | t.write(cid, hoid, 0, bl.length(), bl); | |
6711 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6712 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6713 | ASSERT_EQ(r, 0); |
6714 | ||
11fdf7f2 TL |
6715 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6716 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6717 | |
6718 | bufferlist in; | |
11fdf7f2 | 6719 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6720 | ASSERT_EQ((int)block_size, r); |
6721 | ASSERT_TRUE(bl_eq(orig, in)); | |
6722 | } | |
6723 | { | |
6724 | //'mixed' non-overlapping writes to the same blob | |
6725 | ||
6726 | ObjectStore::Transaction t; | |
6727 | bufferlist bl, orig; | |
6728 | size_t block_size = 8000; | |
6729 | bl.append(std::string(block_size, 'a')); | |
6730 | orig = bl; | |
6731 | t.remove(cid, hoid); | |
6732 | t.write(cid, hoid, 0, bl.length(), bl); | |
6733 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6734 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6735 | ASSERT_EQ(r, 0); |
6736 | ||
11fdf7f2 TL |
6737 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6738 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6739 | |
6740 | ObjectStore::Transaction t2; | |
6741 | t2.write(cid, hoid, block_size*2, bl.length(), bl); | |
6742 | cerr << "Append 'unprotected'" << std::endl; | |
11fdf7f2 | 6743 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6744 | ASSERT_EQ(r, 0); |
6745 | ||
6746 | bufferlist in; | |
11fdf7f2 | 6747 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6748 | ASSERT_EQ((int)block_size, r); |
6749 | ASSERT_TRUE(bl_eq(orig, in)); | |
6750 | in.clear(); | |
11fdf7f2 | 6751 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6752 | ASSERT_EQ((int)block_size, r); |
6753 | ASSERT_TRUE(bl_eq(orig, in)); | |
6754 | ||
11fdf7f2 TL |
6755 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6756 | g_conf().apply_changes(nullptr); | |
7c673cae | 6757 | in.clear(); |
11fdf7f2 | 6758 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6759 | ASSERT_EQ((int)block_size, r); |
6760 | ASSERT_TRUE(bl_eq(orig, in)); | |
6761 | in.clear(); | |
11fdf7f2 | 6762 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6763 | ASSERT_EQ((int)block_size, r); |
6764 | ASSERT_TRUE(bl_eq(orig, in)); | |
6765 | } | |
6766 | { | |
6767 | //partially blob overwrite under a different csum enablement mode | |
6768 | ||
6769 | ObjectStore::Transaction t; | |
6770 | bufferlist bl, orig, orig2; | |
6771 | size_t block_size0 = 0x10000; | |
6772 | size_t block_size = 9000; | |
6773 | size_t block_size2 = 5000; | |
6774 | bl.append(std::string(block_size0, 'a')); | |
6775 | t.remove(cid, hoid); | |
6776 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6777 | t.write(cid, hoid, 0, bl.length(), bl); | |
6778 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6779 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6780 | ASSERT_EQ(r, 0); |
6781 | ||
11fdf7f2 TL |
6782 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6783 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6784 | |
6785 | ObjectStore::Transaction t2; | |
6786 | bl.clear(); | |
6787 | bl.append(std::string(block_size, 'b')); | |
6788 | t2.write(cid, hoid, 0, bl.length(), bl); | |
6789 | t2.write(cid, hoid, block_size0, bl.length(), bl); | |
6790 | cerr << "Overwrite with unprotected data" << std::endl; | |
11fdf7f2 | 6791 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6792 | ASSERT_EQ(r, 0); |
6793 | ||
6794 | orig = bl; | |
6795 | orig2 = bl; | |
6796 | orig.append( std::string(block_size0 - block_size, 'a')); | |
6797 | ||
6798 | bufferlist in; | |
11fdf7f2 | 6799 | r = store->read(ch, hoid, 0, block_size0, in); |
7c673cae FG |
6800 | ASSERT_EQ((int)block_size0, r); |
6801 | ASSERT_TRUE(bl_eq(orig, in)); | |
6802 | ||
11fdf7f2 | 6803 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6804 | ASSERT_EQ((int)block_size, r); |
6805 | ASSERT_TRUE(bl_eq(orig2, in)); | |
6806 | ||
11fdf7f2 TL |
6807 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6808 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6809 | |
6810 | ObjectStore::Transaction t3; | |
6811 | bl.clear(); | |
6812 | bl.append(std::string(block_size2, 'c')); | |
6813 | t3.write(cid, hoid, block_size0, bl.length(), bl); | |
6814 | cerr << "Overwrite with protected data" << std::endl; | |
11fdf7f2 | 6815 | r = queue_transaction(store, ch, std::move(t3)); |
7c673cae FG |
6816 | ASSERT_EQ(r, 0); |
6817 | ||
6818 | in.clear(); | |
6819 | orig = bl; | |
6820 | orig.append( std::string(block_size - block_size2, 'b')); | |
11fdf7f2 | 6821 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6822 | ASSERT_EQ((int)block_size, r); |
6823 | ASSERT_TRUE(bl_eq(orig, in)); | |
6824 | } | |
6825 | ||
6826 | { | |
6827 | ObjectStore::Transaction t; | |
6828 | t.remove(cid, hoid); | |
6829 | t.remove_collection(cid); | |
6830 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6831 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6832 | ASSERT_EQ(r, 0); |
6833 | } | |
6834 | } | |
6835 | #endif | |
6836 | ||
9f95a23c | 6837 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6838 | ObjectStore, |
6839 | StoreTest, | |
6840 | ::testing::Values( | |
6841 | "memstore", | |
6842 | "filestore", | |
11fdf7f2 | 6843 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6844 | "bluestore", |
6845 | #endif | |
6846 | "kstore")); | |
6847 | ||
6848 | // Note: instantiate all stores to preserve store numbering order only | |
9f95a23c | 6849 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6850 | ObjectStore, |
6851 | StoreTestSpecificAUSize, | |
6852 | ::testing::Values( | |
6853 | "memstore", | |
6854 | "filestore", | |
11fdf7f2 | 6855 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6856 | "bluestore", |
6857 | #endif | |
6858 | "kstore")); | |
6859 | ||
a4b75251 TL |
6860 | // Note: instantiate all stores to preserve store numbering order only |
6861 | INSTANTIATE_TEST_SUITE_P( | |
6862 | ObjectStore, | |
6863 | StoreTestOmapUpgrade, | |
6864 | ::testing::Values( | |
6865 | "memstore", | |
6866 | "filestore", | |
6867 | #if defined(WITH_BLUESTORE) | |
6868 | "bluestore", | |
6869 | #endif | |
6870 | "kstore")); | |
6871 | ||
20effc67 TL |
6872 | #if defined(WITH_BLUESTORE) |
6873 | INSTANTIATE_TEST_SUITE_P( | |
6874 | ObjectStore, | |
6875 | StoreTestDeferredSetup, | |
6876 | ::testing::Values( | |
6877 | "bluestore")); | |
6878 | #endif | |
6879 | ||
6880 | void doMany4KWritesTest(ObjectStore* store, | |
7c673cae FG |
6881 | unsigned max_objects, |
6882 | unsigned max_ops, | |
6883 | unsigned max_object_size, | |
6884 | unsigned max_write_size, | |
11fdf7f2 | 6885 | unsigned write_alignment) |
7c673cae | 6886 | { |
7c673cae FG |
6887 | MixedGenerator gen(555); |
6888 | gen_type rng(time(NULL)); | |
6889 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 6890 | store_statfs_t res_stat; |
7c673cae | 6891 | |
20effc67 | 6892 | SyntheticWorkloadState test_obj(store, |
7c673cae FG |
6893 | &gen, |
6894 | &rng, | |
7c673cae FG |
6895 | cid, |
6896 | max_object_size, | |
6897 | max_write_size, | |
6898 | write_alignment); | |
6899 | test_obj.init(); | |
6900 | for (unsigned i = 0; i < max_objects; ++i) { | |
6901 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
6902 | test_obj.touch(); | |
6903 | } | |
6904 | for (unsigned i = 0; i < max_ops; ++i) { | |
6905 | if (!(i % 200)) { | |
6906 | cerr << "Op " << i << std::endl; | |
6907 | test_obj.print_internal_state(); | |
6908 | } | |
6909 | test_obj.write(); | |
6910 | } | |
6911 | test_obj.wait_for_done(); | |
11fdf7f2 TL |
6912 | test_obj.statfs(res_stat); |
6913 | if (!(res_stat.data_stored <= max_object_size) || | |
6914 | !(res_stat.allocated <= max_object_size)) { | |
6915 | // this will provide more insight on the mismatch and | |
6916 | // helps to avoid any races during stats collection | |
6917 | test_obj.fsck(false); | |
6918 | // retrieving stats once again and assert if still broken | |
6919 | test_obj.statfs(res_stat); | |
6920 | ASSERT_LE(res_stat.data_stored, max_object_size); | |
6921 | ASSERT_LE(res_stat.allocated, max_object_size); | |
7c673cae FG |
6922 | } |
6923 | test_obj.shutdown(); | |
6924 | } | |
6925 | ||
6926 | TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { | |
6927 | if (string(GetParam()) != "bluestore") | |
6928 | return; | |
20effc67 TL |
6929 | if (smr) { |
6930 | cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" | |
6931 | << std::endl; | |
6932 | return; | |
6933 | } | |
7c673cae FG |
6934 | |
6935 | StartDeferred(0x10000); | |
6936 | ||
11fdf7f2 | 6937 | const unsigned max_object = 4*1024*1024; |
20effc67 | 6938 | doMany4KWritesTest(store.get(), 1, 1000, max_object, 4*1024, 0); |
7c673cae FG |
6939 | } |
6940 | ||
6941 | TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { | |
6942 | if (string(GetParam()) != "bluestore") | |
6943 | return; | |
20effc67 TL |
6944 | if (smr) { |
6945 | cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" | |
6946 | << std::endl; | |
6947 | return; | |
6948 | } | |
7c673cae | 6949 | StartDeferred(0x10000); |
11fdf7f2 TL |
6950 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6951 | g_ceph_context->_conf.apply_changes(nullptr); | |
6952 | const unsigned max_object = 4*1024*1024; | |
7c673cae | 6953 | |
20effc67 | 6954 | doMany4KWritesTest(store.get(), 1, 1000, max_object, 4*1024, 0 ); |
7c673cae FG |
6955 | } |
6956 | ||
6957 | TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { | |
6958 | if (string(GetParam()) != "bluestore") | |
6959 | return; | |
20effc67 TL |
6960 | if (smr) { |
6961 | cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" | |
6962 | << std::endl; | |
6963 | return; | |
6964 | } | |
7c673cae | 6965 | StartDeferred(0x10000); |
11fdf7f2 | 6966 | const unsigned max_object = 4*1024*1024; |
20effc67 | 6967 | doMany4KWritesTest(store.get(), 1, 1000, max_object, 4*1024, 0); |
7c673cae FG |
6968 | } |
6969 | ||
11fdf7f2 | 6970 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6971 | void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items) |
6972 | { | |
f91f0fd5 | 6973 | uint64_t meta_allocated = mempool::bluestore_cache_meta::allocated_bytes(); |
31f18b77 FG |
6974 | uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes(); |
6975 | uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes(); | |
7c673cae | 6976 | |
f91f0fd5 | 6977 | uint64_t meta_items = mempool::bluestore_cache_meta::allocated_items(); |
31f18b77 FG |
6978 | uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items(); |
6979 | uint64_t other_items = mempool::bluestore_cache_other::allocated_items(); | |
f91f0fd5 TL |
6980 | cout << "meta(" << meta_allocated << "/" << meta_items |
6981 | << ") onode(" << onode_allocated << "/" << onode_items | |
7c673cae FG |
6982 | << ") other(" << other_allocated << "/" << other_items |
6983 | << ")" << std::endl; | |
f91f0fd5 | 6984 | *total_bytes = meta_allocated + onode_allocated + other_allocated; |
7c673cae FG |
6985 | *total_items = onode_items; |
6986 | } | |
6987 | ||
6988 | TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) { | |
6989 | ||
6990 | if (string(GetParam()) != "bluestore") | |
6991 | return; | |
6992 | ||
6993 | size_t block_size = 4096; | |
6994 | StartDeferred(block_size); | |
11fdf7f2 TL |
6995 | SetVal(g_conf(), "bluestore_compression_mode", "none"); |
6996 | SetVal(g_conf(), "bluestore_csum_type", "none"); | |
6997 | SetVal(g_conf(), "bluestore_cache_size_hdd", "400000000"); | |
6998 | SetVal(g_conf(), "bluestore_cache_size_ssd", "400000000"); | |
6999 | g_conf().apply_changes(nullptr); | |
7c673cae | 7000 | |
7c673cae FG |
7001 | int r; |
7002 | coll_t cid; | |
7003 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
7004 | size_t obj_size = 4 * 1024 * 1024; | |
7005 | uint64_t total_bytes, total_bytes2; | |
7006 | uint64_t total_onodes; | |
7007 | get_mempool_stats(&total_bytes, &total_onodes); | |
7008 | ASSERT_EQ(total_onodes, 0u); | |
7009 | ||
11fdf7f2 | 7010 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7011 | { |
7012 | ObjectStore::Transaction t; | |
7013 | t.create_collection(cid, 0); | |
11fdf7f2 | 7014 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7015 | ASSERT_EQ(r, 0); |
7016 | } | |
7017 | { | |
7018 | ObjectStore::Transaction t; | |
7019 | bufferlist bl, orig, orig2; | |
7020 | ||
7021 | bl.append(std::string(obj_size, 'a')); | |
7022 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 7023 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7024 | ASSERT_EQ(r, 0); |
7025 | } | |
7026 | get_mempool_stats(&total_bytes, &total_onodes); | |
7027 | ASSERT_NE(total_bytes, 0u); | |
7028 | ASSERT_EQ(total_onodes, 1u); | |
7029 | ||
7030 | { | |
7031 | ObjectStore::Transaction t; | |
7032 | t.truncate(cid, hoid, 0); | |
11fdf7f2 | 7033 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7034 | ASSERT_EQ(r, 0); |
7035 | } | |
7036 | ||
7037 | for(size_t i = 0; i < 1; ++i) { | |
7038 | bufferlist bl; | |
7039 | bl.append(std::string(block_size * (i+1), 'a')); | |
7040 | for( size_t j = 0; j < obj_size; j+= bl.length()) { | |
7041 | ObjectStore::Transaction t; | |
7042 | t.write(cid, hoid, j, bl.length(), bl); | |
11fdf7f2 | 7043 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7044 | ASSERT_EQ(r, 0); |
7045 | } | |
7046 | get_mempool_stats(&total_bytes2, &total_onodes); | |
7047 | ASSERT_NE(total_bytes2, 0u); | |
7048 | ASSERT_EQ(total_onodes, 1u); | |
7049 | } | |
7050 | { | |
7051 | cout <<" mempool dump:\n"; | |
7052 | JSONFormatter f(true); | |
7053 | f.open_object_section("transaction"); | |
7054 | mempool::dump(&f); | |
7055 | f.close_section(); | |
7056 | f.flush(cout); | |
7057 | cout << std::endl; | |
7058 | } | |
7059 | { | |
7060 | bufferlist bl; | |
7061 | for (size_t i = 0; i < obj_size; i += 0x1000) { | |
11fdf7f2 | 7062 | store->read(ch, hoid, i, 0x1000, bl); |
7c673cae FG |
7063 | } |
7064 | } | |
7065 | get_mempool_stats(&total_bytes, &total_onodes); | |
7066 | ASSERT_NE(total_bytes, 0u); | |
7067 | ASSERT_EQ(total_onodes, 1u); | |
7068 | ||
7069 | { | |
7070 | cout <<" mempool dump:\n"; | |
7071 | JSONFormatter f(true); | |
7072 | f.open_object_section("transaction"); | |
7073 | mempool::dump(&f); | |
7074 | f.close_section(); | |
7075 | f.flush(cout); | |
7076 | cout << std::endl; | |
7077 | } | |
7078 | { | |
7079 | ObjectStore::Transaction t; | |
7080 | t.remove(cid, hoid); | |
7081 | t.remove_collection(cid); | |
7082 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7083 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7084 | ASSERT_EQ(r, 0); |
7085 | } | |
7c673cae FG |
7086 | } |
7087 | ||
7088 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) { | |
7089 | ||
7090 | if (string(GetParam()) != "bluestore") | |
7091 | return; | |
7092 | ||
7093 | size_t block_size = 4096; | |
7094 | StartDeferred(block_size); | |
11fdf7f2 TL |
7095 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
7096 | g_conf().apply_changes(nullptr); | |
7c673cae | 7097 | |
7c673cae FG |
7098 | int r; |
7099 | coll_t cid; | |
7100 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
7101 | ||
7102 | const PerfCounters* logger = store->get_perf_counters(); | |
7103 | ||
11fdf7f2 | 7104 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7105 | { |
7106 | ObjectStore::Transaction t; | |
7107 | t.create_collection(cid, 0); | |
11fdf7f2 | 7108 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7109 | ASSERT_EQ(r, 0); |
7110 | } | |
7111 | { | |
7112 | ObjectStore::Transaction t; | |
7113 | bufferlist bl; | |
7114 | ||
7115 | bl.append(std::string(block_size * 2, 'a')); | |
7116 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7117 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7118 | ASSERT_EQ(r, 0); |
7119 | } | |
7120 | { | |
7121 | // overwrite at the beginning | |
7122 | ObjectStore::Transaction t; | |
7123 | bufferlist bl; | |
7124 | ||
7125 | bl.append(std::string(block_size, 'b')); | |
7126 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7127 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7128 | ASSERT_EQ(r, 0); |
7129 | } | |
7130 | { | |
7131 | // append | |
7132 | ObjectStore::Transaction t; | |
7133 | bufferlist bl; | |
7134 | ||
7135 | bl.append(std::string(block_size * 2, 'c')); | |
7136 | t.write(cid, hoid, block_size * 2, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7137 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7138 | ASSERT_EQ(r, 0); |
7139 | } | |
7140 | { | |
7141 | // append with a gap | |
7142 | ObjectStore::Transaction t; | |
7143 | bufferlist bl; | |
7144 | ||
7145 | bl.append(std::string(block_size * 2, 'd')); | |
7146 | t.write(cid, hoid, block_size * 5, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7147 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7148 | ASSERT_EQ(r, 0); |
7149 | } | |
7150 | { | |
7151 | // We need to issue a read to trigger cache stat update that refresh | |
7152 | // perf counters. additionally we need to wait some time for mempool | |
7153 | // thread to update stats. | |
7154 | sleep(1); | |
7155 | bufferlist bl, expected; | |
11fdf7f2 | 7156 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
7157 | ASSERT_EQ(r, (int)block_size); |
7158 | expected.append(string(block_size, 'b')); | |
7159 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7160 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7161 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
7162 | } | |
7163 | { | |
7164 | // overwrite at end | |
7165 | ObjectStore::Transaction t; | |
7166 | bufferlist bl; | |
7167 | ||
7168 | bl.append(std::string(block_size * 2, 'e')); | |
7169 | ||
7170 | // Currently we are unable to reuse blob when overwriting in a single step | |
7171 | t.write(cid, hoid, block_size * 6, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7172 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7173 | ASSERT_EQ(r, 0); |
7174 | } | |
7175 | { | |
7176 | // We need to issue a read to trigger cache stat update that refresh | |
7177 | // perf counters. additionally we need to wait some time for mempool | |
7178 | // thread to update stats. | |
7179 | sleep(1); | |
7180 | bufferlist bl, expected; | |
11fdf7f2 | 7181 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
7182 | ASSERT_EQ(r, (int)block_size); |
7183 | expected.append(string(block_size, 'b')); | |
7184 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7185 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7186 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
7187 | } | |
7188 | { | |
7189 | // fill the gap | |
7190 | ObjectStore::Transaction t; | |
7191 | bufferlist bl; | |
7192 | ||
7193 | bl.append(std::string(block_size, 'f')); | |
7194 | ||
7195 | t.write(cid, hoid, block_size * 4, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7196 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7197 | ASSERT_EQ(r, 0); |
7198 | } | |
7199 | { | |
7200 | // we need to wait some time for mempool | |
7201 | // thread to update stats to be able to check blob/extent numbers from | |
7202 | // perf counters. | |
7203 | sleep(1); | |
7204 | ||
7205 | bufferlist bl, expected; | |
11fdf7f2 | 7206 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
7207 | ASSERT_EQ(r, (int)block_size); |
7208 | expected.append(string(block_size, 'b')); | |
7209 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7210 | ||
7211 | bl.clear(); | |
7212 | expected.clear(); | |
11fdf7f2 | 7213 | r = store->read(ch, hoid, block_size, block_size, bl); |
7c673cae FG |
7214 | ASSERT_EQ(r, (int)block_size); |
7215 | expected.append(string(block_size, 'a')); | |
7216 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7217 | ||
7218 | bl.clear(); | |
7219 | expected.clear(); | |
11fdf7f2 | 7220 | r = store->read(ch, hoid, block_size * 2, block_size * 2, bl); |
7c673cae FG |
7221 | ASSERT_EQ(r, (int)block_size * 2); |
7222 | expected.append(string(block_size * 2, 'c')); | |
7223 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7224 | ||
7225 | bl.clear(); | |
7226 | expected.clear(); | |
11fdf7f2 | 7227 | r = store->read(ch, hoid, block_size * 4, block_size, bl); |
7c673cae FG |
7228 | ASSERT_EQ(r, (int)block_size); |
7229 | expected.append(string(block_size, 'f')); | |
7230 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7231 | ||
7232 | bl.clear(); | |
7233 | expected.clear(); | |
11fdf7f2 | 7234 | r = store->read(ch, hoid, block_size * 5, block_size, bl); |
7c673cae FG |
7235 | ASSERT_EQ(r, (int)block_size); |
7236 | expected.append(string(block_size, 'd')); | |
7237 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7238 | ||
7239 | bl.clear(); | |
7240 | expected.clear(); | |
11fdf7f2 | 7241 | r = store->read(ch, hoid, block_size * 5, block_size * 3, bl); |
7c673cae FG |
7242 | ASSERT_EQ(r, (int)block_size * 3); |
7243 | expected.append(string(block_size, 'd')); | |
7244 | expected.append(string(block_size * 2, 'e')); | |
7245 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7246 | } | |
7247 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7248 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
7249 | ||
7250 | ||
7251 | { | |
7252 | ObjectStore::Transaction t; | |
7253 | t.remove(cid, hoid); | |
7254 | t.remove_collection(cid); | |
7255 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7256 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7257 | ASSERT_EQ(r, 0); |
7258 | } | |
7c673cae FG |
7259 | } |
7260 | ||
20effc67 TL |
7261 | TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionSmallAppend) { |
7262 | if (string(GetParam()) != "bluestore") { | |
7263 | return; | |
7264 | } | |
7265 | ||
7266 | size_t block_size = 65536; | |
7267 | StartDeferred(block_size); | |
7268 | ||
7269 | int r; | |
7270 | coll_t cid; | |
7271 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
7272 | ||
7273 | const PerfCounters* logger = store->get_perf_counters(); | |
7274 | ||
7275 | auto ch = store->create_new_collection(cid); | |
7276 | { | |
7277 | ObjectStore::Transaction t; | |
7278 | t.create_collection(cid, 0); | |
7279 | r = queue_transaction(store, ch, std::move(t)); | |
7280 | ASSERT_EQ(r, 0); | |
7281 | } | |
7282 | { | |
7283 | // [1] append zeros | |
7284 | ObjectStore::Transaction t; | |
7285 | bufferlist bl; | |
7286 | ||
7287 | bl.append_zero(4096); | |
7288 | ||
7289 | t.write(cid, hoid, 0, bl.length(), bl); | |
7290 | r = queue_transaction(store, ch, std::move(t)); | |
7291 | ASSERT_EQ(r, 0); | |
7292 | ASSERT_EQ(logger->get(l_bluestore_write_small), 1u); | |
7293 | ASSERT_EQ(logger->get(l_bluestore_write_small_bytes), 4096u); | |
7294 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped), 1u); | |
7295 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped_bytes), 4096u); | |
7296 | ||
7297 | bufferlist in; | |
7298 | r = store->read(ch, hoid, 0, 0x4000, in); | |
7299 | ASSERT_EQ(4096, r); | |
7300 | ASSERT_TRUE(in.is_zero()); | |
7301 | } | |
7302 | ||
7303 | { | |
7304 | // [2] append non-zeros | |
7305 | ObjectStore::Transaction t; | |
7306 | bufferlist bl; | |
7307 | ||
7308 | bl.append(std::string(4096, 'c')); | |
7309 | ||
7310 | t.write(cid, hoid, 4096, bl.length(), bl); | |
7311 | r = queue_transaction(store, ch, std::move(t)); | |
7312 | ASSERT_EQ(r, 0); | |
7313 | ASSERT_EQ(logger->get(l_bluestore_write_small), 2u); | |
7314 | ASSERT_EQ(logger->get(l_bluestore_write_small_bytes), 4096u*2); | |
7315 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped), 1u); | |
7316 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped_bytes), 4096u); | |
7317 | ||
7318 | bufferlist in, _exp; | |
7319 | r = store->read(ch, hoid, 0, 0x4000, in); | |
7320 | ASSERT_EQ(4096 * 2, r); | |
7321 | _exp.append_zero(4096); | |
7322 | _exp.append(bl); | |
7323 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7324 | } | |
7325 | ||
7326 | { | |
7327 | ObjectStore::Transaction t; | |
7328 | t.remove(cid, hoid); | |
7329 | t.remove_collection(cid); | |
7330 | cerr << "Cleaning" << std::endl; | |
7331 | r = queue_transaction(store, ch, std::move(t)); | |
7332 | ASSERT_EQ(r, 0); | |
7333 | } | |
7334 | } | |
7335 | ||
7336 | TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionSmallOverwrite) { | |
7337 | if (string(GetParam()) != "bluestore") { | |
7338 | return; | |
7339 | } | |
7340 | if (smr) { | |
7341 | cout << "SKIP" << std::endl; | |
7342 | return; | |
7343 | } | |
7344 | ||
7345 | size_t block_size = 65536; | |
7346 | StartDeferred(block_size); | |
7347 | ||
7348 | int r; | |
7349 | coll_t cid; | |
7350 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
7351 | ||
7352 | const PerfCounters* logger = store->get_perf_counters(); | |
7353 | ||
7354 | auto ch = store->create_new_collection(cid); | |
7355 | { | |
7356 | ObjectStore::Transaction t; | |
7357 | t.create_collection(cid, 0); | |
7358 | r = queue_transaction(store, ch, std::move(t)); | |
7359 | ASSERT_EQ(r, 0); | |
7360 | } | |
7361 | ||
7362 | { | |
7363 | // {setting up the scenario} append non-zeros | |
7364 | ObjectStore::Transaction t; | |
7365 | bufferlist bl; | |
7366 | ||
7367 | bl.append(std::string(4096, 'c')); | |
7368 | ||
7369 | t.write(cid, hoid, 0, bl.length(), bl); | |
7370 | r = queue_transaction(store, ch, std::move(t)); | |
7371 | ASSERT_EQ(r, 0); | |
7372 | ASSERT_EQ(logger->get(l_bluestore_write_small), 1u); | |
7373 | ASSERT_EQ(logger->get(l_bluestore_write_small_bytes), 4096u); | |
7374 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped), 0u); | |
7375 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped_bytes), 0u); | |
7376 | ||
7377 | bufferlist in, _exp; | |
7378 | r = store->read(ch, hoid, 0, 0x4000, in); | |
7379 | ASSERT_EQ(4096, r); | |
7380 | _exp.append(bl); | |
7381 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7382 | } | |
7383 | ||
7384 | { | |
7385 | // [1] overwrite non-zeros with zeros | |
7386 | ObjectStore::Transaction t; | |
7387 | bufferlist bl; | |
7388 | ||
7389 | bl.append_zero(4096); | |
7390 | ||
7391 | t.write(cid, hoid, 0, bl.length(), bl); | |
7392 | r = queue_transaction(store, ch, std::move(t)); | |
7393 | ASSERT_EQ(r, 0); | |
7394 | ASSERT_EQ(logger->get(l_bluestore_write_small), 2u); | |
7395 | ASSERT_EQ(logger->get(l_bluestore_write_small_bytes), 4096u*2); | |
7396 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped), 0u); | |
7397 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped_bytes), 0u); | |
7398 | ||
7399 | bufferlist in; | |
7400 | r = store->read(ch, hoid, 0, 0x4000, in); | |
7401 | ASSERT_EQ(4096, r); | |
7402 | ASSERT_TRUE(in.is_zero()); | |
7403 | } | |
7404 | ||
7405 | { | |
7406 | // [2] overwrite zeros with non-zeros | |
7407 | ObjectStore::Transaction t; | |
7408 | bufferlist bl; | |
7409 | ||
7410 | bl.append(std::string(4096, 'c')); | |
7411 | ||
7412 | t.write(cid, hoid, 0, bl.length(), bl); | |
7413 | r = queue_transaction(store, ch, std::move(t)); | |
7414 | ASSERT_EQ(r, 0); | |
7415 | ASSERT_EQ(logger->get(l_bluestore_write_small), 3u); | |
7416 | ASSERT_EQ(logger->get(l_bluestore_write_small_bytes), 4096u*3); | |
7417 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped), 0u); | |
7418 | ASSERT_EQ(logger->get(l_bluestore_write_small_skipped_bytes), 0u); | |
7419 | ||
7420 | bufferlist in, _exp; | |
7421 | r = store->read(ch, hoid, 0, 0x4000, in); | |
7422 | ASSERT_EQ(4096, r); | |
7423 | _exp.append(bl); | |
7424 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7425 | } | |
7426 | ||
7427 | { | |
7428 | ObjectStore::Transaction t; | |
7429 | t.remove(cid, hoid); | |
7430 | t.remove_collection(cid); | |
7431 | cerr << "Cleaning" << std::endl; | |
7432 | r = queue_transaction(store, ch, std::move(t)); | |
7433 | ASSERT_EQ(r, 0); | |
7434 | } | |
7435 | } | |
7436 | ||
7437 | TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionBigAppend) { | |
7438 | if (string(GetParam()) != "bluestore") { | |
7439 | return; | |
7440 | } | |
7441 | ||
7442 | size_t block_size = 4096; | |
7443 | StartDeferred(block_size); | |
7444 | ||
7445 | int r; | |
7446 | coll_t cid; | |
7447 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
7448 | ||
7449 | const PerfCounters* logger = store->get_perf_counters(); | |
7450 | ||
7451 | auto ch = store->create_new_collection(cid); | |
7452 | { | |
7453 | ObjectStore::Transaction t; | |
7454 | t.create_collection(cid, 0); | |
7455 | r = queue_transaction(store, ch, std::move(t)); | |
7456 | ASSERT_EQ(r, 0); | |
7457 | } | |
7458 | ||
7459 | { | |
7460 | // [1] append zeros | |
7461 | ObjectStore::Transaction t; | |
7462 | bufferlist bl; | |
7463 | ||
7464 | bl.append_zero(block_size * 2); | |
7465 | ||
7466 | t.write(cid, hoid, 0, bl.length(), bl); | |
7467 | r = queue_transaction(store, ch, std::move(t)); | |
7468 | ASSERT_EQ(r, 0); | |
7469 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
7470 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), 4096u*2); | |
7471 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 0u); | |
7472 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_blobs), 1u); | |
7473 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_bytes), 4096u*2); | |
7474 | ||
7475 | bufferlist in; | |
7476 | r = store->read(ch, hoid, 0, block_size * 8, in); | |
7477 | ASSERT_EQ(block_size * 2, r); | |
7478 | ASSERT_TRUE(in.is_zero()); | |
7479 | } | |
7480 | ||
7481 | { | |
7482 | // [2] append non-zeros | |
7483 | ObjectStore::Transaction t; | |
7484 | bufferlist bl; | |
7485 | ||
7486 | bl.append(std::string(block_size * 2, 'c')); | |
7487 | ||
7488 | t.write(cid, hoid, block_size * 2, bl.length(), bl); | |
7489 | r = queue_transaction(store, ch, std::move(t)); | |
7490 | ASSERT_EQ(r, 0); | |
7491 | ASSERT_EQ(logger->get(l_bluestore_write_big), 2u); | |
7492 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), 4096u*4); | |
7493 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
7494 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_blobs), 1u); | |
7495 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_bytes), 4096u*2); | |
7496 | ||
7497 | bufferlist in, _exp; | |
7498 | r = store->read(ch, hoid, 0, block_size * 8, in); | |
7499 | ASSERT_EQ(block_size * 4, r); | |
7500 | _exp.append_zero(block_size * 2); | |
7501 | _exp.append(bl); | |
7502 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7503 | } | |
7504 | ||
7505 | { | |
7506 | ObjectStore::Transaction t; | |
7507 | t.remove(cid, hoid); | |
7508 | t.remove_collection(cid); | |
7509 | cerr << "Cleaning" << std::endl; | |
7510 | r = queue_transaction(store, ch, std::move(t)); | |
7511 | ASSERT_EQ(r, 0); | |
7512 | } | |
7513 | } | |
7514 | ||
7515 | TEST_P(StoreTestSpecificAUSize, ZeroBlockDetectionBigOverwrite) { | |
7516 | if (string(GetParam()) != "bluestore") { | |
7517 | return; | |
7518 | } | |
7519 | if (smr) { | |
7520 | cout << "SKIP" << std::endl; | |
7521 | return; | |
7522 | } | |
7523 | ||
7524 | size_t block_size = 4096; | |
7525 | StartDeferred(block_size); | |
7526 | ||
7527 | int r; | |
7528 | coll_t cid; | |
7529 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
7530 | ||
7531 | const PerfCounters* logger = store->get_perf_counters(); | |
7532 | ||
7533 | auto ch = store->create_new_collection(cid); | |
7534 | { | |
7535 | ObjectStore::Transaction t; | |
7536 | t.create_collection(cid, 0); | |
7537 | r = queue_transaction(store, ch, std::move(t)); | |
7538 | ASSERT_EQ(r, 0); | |
7539 | } | |
7540 | ||
7541 | { | |
7542 | // {setting up the scenario} append non-zeros | |
7543 | ObjectStore::Transaction t; | |
7544 | bufferlist bl; | |
7545 | ||
7546 | bl.append(std::string(block_size * 2, 'c')); | |
7547 | ||
7548 | t.write(cid, hoid, 0, bl.length(), bl); | |
7549 | r = queue_transaction(store, ch, std::move(t)); | |
7550 | ASSERT_EQ(r, 0); | |
7551 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
7552 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), 4096u*2); | |
7553 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
7554 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_blobs), 0u); | |
7555 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_bytes), 0u); | |
7556 | ||
7557 | bufferlist in, _exp; | |
7558 | r = store->read(ch, hoid, 0, block_size * 8, in); | |
7559 | ASSERT_EQ(block_size * 2, r); | |
7560 | _exp.append(bl); | |
7561 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7562 | } | |
7563 | ||
7564 | { | |
7565 | // [1] overwrite non-zeros with zeros | |
7566 | ObjectStore::Transaction t; | |
7567 | bufferlist bl; | |
7568 | ||
7569 | bl.append_zero(block_size * 2); | |
7570 | ||
7571 | t.write(cid, hoid, 0, bl.length(), bl); | |
7572 | r = queue_transaction(store, ch, std::move(t)); | |
7573 | ASSERT_EQ(r, 0); | |
7574 | ASSERT_EQ(logger->get(l_bluestore_write_big), 2u); | |
7575 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), 4096u*4); | |
7576 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
7577 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_blobs), 1u); | |
7578 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_bytes), 4096u*2); | |
7579 | ||
7580 | bufferlist in; | |
7581 | r = store->read(ch, hoid, 0, block_size * 8, in); | |
7582 | ASSERT_EQ(block_size * 2, r); | |
7583 | ASSERT_TRUE(in.is_zero()); | |
7584 | } | |
7585 | ||
7586 | { | |
7587 | // [2] overwrite zeros with non-zeros | |
7588 | ObjectStore::Transaction t; | |
7589 | bufferlist bl; | |
7590 | ||
7591 | bl.append(std::string(block_size * 2, 'c')); | |
7592 | ||
7593 | t.write(cid, hoid, 0, bl.length(), bl); | |
7594 | r = queue_transaction(store, ch, std::move(t)); | |
7595 | ASSERT_EQ(r, 0); | |
7596 | ASSERT_EQ(logger->get(l_bluestore_write_big), 3u); | |
7597 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), 4096u*6); | |
7598 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 2u); | |
7599 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_blobs), 1u); | |
7600 | ASSERT_EQ(logger->get(l_bluestore_write_big_skipped_bytes), 4096u*2); | |
7601 | ||
7602 | bufferlist in, _exp; | |
7603 | r = store->read(ch, hoid, 0, block_size * 8, in); | |
7604 | ASSERT_EQ(block_size * 2, r); | |
7605 | _exp.append(bl); | |
7606 | ASSERT_TRUE(bl_eq(_exp, in)); | |
7607 | } | |
7608 | ||
7609 | { | |
7610 | ObjectStore::Transaction t; | |
7611 | t.remove(cid, hoid); | |
7612 | t.remove_collection(cid); | |
7613 | cerr << "Cleaning" << std::endl; | |
7614 | r = queue_transaction(store, ch, std::move(t)); | |
7615 | ASSERT_EQ(r, 0); | |
7616 | } | |
7617 | } | |
7618 | ||
f67539c2 | 7619 | TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) { |
7c673cae FG |
7620 | |
7621 | if (string(GetParam()) != "bluestore") | |
7622 | return; | |
20effc67 TL |
7623 | if (smr) { |
7624 | cout << "SKIP: no deferred" << std::endl; | |
7625 | return; | |
7626 | } | |
7c673cae FG |
7627 | |
7628 | size_t block_size = 4096; | |
7629 | StartDeferred(block_size); | |
f67539c2 TL |
7630 | SetVal(g_conf(), "bluestore_max_blob_size", "131072"); |
7631 | SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536"); | |
7632 | ||
11fdf7f2 | 7633 | g_conf().apply_changes(nullptr); |
7c673cae | 7634 | |
7c673cae FG |
7635 | int r; |
7636 | coll_t cid; | |
f67539c2 TL |
7637 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); |
7638 | ghobject_t hoid2(hobject_t("test2", "", CEPH_NOSNAP, 0, -1, "")); | |
11fdf7f2 | 7639 | |
522d829b | 7640 | PerfCounters* logger = const_cast<PerfCounters*>(store->get_perf_counters()); |
f67539c2 TL |
7641 | |
7642 | auto ch = store->create_new_collection(cid); | |
7c673cae FG |
7643 | { |
7644 | ObjectStore::Transaction t; | |
7645 | t.create_collection(cid, 0); | |
11fdf7f2 | 7646 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7647 | ASSERT_EQ(r, 0); |
7648 | } | |
7649 | { | |
7650 | ObjectStore::Transaction t; | |
f67539c2 | 7651 | bufferlist bl, bl2; |
7c673cae | 7652 | |
f67539c2 TL |
7653 | bl.append(std::string(block_size * 2, 'c')); |
7654 | bl2.append(std::string(block_size * 3, 'd')); | |
7655 | ||
7656 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7657 | t.set_alloc_hint(cid, hoid2, block_size * 4, block_size * 4, | |
7658 | CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ); | |
7659 | t.write(cid, hoid2, 0, bl2.length(), bl2, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
11fdf7f2 | 7660 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7661 | ASSERT_EQ(r, 0); |
7662 | } | |
f67539c2 TL |
7663 | ASSERT_EQ(logger->get(l_bluestore_write_big), 2u); |
7664 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 0u); | |
7665 | ||
7666 | { | |
7667 | struct store_statfs_t statfs; | |
7668 | int r = store->statfs(&statfs); | |
7669 | ASSERT_EQ(r, 0); | |
7670 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 5); | |
7671 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 5); | |
7672 | } | |
7673 | ||
7674 | // overwrite at the beginning, 4K alignment | |
7c673cae | 7675 | { |
7c673cae FG |
7676 | ObjectStore::Transaction t; |
7677 | bufferlist bl; | |
7678 | ||
7679 | bl.append(std::string(block_size, 'b')); | |
f67539c2 | 7680 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
11fdf7f2 | 7681 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7682 | ASSERT_EQ(r, 0); |
7683 | } | |
f67539c2 TL |
7684 | ASSERT_EQ(logger->get(l_bluestore_write_big), 3u); |
7685 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u); | |
7686 | ||
7c673cae | 7687 | { |
7c673cae | 7688 | bufferlist bl, expected; |
f67539c2 TL |
7689 | r = store->read(ch, hoid, 0, block_size, bl); |
7690 | ASSERT_EQ(r, (int)block_size); | |
7c673cae | 7691 | expected.append(string(block_size, 'b')); |
7c673cae | 7692 | ASSERT_TRUE(bl_eq(expected, bl)); |
f67539c2 TL |
7693 | } |
7694 | { | |
7695 | bufferlist bl, expected; | |
7696 | r = store->read(ch, hoid, block_size, block_size, bl); | |
7697 | ASSERT_EQ(r, (int)block_size); | |
7698 | expected.append(string(block_size, 'c')); | |
7699 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7c673cae FG |
7700 | } |
7701 | ||
f67539c2 | 7702 | // overwrite at the end, 4K alignment |
7c673cae | 7703 | { |
7c673cae FG |
7704 | ObjectStore::Transaction t; |
7705 | bufferlist bl; | |
7706 | ||
f67539c2 TL |
7707 | bl.append(std::string(block_size, 'g')); |
7708 | t.write(cid, hoid, block_size, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
11fdf7f2 | 7709 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7710 | ASSERT_EQ(r, 0); |
7711 | } | |
f67539c2 TL |
7712 | ASSERT_EQ(logger->get(l_bluestore_write_big), 4u); |
7713 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 2u); | |
7714 | ||
7c673cae | 7715 | { |
7c673cae | 7716 | bufferlist bl, expected; |
f67539c2 TL |
7717 | r = store->read(ch, hoid, 0, block_size, bl); |
7718 | ASSERT_EQ(r, (int)block_size); | |
7c673cae FG |
7719 | expected.append(string(block_size, 'b')); |
7720 | ASSERT_TRUE(bl_eq(expected, bl)); | |
f67539c2 TL |
7721 | } |
7722 | { | |
7723 | bufferlist bl, expected; | |
7724 | r = store->read(ch, hoid, block_size, block_size, bl); | |
7725 | ASSERT_EQ(r, (int)block_size); | |
7726 | expected.append(string(block_size, 'g')); | |
7727 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7c673cae FG |
7728 | } |
7729 | ||
f67539c2 | 7730 | // overwrite at 4K, 12K alignment |
7c673cae | 7731 | { |
7c673cae FG |
7732 | ObjectStore::Transaction t; |
7733 | bufferlist bl; | |
7734 | ||
f67539c2 TL |
7735 | bl.append(std::string(block_size, 'e')); |
7736 | t.write(cid, hoid2, block_size , bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
11fdf7f2 | 7737 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7738 | ASSERT_EQ(r, 0); |
7739 | } | |
f67539c2 TL |
7740 | ASSERT_EQ(logger->get(l_bluestore_write_big), 5u); |
7741 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 3u); | |
7742 | ||
7743 | // makes sure deferred has been submitted | |
7744 | // and do all the checks again | |
7745 | sleep(g_conf().get_val<double>("bluestore_max_defer_interval") + 2); | |
7746 | ||
7747 | ASSERT_EQ(logger->get(l_bluestore_write_big), 5u); | |
7748 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 3u); | |
7749 | ||
7c673cae | 7750 | { |
7c673cae | 7751 | bufferlist bl, expected; |
f67539c2 TL |
7752 | r = store->read(ch, hoid, 0, block_size, bl); |
7753 | ASSERT_EQ(r, (int)block_size); | |
7754 | expected.append(string(block_size, 'b')); | |
7c673cae | 7755 | ASSERT_TRUE(bl_eq(expected, bl)); |
7c673cae | 7756 | } |
7c673cae | 7757 | { |
f67539c2 TL |
7758 | bufferlist bl, expected; |
7759 | r = store->read(ch, hoid, block_size, block_size, bl); | |
7760 | ASSERT_EQ(r, (int)block_size); | |
7761 | expected.append(string(block_size, 'g')); | |
7762 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7c673cae FG |
7763 | } |
7764 | { | |
7c673cae | 7765 | bufferlist bl, expected; |
f67539c2 TL |
7766 | r = store->read(ch, hoid2, 0, block_size, bl); |
7767 | ASSERT_EQ(r, (int)block_size); | |
7768 | expected.append(string(block_size, 'd')); | |
7c673cae | 7769 | ASSERT_TRUE(bl_eq(expected, bl)); |
f67539c2 TL |
7770 | } |
7771 | { | |
7772 | bufferlist bl, expected; | |
7773 | r = store->read(ch, hoid2, block_size, block_size, bl); | |
7774 | ASSERT_EQ(r, (int)block_size); | |
7775 | expected.append(string(block_size, 'e')); | |
7776 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7777 | } | |
7778 | { | |
7779 | bufferlist bl, expected; | |
7780 | r = store->read(ch, hoid2, block_size * 2, block_size, bl); | |
7781 | ASSERT_EQ(r, (int)block_size); | |
7782 | expected.append(string(block_size, 'd')); | |
7783 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7784 | } | |
7785 | ||
7786 | { | |
7787 | struct store_statfs_t statfs; | |
7788 | int r = store->statfs(&statfs); | |
7789 | ASSERT_EQ(r, 0); | |
7790 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 5); | |
7791 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 5); | |
7792 | } | |
7793 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
7794 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
7795 | ||
7796 | { | |
7797 | ObjectStore::Transaction t; | |
7798 | t.remove(cid, hoid); | |
7799 | t.remove(cid, hoid2); | |
7800 | r = queue_transaction(store, ch, std::move(t)); | |
7801 | ASSERT_EQ(r, 0); | |
7802 | } | |
7803 | ||
7804 | { | |
7805 | ObjectStore::Transaction t; | |
7806 | bufferlist bl; | |
7807 | bl.append(std::string(block_size * 2, 'f')); | |
7808 | ||
7809 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7810 | r = queue_transaction(store, ch, std::move(t)); | |
7811 | ASSERT_EQ(r, 0); | |
7812 | } | |
7813 | ASSERT_EQ(logger->get(l_bluestore_write_big), 6u); | |
7814 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 3u); | |
7815 | ||
7816 | { | |
7817 | ObjectStore::Transaction t; | |
7818 | t.zero(cid, hoid, 0, 100); | |
7819 | r = queue_transaction(store, ch, std::move(t)); | |
7820 | ASSERT_EQ(r, 0); | |
7821 | } | |
7822 | { | |
7823 | bufferlist bl, expected; | |
7824 | r = store->read(ch, hoid, 0, 100, bl); | |
7825 | ASSERT_EQ(r, (int)100); | |
7826 | expected.append(string(100, 0)); | |
7827 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7828 | } | |
7829 | { | |
7830 | bufferlist bl, expected; | |
7831 | r = store->read(ch, hoid, 100, block_size * 2 - 100, bl); | |
7832 | ASSERT_EQ(r, (int)block_size * 2 - 100); | |
7833 | expected.append(string(block_size * 2 - 100, 'f')); | |
7834 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7835 | } | |
7836 | sleep(2); | |
7837 | { | |
7838 | struct store_statfs_t statfs; | |
7839 | int r = store->statfs(&statfs); | |
7840 | ASSERT_EQ(r, 0); | |
7841 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 2 - 100); | |
7842 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 2); | |
7843 | } | |
7844 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7845 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
7846 | ||
7847 | { | |
7848 | ObjectStore::Transaction t; | |
7849 | bufferlist bl; | |
7850 | bl.append(std::string(block_size, 'g')); | |
7851 | ||
7852 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7853 | r = queue_transaction(store, ch, std::move(t)); | |
7854 | ASSERT_EQ(r, 0); | |
7855 | } | |
7856 | ASSERT_EQ(logger->get(l_bluestore_write_big), 7u); | |
7857 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u); | |
7858 | { | |
7859 | bufferlist bl, expected; | |
7860 | r = store->read(ch, hoid, 0, block_size, bl); | |
7861 | ASSERT_EQ(r, (int)block_size); | |
7862 | expected.append(string(block_size, 'g')); | |
7863 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7864 | } | |
7865 | { | |
7866 | bufferlist bl, expected; | |
7867 | r = store->read(ch, hoid, block_size, block_size, bl); | |
7868 | ASSERT_EQ(r, (int)block_size); | |
7869 | expected.append(string(block_size, 'f')); | |
7870 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7871 | } | |
7872 | ||
7873 | { | |
7874 | struct store_statfs_t statfs; | |
7875 | int r = store->statfs(&statfs); | |
7876 | ASSERT_EQ(r, 0); | |
7877 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 2); | |
7878 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 2); | |
7879 | } | |
7880 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7881 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
7882 | ||
7883 | // check whether full overwrite bypass deferred | |
7884 | { | |
7885 | ObjectStore::Transaction t; | |
7886 | bufferlist bl; | |
7887 | bl.append(std::string(block_size * 2, 'h')); | |
7888 | ||
7889 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7890 | r = queue_transaction(store, ch, std::move(t)); | |
7891 | ASSERT_EQ(r, 0); | |
7892 | } | |
7893 | ASSERT_EQ(logger->get(l_bluestore_write_big), 8u); | |
7894 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u); | |
7895 | ||
7896 | { | |
7897 | bufferlist bl, expected; | |
7898 | r = store->read(ch, hoid, 0, block_size * 2, bl); | |
7899 | ASSERT_EQ(r, (int)block_size * 2); | |
7900 | expected.append(string(block_size * 2, 'h')); | |
7901 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7902 | } | |
7903 | ||
7904 | { | |
7905 | struct store_statfs_t statfs; | |
7906 | int r = store->statfs(&statfs); | |
7907 | ASSERT_EQ(r, 0); | |
7908 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 2); | |
7909 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 2); | |
7910 | } | |
7911 | ||
7912 | { | |
7913 | ObjectStore::Transaction t; | |
7914 | t.remove(cid, hoid); | |
7915 | t.remove(cid, hoid2); | |
7916 | r = queue_transaction(store, ch, std::move(t)); | |
7917 | ASSERT_EQ(r, 0); | |
7918 | } | |
7919 | ||
7920 | { | |
7921 | ObjectStore::Transaction t; | |
7922 | bufferlist bl; | |
7923 | bl.append(std::string(block_size * 32, 'a')); | |
7924 | ||
7925 | // this will create two 128K aligned blobs | |
7926 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7927 | t.write(cid, hoid, bl.length(), bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7928 | r = queue_transaction(store, ch, std::move(t)); | |
7929 | ASSERT_EQ(r, 0); | |
7930 | } | |
7931 | ASSERT_EQ(logger->get(l_bluestore_write_big), 10u); | |
7932 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 4u); | |
7933 | ||
7934 | // check whether overwrite (less than prefer_deferred_size) partially overlapping two adjacent blobs goes | |
7935 | // deferred | |
7936 | { | |
7937 | ObjectStore::Transaction t; | |
7938 | bufferlist bl; | |
7939 | bl.append(std::string(block_size * 3, 'b')); | |
7940 | ||
7941 | t.write(cid, hoid, 0x20000 - block_size, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7942 | r = queue_transaction(store, ch, std::move(t)); | |
7943 | ASSERT_EQ(r, 0); | |
7944 | } | |
7945 | ASSERT_EQ(logger->get(l_bluestore_write_big), 11u); | |
7946 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 6u); | |
7947 | ||
7948 | { | |
7949 | bufferlist bl, expected; | |
7950 | r = store->read(ch, hoid, 0, 0x20000 - block_size, bl); | |
7951 | ASSERT_EQ(r, 0x20000 - block_size); | |
7952 | expected.append(string(r, 'a')); | |
7953 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7954 | expected.clear(); | |
7955 | ||
7956 | r = store->read(ch, hoid, 0x20000 - block_size, block_size * 3, bl); | |
7957 | ASSERT_EQ(r, 3 * block_size); | |
7958 | expected.append(string(r, 'b')); | |
7959 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7960 | expected.clear(); | |
7961 | ||
7962 | r = store->read(ch, hoid, 0x20000 + 2 * block_size, block_size * 30, bl); | |
7963 | ASSERT_EQ(r, 30 * block_size); | |
7964 | expected.append(string(r, 'a')); | |
7965 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7966 | expected.clear(); | |
7967 | } | |
7968 | ||
7969 | { | |
7970 | struct store_statfs_t statfs; | |
7971 | int r = store->statfs(&statfs); | |
7972 | ASSERT_EQ(r, 0); | |
7973 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 64); | |
7974 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 64); | |
7975 | } | |
7976 | ||
7977 | // check whether overwrite (larger than prefer_deferred_size) partially | |
7978 | // overlapping two adjacent blobs goes deferred | |
7979 | { | |
7980 | ObjectStore::Transaction t; | |
7981 | bufferlist bl; | |
7982 | bl.append(std::string(block_size * 30, 'c')); | |
7983 | ||
7984 | t.write(cid, hoid, 0x10000 + block_size, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
7985 | r = queue_transaction(store, ch, std::move(t)); | |
7986 | ASSERT_EQ(r, 0); | |
7987 | } | |
7988 | sleep(2); | |
7989 | ASSERT_EQ(logger->get(l_bluestore_write_big), 12u); | |
7990 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 8u); | |
7991 | ||
7992 | { | |
7993 | bufferlist bl, expected; | |
7994 | r = store->read(ch, hoid, 0, 0x11000, bl); | |
7995 | ASSERT_EQ(r, 0x11000); | |
7996 | expected.append(string(r, 'a')); | |
7997 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7998 | expected.clear(); | |
7999 | ||
8000 | r = store->read(ch, hoid, 0x11000, block_size * 30, bl); | |
8001 | ASSERT_EQ(r, block_size * 30); | |
8002 | expected.append(string(r, 'c')); | |
8003 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8004 | expected.clear(); | |
8005 | ||
8006 | r = store->read(ch, hoid, block_size * 47, 0x10000 + block_size, bl); | |
8007 | ASSERT_EQ(r, 0x10000 + block_size); | |
8008 | expected.append(string(r, 'a')); | |
8009 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8010 | expected.clear(); | |
8011 | } | |
8012 | ||
8013 | { | |
8014 | struct store_statfs_t statfs; | |
8015 | int r = store->statfs(&statfs); | |
8016 | ASSERT_EQ(r, 0); | |
8017 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 64); | |
8018 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 64); | |
8019 | } | |
8020 | ||
522d829b TL |
8021 | logger->reset(); |
8022 | // check whether overwrite (prefer_deferred_size < 120K < 2 * prefer_defer_size) partially | |
8023 | // overlapping two adjacent blobs goes partly deferred | |
f67539c2 TL |
8024 | { |
8025 | ObjectStore::Transaction t; | |
8026 | bufferlist bl; | |
8027 | bl.append(std::string(block_size * 30, 'e')); | |
8028 | ||
8029 | t.write(cid, hoid, 0x20000 - block_size, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8030 | r = queue_transaction(store, ch, std::move(t)); | |
8031 | ASSERT_EQ(r, 0); | |
8032 | } | |
8033 | sleep(2); | |
522d829b TL |
8034 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); |
8035 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u); | |
20effc67 TL |
8036 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 1u); |
8037 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_write_bytes), block_size); | |
f67539c2 TL |
8038 | |
8039 | { | |
8040 | struct store_statfs_t statfs; | |
8041 | int r = store->statfs(&statfs); | |
8042 | ASSERT_EQ(r, 0); | |
8043 | ASSERT_EQ(statfs.data_stored, (unsigned)block_size * 64); | |
8044 | ASSERT_LE(statfs.allocated, (unsigned)block_size * 64); | |
8045 | } | |
8046 | ||
8047 | { | |
8048 | ObjectStore::Transaction t; | |
8049 | t.remove(cid, hoid); | |
8050 | t.remove(cid, hoid2); | |
8051 | t.remove_collection(cid); | |
8052 | cerr << "Cleaning" << std::endl; | |
8053 | r = queue_transaction(store, ch, std::move(t)); | |
8054 | ASSERT_EQ(r, 0); | |
8055 | } | |
8056 | } | |
8057 | ||
522d829b TL |
8058 | TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) { |
8059 | ||
8060 | if (string(GetParam()) != "bluestore") | |
8061 | return; | |
20effc67 TL |
8062 | if (smr) { |
8063 | cout << "SKIP: no deferred" << std::endl; | |
8064 | return; | |
8065 | } | |
522d829b TL |
8066 | |
8067 | size_t block_size = 4096; | |
8068 | StartDeferred(block_size); | |
8069 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); | |
8070 | SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536"); | |
8071 | ||
8072 | g_conf().apply_changes(nullptr); | |
8073 | ||
8074 | int r; | |
8075 | coll_t cid; | |
8076 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
8077 | ||
8078 | PerfCounters* logger = const_cast<PerfCounters*>(store->get_perf_counters()); | |
8079 | ||
8080 | auto ch = store->create_new_collection(cid); | |
8081 | { | |
8082 | ObjectStore::Transaction t; | |
8083 | t.create_collection(cid, 0); | |
8084 | r = queue_transaction(store, ch, std::move(t)); | |
8085 | ASSERT_EQ(r, 0); | |
8086 | } | |
8087 | { | |
8088 | ObjectStore::Transaction t; | |
8089 | bufferlist bl; | |
8090 | ||
8091 | bl.append(std::string(128 * 1024, 'c')); | |
8092 | ||
8093 | t.write(cid, hoid, 0x1000, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8094 | r = queue_transaction(store, ch, std::move(t)); | |
8095 | ASSERT_EQ(r, 0); | |
8096 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
8097 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), bl.length()); | |
8098 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 3u); | |
8099 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 0u); | |
20effc67 TL |
8100 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 0u); |
8101 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_write_bytes), 0); | |
522d829b TL |
8102 | } |
8103 | ||
8104 | logger->reset(); | |
8105 | { | |
8106 | ObjectStore::Transaction t; | |
8107 | bufferlist bl; | |
8108 | ||
8109 | bl.append(std::string(128 * 1024, 'c')); | |
8110 | ||
8111 | t.write(cid, hoid, 0x2000, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8112 | r = queue_transaction(store, ch, std::move(t)); | |
8113 | ASSERT_EQ(r, 0); | |
8114 | ||
8115 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
8116 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), bl.length()); | |
8117 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 3u); | |
8118 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u); | |
20effc67 TL |
8119 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 1u); |
8120 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_write_bytes), 57344); | |
522d829b TL |
8121 | } |
8122 | ||
8123 | { | |
8124 | ObjectStore::Transaction t; | |
8125 | t.remove(cid, hoid); | |
8126 | t.remove(cid, hoid); | |
8127 | t.remove_collection(cid); | |
8128 | cerr << "Cleaning" << std::endl; | |
8129 | r = queue_transaction(store, ch, std::move(t)); | |
8130 | ASSERT_EQ(r, 0); | |
8131 | } | |
8132 | } | |
8133 | ||
8134 | TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) { | |
8135 | ||
8136 | if (string(GetParam()) != "bluestore") | |
8137 | return; | |
20effc67 TL |
8138 | if (smr) { |
8139 | cout << "SKIP: no deferred" << std::endl; | |
8140 | return; | |
8141 | } | |
522d829b TL |
8142 | |
8143 | size_t block_size = 4096; | |
8144 | StartDeferred(block_size); | |
8145 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); | |
8146 | SetVal(g_conf(), "bluestore_prefer_deferred_size", "65536"); | |
8147 | ||
8148 | g_conf().apply_changes(nullptr); | |
8149 | ||
8150 | int r; | |
8151 | coll_t cid; | |
8152 | ghobject_t hoid(hobject_t("test", "", CEPH_NOSNAP, 0, -1, "")); | |
8153 | ||
8154 | PerfCounters* logger = const_cast<PerfCounters*>(store->get_perf_counters()); | |
8155 | ||
8156 | auto ch = store->create_new_collection(cid); | |
8157 | { | |
8158 | ObjectStore::Transaction t; | |
8159 | t.create_collection(cid, 0); | |
8160 | r = queue_transaction(store, ch, std::move(t)); | |
8161 | ASSERT_EQ(r, 0); | |
8162 | } | |
8163 | ||
8164 | logger->reset(); | |
8165 | { | |
8166 | ObjectStore::Transaction t; | |
8167 | bufferlist bl; | |
8168 | ||
8169 | bl.append(std::string(4096 * 1024, 'c')); | |
8170 | ||
8171 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8172 | r = queue_transaction(store, ch, std::move(t)); | |
8173 | ASSERT_EQ(r, 0); | |
8174 | ||
8175 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
8176 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), bl.length()); | |
8177 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 64u); | |
8178 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 0u); | |
20effc67 TL |
8179 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 0u); |
8180 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_write_bytes), 0u); | |
522d829b TL |
8181 | } |
8182 | logger->reset(); | |
8183 | { | |
8184 | ObjectStore::Transaction t; | |
8185 | bufferlist bl; | |
8186 | ||
8187 | bl.append(std::string(4096 * 1024, 'c')); | |
8188 | ||
8189 | t.write(cid, hoid, 0x1000, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8190 | r = queue_transaction(store, ch, std::move(t)); | |
8191 | ASSERT_EQ(r, 0); | |
8192 | ||
8193 | ASSERT_EQ(logger->get(l_bluestore_write_big), 1u); | |
8194 | ASSERT_EQ(logger->get(l_bluestore_write_big_bytes), bl.length()); | |
8195 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 65u); | |
8196 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), 1u); | |
20effc67 TL |
8197 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_writes), 1u); |
8198 | ASSERT_EQ(logger->get(l_bluestore_issued_deferred_write_bytes), 61440); | |
522d829b TL |
8199 | } |
8200 | { | |
8201 | ObjectStore::Transaction t; | |
8202 | t.remove(cid, hoid); | |
8203 | t.remove_collection(cid); | |
8204 | cerr << "Cleaning" << std::endl; | |
8205 | r = queue_transaction(store, ch, std::move(t)); | |
8206 | ASSERT_EQ(r, 0); | |
8207 | } | |
8208 | } | |
f67539c2 TL |
8209 | |
8210 | TEST_P(StoreTestSpecificAUSize, DeferredDifferentChunks) { | |
8211 | ||
8212 | if (string(GetParam()) != "bluestore") | |
8213 | return; | |
20effc67 TL |
8214 | if (smr) { |
8215 | cout << "SKIP: no deferred" << std::endl; | |
8216 | return; | |
8217 | } | |
f67539c2 TL |
8218 | |
8219 | size_t alloc_size = 4096; | |
8220 | size_t large_object_size = 1 * 1024 * 1024; | |
522d829b | 8221 | size_t prefer_deferred_size = 65536; |
f67539c2 TL |
8222 | StartDeferred(alloc_size); |
8223 | SetVal(g_conf(), "bluestore_max_blob_size", "131072"); | |
522d829b TL |
8224 | SetVal(g_conf(), "bluestore_prefer_deferred_size", |
8225 | stringify(prefer_deferred_size).c_str()); | |
f67539c2 TL |
8226 | g_conf().apply_changes(nullptr); |
8227 | ||
8228 | int r; | |
8229 | coll_t cid; | |
8230 | const PerfCounters* logger = store->get_perf_counters(); | |
8231 | size_t exp_bluestore_write_big = 0; | |
8232 | size_t exp_bluestore_write_big_deferred = 0; | |
8233 | ||
8234 | ObjectStore::CollectionHandle ch = store->create_new_collection(cid); | |
8235 | { | |
8236 | ObjectStore::Transaction t; | |
8237 | t.create_collection(cid, 0); | |
8238 | r = queue_transaction(store, ch, std::move(t)); | |
8239 | ASSERT_EQ(r, 0); | |
8240 | } | |
522d829b | 8241 | for (size_t expected_write_size = 1024; expected_write_size <= prefer_deferred_size; expected_write_size *= 2) { |
f67539c2 TL |
8242 | //create object with hint |
8243 | ghobject_t hoid(hobject_t("test-"+to_string(expected_write_size), "", CEPH_NOSNAP, 0, -1, "")); | |
8244 | { | |
8245 | ObjectStore::Transaction t; | |
8246 | t.touch(cid, hoid); | |
8247 | t.set_alloc_hint(cid, hoid, large_object_size, expected_write_size, | |
8248 | CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ | | |
8249 | CEPH_OSD_ALLOC_HINT_FLAG_APPEND_ONLY); | |
8250 | r = queue_transaction(store, ch, std::move(t)); | |
8251 | ASSERT_EQ(r, 0); | |
8252 | } | |
8253 | ||
8254 | //fill object | |
8255 | { | |
8256 | ObjectStore::Transaction t; | |
8257 | bufferlist bl; | |
8258 | bl.append(std::string(large_object_size, 'h')); | |
8259 | t.write(cid, hoid, 0, bl.length(), bl, | |
8260 | CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8261 | r = queue_transaction(store, ch, std::move(t)); | |
8262 | ++exp_bluestore_write_big; | |
8263 | ASSERT_EQ(r, 0); | |
8264 | } | |
8265 | ASSERT_EQ(logger->get(l_bluestore_write_big), exp_bluestore_write_big); | |
8266 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), exp_bluestore_write_big_deferred); | |
8267 | ||
8268 | // check whether write will properly use deferred | |
8269 | { | |
8270 | ObjectStore::Transaction t; | |
8271 | bufferlist bl; | |
8272 | bl.append(std::string(alloc_size + 2, 'z')); | |
8273 | t.write(cid, hoid, large_object_size - 2 * alloc_size - 1, bl.length(), bl, | |
8274 | CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); | |
8275 | r = queue_transaction(store, ch, std::move(t)); | |
8276 | ++exp_bluestore_write_big; | |
522d829b TL |
8277 | if (expected_write_size < prefer_deferred_size) |
8278 | ++exp_bluestore_write_big_deferred; | |
f67539c2 TL |
8279 | ASSERT_EQ(r, 0); |
8280 | } | |
8281 | ASSERT_EQ(logger->get(l_bluestore_write_big), exp_bluestore_write_big); | |
8282 | ASSERT_EQ(logger->get(l_bluestore_write_big_deferred), exp_bluestore_write_big_deferred); | |
8283 | } | |
8284 | ch.reset(nullptr); | |
8285 | CloseAndReopen(); | |
8286 | ch = store->open_collection(cid); | |
8287 | // check values | |
8288 | for (size_t expected_write_size = 1024; expected_write_size <= 65536; expected_write_size *= 2) { | |
8289 | ghobject_t hoid(hobject_t("test-"+to_string(expected_write_size), "", CEPH_NOSNAP, 0, -1, "")); | |
8290 | { | |
8291 | bufferlist bl, expected; | |
8292 | r = store->read(ch, hoid, 0, large_object_size, bl); | |
8293 | ASSERT_EQ(r, large_object_size); | |
8294 | expected.append(string(large_object_size - 2 * alloc_size - 1, 'h')); | |
8295 | expected.append(string(alloc_size + 2, 'z')); | |
8296 | expected.append(string(alloc_size - 1, 'h')); | |
8297 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8298 | } | |
8299 | } | |
8300 | { | |
8301 | ObjectStore::Transaction t; | |
8302 | for (size_t expected_write_size = 1024; expected_write_size <= 65536; expected_write_size *= 2) { | |
8303 | ghobject_t hoid(hobject_t("test-"+to_string(expected_write_size), "", CEPH_NOSNAP, 0, -1, "")); | |
8304 | t.remove(cid, hoid); | |
8305 | } | |
8306 | t.remove_collection(cid); | |
8307 | cerr << "Cleaning" << std::endl; | |
8308 | r = queue_transaction(store, ch, std::move(t)); | |
8309 | ASSERT_EQ(r, 0); | |
8310 | } | |
8311 | } | |
8312 | ||
8313 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) { | |
8314 | ||
8315 | if (string(GetParam()) != "bluestore") | |
8316 | return; | |
20effc67 TL |
8317 | if (smr) { |
8318 | cout << "SKIP: no overwrite" << std::endl; | |
8319 | return; | |
8320 | } | |
f67539c2 TL |
8321 | |
8322 | size_t block_size = 4096; | |
8323 | StartDeferred(block_size); | |
8324 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); | |
8325 | g_conf().apply_changes(nullptr); | |
8326 | ||
8327 | int r; | |
8328 | coll_t cid; | |
8329 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
8330 | ||
8331 | auto ch = store->create_new_collection(cid); | |
8332 | ||
8333 | const PerfCounters* logger = store->get_perf_counters(); | |
8334 | { | |
8335 | ObjectStore::Transaction t; | |
8336 | t.create_collection(cid, 0); | |
8337 | r = queue_transaction(store, ch, std::move(t)); | |
8338 | ASSERT_EQ(r, 0); | |
8339 | } | |
8340 | { | |
8341 | ObjectStore::Transaction t; | |
8342 | bufferlist bl; | |
8343 | ||
8344 | bl.append(std::string(block_size * 2, 'a')); | |
8345 | t.write(cid, hoid, block_size * 10, bl.length(), bl, | |
8346 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8347 | r = queue_transaction(store, ch, std::move(t)); | |
8348 | ASSERT_EQ(r, 0); | |
8349 | } | |
8350 | { | |
8351 | // prepend existing | |
8352 | ObjectStore::Transaction t; | |
8353 | bufferlist bl; | |
8354 | ||
8355 | bl.append(std::string(block_size, 'b')); | |
8356 | t.write(cid, hoid, block_size * 9, bl.length(), bl, | |
8357 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8358 | r = queue_transaction(store, ch, std::move(t)); | |
8359 | ASSERT_EQ(r, 0); | |
8360 | } | |
8361 | { | |
8362 | // We need to issue a read to trigger cache stat update that refresh | |
8363 | // perf counters. additionally we need to wait some time for mempool | |
8364 | // thread to update stats. | |
8365 | sleep(1); | |
8366 | bufferlist bl, expected; | |
8367 | r = store->read(ch, hoid, block_size * 9, block_size * 2, bl); | |
8368 | ASSERT_EQ(r, (int)block_size * 2); | |
8369 | expected.append(string(block_size, 'b')); | |
8370 | expected.append(string(block_size, 'a')); | |
8371 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8372 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
8373 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
8374 | } | |
8375 | ||
8376 | ||
8377 | { | |
8378 | // prepend existing with a gap | |
8379 | ObjectStore::Transaction t; | |
8380 | bufferlist bl; | |
8381 | ||
8382 | bl.append(std::string(block_size, 'c')); | |
8383 | t.write(cid, hoid, block_size * 7, bl.length(), bl, | |
8384 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8385 | r = queue_transaction(store, ch, std::move(t)); | |
8386 | ASSERT_EQ(r, 0); | |
8387 | } | |
8388 | { | |
8389 | // We need to issue a read to trigger cache stat update that refresh | |
8390 | // perf counters. additionally we need to wait some time for mempool | |
8391 | // thread to update stats. | |
8392 | sleep(1); | |
8393 | bufferlist bl, expected; | |
8394 | r = store->read(ch, hoid, block_size * 7, block_size * 3, bl); | |
8395 | ASSERT_EQ(r, (int)block_size * 3); | |
8396 | expected.append(string(block_size, 'c')); | |
8397 | expected.append(string(block_size, 0)); | |
8398 | expected.append(string(block_size, 'b')); | |
8399 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8400 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
8401 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
8402 | } | |
8403 | ||
8404 | { | |
8405 | // append after existing with a gap | |
8406 | ObjectStore::Transaction t; | |
8407 | bufferlist bl; | |
8408 | ||
8409 | bl.append(std::string(block_size, 'd')); | |
8410 | t.write(cid, hoid, block_size * 13, bl.length(), bl, | |
8411 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8412 | r = queue_transaction(store, ch, std::move(t)); | |
8413 | ASSERT_EQ(r, 0); | |
8414 | } | |
8415 | { | |
8416 | // We need to issue a read to trigger cache stat update that refresh | |
8417 | // perf counters. additionally we need to wait some time for mempool | |
8418 | // thread to update stats. | |
8419 | sleep(1); | |
8420 | bufferlist bl, expected; | |
8421 | r = store->read(ch, hoid, block_size * 11, block_size * 3, bl); | |
8422 | ASSERT_EQ(r, (int)block_size * 3); | |
8423 | expected.append(string(block_size, 'a')); | |
8424 | expected.append(string(block_size, 0)); | |
8425 | expected.append(string(block_size, 'd')); | |
8426 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8427 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
8428 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
8429 | } | |
8430 | ||
8431 | { | |
8432 | // append twice to the next max_blob slot | |
8433 | ObjectStore::Transaction t; | |
8434 | bufferlist bl; | |
8435 | ||
8436 | bl.append(std::string(block_size, 'e')); | |
8437 | t.write(cid, hoid, block_size * 17, bl.length(), bl, | |
8438 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8439 | t.write(cid, hoid, block_size * 19, bl.length(), bl, | |
8440 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8441 | r = queue_transaction(store, ch, std::move(t)); | |
8442 | ASSERT_EQ(r, 0); | |
8443 | } | |
8444 | { | |
8445 | // We need to issue a read to trigger cache stat update that refresh | |
8446 | // perf counters. additionally we need to wait some time for mempool | |
8447 | // thread to update stats. | |
8448 | sleep(1); | |
8449 | bufferlist bl, expected; | |
8450 | r = store->read(ch, hoid, block_size * 17, block_size * 3, bl); | |
8451 | ASSERT_EQ(r, (int)block_size * 3); | |
8452 | expected.append(string(block_size, 'e')); | |
8453 | expected.append(string(block_size, 0)); | |
8454 | expected.append(string(block_size, 'e')); | |
8455 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8456 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
8457 | ASSERT_EQ(logger->get(l_bluestore_extents), 5u); | |
7c673cae FG |
8458 | } |
8459 | { | |
8460 | // fill gaps at the second slot | |
8461 | ObjectStore::Transaction t; | |
8462 | bufferlist bl; | |
8463 | ||
8464 | bl.append(std::string(block_size, 'f')); | |
8465 | t.write(cid, hoid, block_size * 16, bl.length(), bl, | |
8466 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8467 | t.write(cid, hoid, block_size * 18, bl.length(), bl, | |
8468 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 8469 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8470 | ASSERT_EQ(r, 0); |
8471 | } | |
8472 | { | |
8473 | // We need to issue a read to trigger cache stat update that refresh | |
8474 | // perf counters. additionally we need to wait some time for mempool | |
8475 | // thread to update stats. | |
8476 | sleep(1); | |
8477 | bufferlist bl, expected; | |
11fdf7f2 | 8478 | r = store->read(ch, hoid, block_size * 16, block_size * 4, bl); |
7c673cae FG |
8479 | ASSERT_EQ(r, (int)block_size * 4); |
8480 | expected.append(string(block_size, 'f')); | |
8481 | expected.append(string(block_size, 'e')); | |
8482 | expected.append(string(block_size, 'f')); | |
8483 | expected.append(string(block_size, 'e')); | |
8484 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8485 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
8486 | ASSERT_EQ(logger->get(l_bluestore_extents), 4u); | |
8487 | } | |
8488 | { | |
8489 | ObjectStore::Transaction t; | |
8490 | t.remove(cid, hoid); | |
8491 | t.remove_collection(cid); | |
8492 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 8493 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8494 | ASSERT_EQ(r, 0); |
8495 | } | |
7c673cae FG |
8496 | } |
8497 | ||
8498 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) { | |
8499 | ||
8500 | if (string(GetParam()) != "bluestore") | |
8501 | return; | |
20effc67 TL |
8502 | if (smr) { |
8503 | cout << "SKIP: no overwrite" << std::endl; | |
8504 | return; | |
8505 | } | |
7c673cae FG |
8506 | |
8507 | size_t block_size = 4096; | |
8508 | StartDeferred(block_size); | |
11fdf7f2 TL |
8509 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
8510 | g_conf().apply_changes(nullptr); | |
7c673cae | 8511 | |
7c673cae FG |
8512 | int r; |
8513 | coll_t cid; | |
8514 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
8515 | ||
8516 | const PerfCounters* logger = store->get_perf_counters(); | |
11fdf7f2 | 8517 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
8518 | |
8519 | { | |
8520 | ObjectStore::Transaction t; | |
8521 | t.create_collection(cid, 0); | |
11fdf7f2 | 8522 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8523 | ASSERT_EQ(r, 0); |
8524 | } | |
8525 | { | |
8526 | ObjectStore::Transaction t; | |
8527 | bufferlist bl; | |
8528 | ||
8529 | bl.append(std::string(block_size, 'a')); | |
8530 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
8531 | t.write(cid, hoid, block_size * 2, bl.length(), bl, | |
8532 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 8533 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8534 | ASSERT_EQ(r, 0); |
8535 | } | |
8536 | { | |
8537 | // write small into the gap | |
8538 | ObjectStore::Transaction t; | |
8539 | bufferlist bl; | |
8540 | ||
8541 | bl.append(std::string(3, 'b')); | |
8542 | t.write(cid, hoid, block_size + 1, bl.length(), bl, | |
8543 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 8544 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8545 | ASSERT_EQ(r, 0); |
8546 | } | |
8547 | { | |
8548 | // We need to issue a read to trigger cache stat update that refresh | |
8549 | // perf counters. additionally we need to wait some time for mempool | |
8550 | // thread to update stats. | |
8551 | sleep(1); | |
8552 | bufferlist bl, expected; | |
11fdf7f2 | 8553 | r = store->read(ch, hoid, 0, block_size * 3, bl); |
7c673cae FG |
8554 | ASSERT_EQ(r, (int)block_size * 3); |
8555 | expected.append(string(block_size, 'a')); | |
8556 | expected.append(string(1, 0)); | |
8557 | expected.append(string(3, 'b')); | |
8558 | expected.append(string(block_size - 4, 0)); | |
8559 | expected.append(string(block_size, 'a')); | |
8560 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8561 | ||
8562 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
8563 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
8564 | } | |
8565 | { | |
8566 | ObjectStore::Transaction t; | |
8567 | t.remove(cid, hoid); | |
8568 | t.remove_collection(cid); | |
8569 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 8570 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8571 | ASSERT_EQ(r, 0); |
8572 | } | |
7c673cae FG |
8573 | } |
8574 | ||
8575 | // The test case to reproduce an issue when write happens | |
8576 | // to a zero space between the extents sharing the same spanning blob | |
8577 | // with unloaded shard map. | |
8578 | // Second extent might be filled with zeros this way due to wrong result | |
8579 | // returned by has_any_extents() call in do_write_small. The latter is caused | |
8580 | // by incompletly loaded extent map. | |
8581 | TEST_P(StoreTestSpecificAUSize, SmallWriteOnShardedExtents) { | |
8582 | if (string(GetParam()) != "bluestore") | |
8583 | return; | |
8584 | ||
8585 | size_t block_size = 0x10000; | |
8586 | StartDeferred(block_size); | |
8587 | ||
11fdf7f2 TL |
8588 | SetVal(g_conf(), "bluestore_csum_type", "xxhash64"); |
8589 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); // for sure | |
7c673cae | 8590 | |
11fdf7f2 | 8591 | g_conf().apply_changes(nullptr); |
7c673cae | 8592 | |
7c673cae FG |
8593 | int r; |
8594 | coll_t cid; | |
8595 | ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 8596 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
8597 | |
8598 | { | |
8599 | ObjectStore::Transaction t; | |
8600 | t.create_collection(cid, 0); | |
11fdf7f2 | 8601 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8602 | ASSERT_EQ(r, 0); |
8603 | } | |
8604 | { | |
8605 | //doing some tricks to have sharded extents/spanning objects | |
8606 | ObjectStore::Transaction t; | |
8607 | bufferlist bl, bl2; | |
8608 | ||
8609 | bl.append(std::string(0x80000, 'a')); | |
8610 | t.write(cid, hoid1, 0, bl.length(), bl, 0); | |
8611 | t.zero(cid, hoid1, 0x719e0, 0x75b0 ); | |
11fdf7f2 | 8612 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8613 | ASSERT_EQ(r, 0); |
8614 | ||
8615 | bl2.append(std::string(0x70000, 'b')); | |
8616 | t.write(cid, hoid1, 0, bl2.length(), bl2, 0); | |
8617 | t.zero(cid, hoid1, 0, 0x50000); | |
11fdf7f2 | 8618 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 8619 | ASSERT_EQ(r, 0); |
7c673cae FG |
8620 | |
8621 | } | |
11fdf7f2 | 8622 | ch.reset(); |
7c673cae FG |
8623 | store->umount(); |
8624 | store->mount(); | |
11fdf7f2 | 8625 | ch = store->open_collection(cid); |
7c673cae FG |
8626 | |
8627 | { | |
8628 | // do a write to zero space in between some extents sharing the same blob | |
8629 | ObjectStore::Transaction t; | |
8630 | bufferlist bl, bl2; | |
8631 | ||
8632 | bl.append(std::string(0x6520, 'c')); | |
8633 | t.write(cid, hoid1, 0x71c00, bl.length(), bl, 0); | |
8634 | ||
11fdf7f2 | 8635 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8636 | ASSERT_EQ(r, 0); |
8637 | } | |
8638 | ||
8639 | { | |
8640 | ObjectStore::Transaction t; | |
8641 | bufferlist bl, expected; | |
8642 | ||
11fdf7f2 | 8643 | r = store->read(ch, hoid1, 0x70000, 0x9c00, bl); |
7c673cae FG |
8644 | ASSERT_EQ(r, (int)0x9c00); |
8645 | expected.append(string(0x19e0, 'a')); | |
8646 | expected.append(string(0x220, 0)); | |
8647 | expected.append(string(0x6520, 'c')); | |
8648 | expected.append(string(0xe70, 0)); | |
8649 | expected.append(string(0xc70, 'a')); | |
8650 | ASSERT_TRUE(bl_eq(expected, bl)); | |
8651 | bl.clear(); | |
8652 | ||
8653 | } | |
8654 | ||
8655 | { | |
8656 | ObjectStore::Transaction t; | |
8657 | t.remove(cid, hoid1); | |
8658 | t.remove_collection(cid); | |
8659 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 8660 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8661 | ASSERT_EQ(r, 0); |
8662 | } | |
7c673cae FG |
8663 | } |
8664 | ||
11fdf7f2 | 8665 | #endif //#if defined(WITH_BLUESTORE) |
7c673cae FG |
8666 | |
8667 | TEST_P(StoreTest, KVDBHistogramTest) { | |
8668 | if (string(GetParam()) != "bluestore") | |
8669 | return; | |
8670 | ||
7c673cae FG |
8671 | int NUM_OBJS = 200; |
8672 | int r = 0; | |
8673 | coll_t cid; | |
8674 | string base("testobj."); | |
8675 | bufferlist a; | |
8676 | bufferptr ap(0x1000); | |
8677 | memset(ap.c_str(), 'a', 0x1000); | |
8678 | a.append(ap); | |
11fdf7f2 | 8679 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
8680 | { |
8681 | ObjectStore::Transaction t; | |
8682 | t.create_collection(cid, 0); | |
11fdf7f2 | 8683 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8684 | ASSERT_EQ(r, 0); |
8685 | } | |
8686 | for (int i = 0; i < NUM_OBJS; ++i) { | |
8687 | ObjectStore::Transaction t; | |
8688 | char buf[100]; | |
8689 | snprintf(buf, sizeof(buf), "%d", i); | |
8690 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
8691 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 8692 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8693 | ASSERT_EQ(r, 0); |
8694 | } | |
8695 | ||
f67539c2 TL |
8696 | std::unique_ptr<Formatter> f(Formatter::create("store_test", "json-pretty", "json-pretty")); |
8697 | store->generate_db_histogram(f.get()); | |
7c673cae FG |
8698 | f->flush(cout); |
8699 | cout << std::endl; | |
8700 | } | |
8701 | ||
8702 | TEST_P(StoreTest, KVDBStatsTest) { | |
8703 | if (string(GetParam()) != "bluestore") | |
8704 | return; | |
8705 | ||
11fdf7f2 TL |
8706 | SetVal(g_conf(), "rocksdb_perf", "true"); |
8707 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
8708 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
8709 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
8710 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae FG |
8711 | int r = store->umount(); |
8712 | ASSERT_EQ(r, 0); | |
8713 | r = store->mount(); //to force rocksdb stats | |
8714 | ASSERT_EQ(r, 0); | |
8715 | ||
7c673cae FG |
8716 | int NUM_OBJS = 200; |
8717 | coll_t cid; | |
8718 | string base("testobj."); | |
8719 | bufferlist a; | |
8720 | bufferptr ap(0x1000); | |
8721 | memset(ap.c_str(), 'a', 0x1000); | |
8722 | a.append(ap); | |
11fdf7f2 | 8723 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
8724 | { |
8725 | ObjectStore::Transaction t; | |
8726 | t.create_collection(cid, 0); | |
11fdf7f2 | 8727 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8728 | ASSERT_EQ(r, 0); |
8729 | } | |
8730 | for (int i = 0; i < NUM_OBJS; ++i) { | |
8731 | ObjectStore::Transaction t; | |
8732 | char buf[100]; | |
8733 | snprintf(buf, sizeof(buf), "%d", i); | |
8734 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
8735 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 8736 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8737 | ASSERT_EQ(r, 0); |
8738 | } | |
8739 | ||
f67539c2 TL |
8740 | std::unique_ptr<Formatter> f(Formatter::create("store_test", "json-pretty", "json-pretty")); |
8741 | store->get_db_statistics(f.get()); | |
7c673cae FG |
8742 | f->flush(cout); |
8743 | cout << std::endl; | |
7c673cae FG |
8744 | } |
8745 | ||
11fdf7f2 | 8746 | #if defined(WITH_BLUESTORE) |
7c673cae | 8747 | TEST_P(StoreTestSpecificAUSize, garbageCollection) { |
7c673cae FG |
8748 | int r; |
8749 | coll_t cid; | |
8750 | int buf_len = 256 * 1024; | |
8751 | int overlap_offset = 64 * 1024; | |
8752 | int write_offset = buf_len; | |
8753 | if (string(GetParam()) != "bluestore") | |
8754 | return; | |
20effc67 TL |
8755 | if (smr) { |
8756 | cout << "SKIP: assertions about allocations need to be adjusted" << std::endl; | |
8757 | return; | |
8758 | } | |
7c673cae FG |
8759 | |
8760 | #define WRITE_AT(offset, _length) {\ | |
8761 | ObjectStore::Transaction t;\ | |
8762 | if ((uint64_t)_length != bl.length()) { \ | |
8763 | buffer::ptr p(bl.c_str(), _length);\ | |
8764 | bufferlist bl_tmp;\ | |
8765 | bl_tmp.push_back(p);\ | |
8766 | t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\ | |
8767 | } else {\ | |
8768 | t.write(cid, hoid, offset, bl.length(), bl);\ | |
8769 | }\ | |
11fdf7f2 | 8770 | r = queue_transaction(store, ch, std::move(t));\ |
7c673cae FG |
8771 | ASSERT_EQ(r, 0);\ |
8772 | } | |
8773 | ||
8774 | StartDeferred(65536); | |
8775 | ||
11fdf7f2 TL |
8776 | SetVal(g_conf(), "bluestore_compression_max_blob_size", "524288"); |
8777 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
8778 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
8779 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
8780 | g_conf().apply_changes(nullptr); | |
8781 | ||
8782 | auto ch = store->create_new_collection(cid); | |
7c673cae FG |
8783 | |
8784 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
8785 | { | |
8786 | bufferlist in; | |
11fdf7f2 | 8787 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
8788 | ASSERT_EQ(-ENOENT, r); |
8789 | } | |
8790 | { | |
8791 | ObjectStore::Transaction t; | |
8792 | t.create_collection(cid, 0); | |
8793 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 8794 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8795 | ASSERT_EQ(r, 0); |
8796 | } | |
8797 | ||
8798 | std::string data; | |
8799 | data.resize(buf_len); | |
8800 | ||
8801 | { | |
8802 | { | |
11fdf7f2 | 8803 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
8804 | ASSERT_TRUE(!exists); |
8805 | ||
8806 | ObjectStore::Transaction t; | |
8807 | t.touch(cid, hoid); | |
8808 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 8809 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8810 | ASSERT_EQ(r, 0); |
8811 | ||
11fdf7f2 | 8812 | exists = store->exists(ch, hoid); |
7c673cae FG |
8813 | ASSERT_EQ(true, exists); |
8814 | } | |
8815 | bufferlist bl; | |
8816 | ||
8817 | for(size_t i = 0; i < data.size(); i++) | |
8818 | data[i] = i % 256; | |
8819 | ||
8820 | bl.append(data); | |
8821 | ||
8822 | { | |
8823 | struct store_statfs_t statfs; | |
8824 | WRITE_AT(0, buf_len); | |
8825 | int r = store->statfs(&statfs); | |
8826 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8827 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
8828 | } |
8829 | { | |
8830 | struct store_statfs_t statfs; | |
8831 | WRITE_AT(write_offset - 2 * overlap_offset, buf_len); | |
8832 | int r = store->statfs(&statfs); | |
8833 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8834 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
8835 | const PerfCounters* counters = store->get_perf_counters(); |
8836 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0u); | |
8837 | } | |
8838 | ||
8839 | { | |
8840 | struct store_statfs_t statfs; | |
8841 | WRITE_AT(write_offset - overlap_offset, buf_len); | |
8842 | int r = store->statfs(&statfs); | |
8843 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8844 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
8845 | const PerfCounters* counters = store->get_perf_counters(); |
8846 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x10000u); | |
8847 | } | |
8848 | { | |
8849 | struct store_statfs_t statfs; | |
8850 | WRITE_AT(write_offset - 3 * overlap_offset, buf_len); | |
8851 | int r = store->statfs(&statfs); | |
8852 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8853 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
8854 | const PerfCounters* counters = store->get_perf_counters(); |
8855 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
8856 | } | |
8857 | { | |
8858 | struct store_statfs_t statfs; | |
8859 | WRITE_AT(write_offset + 1, overlap_offset-1); | |
8860 | int r = store->statfs(&statfs); | |
8861 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8862 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
8863 | const PerfCounters* counters = store->get_perf_counters(); |
8864 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
8865 | } | |
8866 | { | |
8867 | struct store_statfs_t statfs; | |
8868 | WRITE_AT(write_offset + 1, overlap_offset); | |
8869 | int r = store->statfs(&statfs); | |
8870 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8871 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
8872 | const PerfCounters* counters = store->get_perf_counters(); |
8873 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x3ffffu); | |
8874 | } | |
8875 | { | |
8876 | struct store_statfs_t statfs; | |
8877 | WRITE_AT(0, buf_len-1); | |
8878 | int r = store->statfs(&statfs); | |
8879 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8880 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
8881 | const PerfCounters* counters = store->get_perf_counters(); |
8882 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
8883 | } | |
11fdf7f2 | 8884 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0 |
7c673cae FG |
8885 | { |
8886 | struct store_statfs_t statfs; | |
8887 | WRITE_AT(1, overlap_offset-2); | |
8888 | WRITE_AT(overlap_offset * 2 + 1, overlap_offset-2); | |
8889 | int r = store->statfs(&statfs); | |
8890 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8891 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
8892 | const PerfCounters* counters = store->get_perf_counters(); |
8893 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
8894 | } | |
8895 | { | |
8896 | struct store_statfs_t statfs; | |
8897 | WRITE_AT(overlap_offset + 1, overlap_offset-2); | |
8898 | int r = store->statfs(&statfs); | |
8899 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 8900 | ASSERT_EQ(statfs.data_compressed_allocated, 0x0); |
7c673cae FG |
8901 | const PerfCounters* counters = store->get_perf_counters(); |
8902 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40007u); | |
8903 | } | |
8904 | { | |
8905 | ObjectStore::Transaction t; | |
8906 | t.remove(cid, hoid); | |
8907 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 8908 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
8909 | ASSERT_EQ(r, 0); |
8910 | } | |
8911 | } | |
7c673cae | 8912 | } |
7c673cae | 8913 | |
b32b8144 FG |
8914 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice) { |
8915 | if (string(GetParam()) != "bluestore") | |
8916 | return; | |
8917 | ||
11fdf7f2 TL |
8918 | SetVal(g_conf(), "bluestore_block_size", |
8919 | stringify(0x280005000).c_str()); //10 Gb + 4K | |
8920 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
8921 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
8922 | StartDeferred(0x4000); |
8923 | store->umount(); | |
8924 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
8925 | store->mount(); | |
8926 | ||
b32b8144 FG |
8927 | } |
8928 | ||
8929 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice2) { | |
8930 | if (string(GetParam()) != "bluestore") | |
8931 | return; | |
8932 | ||
11fdf7f2 TL |
8933 | SetVal(g_conf(), "bluestore_block_size", |
8934 | stringify(0x280005000).c_str()); //10 Gb + 20K | |
8935 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
8936 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
8937 | StartDeferred(0x1000); |
8938 | store->umount(); | |
8939 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
8940 | store->mount(); | |
11fdf7f2 TL |
8941 | } |
8942 | ||
8943 | namespace { | |
8944 | ghobject_t make_object(const char* name, int64_t pool) { | |
8945 | sobject_t soid{name, CEPH_NOSNAP}; | |
8946 | uint32_t hash = std::hash<sobject_t>{}(soid); | |
8947 | return ghobject_t{hobject_t{soid, "", hash, pool, ""}}; | |
8948 | } | |
8949 | } | |
8950 | ||
8951 | TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) { | |
8952 | if (string(GetParam()) != "bluestore") | |
8953 | return; | |
20effc67 TL |
8954 | if (smr) { |
8955 | cout << "TODO: repair mismatched write pointer (+ dead bytes mismatch)" << std::endl; | |
8956 | return; | |
8957 | } | |
11fdf7f2 TL |
8958 | const size_t offs_base = 65536 / 2; |
8959 | ||
f67539c2 TL |
8960 | |
8961 | // Now we need standalone db to pass "false free fix" section below | |
8962 | // Due to new BlueFS allocation model (single allocator for main device) | |
8963 | // it might cause "false free" blob overwrite by BlueFS/DB stuff | |
8964 | // and hence fail the test case and corrupt data. | |
8965 | // | |
8966 | ||
8967 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8968 | SetVal(g_conf(), "bluestore_block_db_size", "4294967296"); | |
8969 | ||
11fdf7f2 TL |
8970 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); |
8971 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
8972 | SetVal(g_conf(), "bluestore_max_blob_size", | |
8973 | stringify(2 * offs_base).c_str()); | |
8974 | SetVal(g_conf(), "bluestore_extent_map_shard_max_size", "12000"); | |
eafe8130 | 8975 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "false"); |
11fdf7f2 TL |
8976 | |
8977 | StartDeferred(0x10000); | |
8978 | ||
8979 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
8980 | ||
8981 | // fill the store with some data | |
8982 | const uint64_t pool = 555; | |
8983 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
8984 | auto ch = store->create_new_collection(cid); | |
8985 | ||
8986 | ghobject_t hoid = make_object("Object 1", pool); | |
8987 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
8988 | ghobject_t hoid2 = make_object("Object 2", pool); | |
8989 | ghobject_t hoid_cloned = hoid2; | |
8990 | hoid_cloned.hobj.snap = 1; | |
8991 | ghobject_t hoid3 = make_object("Object 3", pool); | |
8992 | ghobject_t hoid3_cloned = hoid3; | |
8993 | hoid3_cloned.hobj.snap = 1; | |
8994 | bufferlist bl; | |
8995 | bl.append("1234512345"); | |
8996 | int r; | |
8997 | const size_t repeats = 16; | |
8998 | { | |
8999 | auto ch = store->create_new_collection(cid); | |
9000 | cerr << "create collection + write" << std::endl; | |
9001 | ObjectStore::Transaction t; | |
9002 | t.create_collection(cid, 0); | |
9003 | for( auto i = 0ul; i < repeats; ++i ) { | |
9004 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
9005 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
9006 | } | |
9007 | for( auto i = 0ul; i < repeats; ++i ) { | |
9008 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
9009 | } | |
9010 | t.clone(cid, hoid2, hoid_cloned); | |
9011 | ||
9012 | r = queue_transaction(store, ch, std::move(t)); | |
9013 | ASSERT_EQ(r, 0); | |
9014 | } | |
9015 | ||
9016 | bstore->umount(); | |
20effc67 | 9017 | bool err_was_injected = false; |
11fdf7f2 TL |
9018 | //////////// leaked pextent fix //////////// |
9019 | cerr << "fix leaked pextents" << std::endl; | |
9020 | ASSERT_EQ(bstore->fsck(false), 0); | |
9021 | ASSERT_EQ(bstore->repair(false), 0); | |
9022 | bstore->mount(); | |
20effc67 TL |
9023 | if (!bstore->has_null_fm()) { |
9024 | bstore->inject_leaked(0x30000); | |
9025 | err_was_injected = true; | |
9026 | } | |
9027 | ||
11fdf7f2 | 9028 | bstore->umount(); |
20effc67 TL |
9029 | if (err_was_injected) { |
9030 | ASSERT_EQ(bstore->fsck(false), 1); | |
9031 | } | |
11fdf7f2 TL |
9032 | ASSERT_EQ(bstore->repair(false), 0); |
9033 | ASSERT_EQ(bstore->fsck(false), 0); | |
9034 | ||
9035 | //////////// false free fix //////////// | |
9036 | cerr << "fix false free pextents" << std::endl; | |
9037 | bstore->mount(); | |
20effc67 TL |
9038 | if (!bstore->has_null_fm()) { |
9039 | bstore->inject_false_free(cid, hoid); | |
9040 | err_was_injected = true; | |
9041 | } | |
11fdf7f2 | 9042 | bstore->umount(); |
20effc67 TL |
9043 | if (err_was_injected) { |
9044 | ASSERT_EQ(bstore->fsck(false), 2); | |
9045 | ASSERT_EQ(bstore->repair(false), 0); | |
9046 | } | |
11fdf7f2 TL |
9047 | ASSERT_EQ(bstore->fsck(false), 0); |
9048 | ||
9049 | //////////// verify invalid statfs /////////// | |
9050 | cerr << "fix invalid statfs" << std::endl; | |
9051 | store_statfs_t statfs0, statfs; | |
9052 | bstore->mount(); | |
9053 | ASSERT_EQ(bstore->statfs(&statfs0), 0); | |
9054 | statfs = statfs0; | |
9055 | statfs.allocated += 0x10000; | |
9056 | statfs.data_stored += 0x10000; | |
9057 | ASSERT_FALSE(statfs0 == statfs); | |
9058 | bstore->inject_statfs("bluestore_statfs", statfs); | |
9059 | bstore->umount(); | |
9060 | ||
eafe8130 | 9061 | ASSERT_EQ(bstore->fsck(false), 2); |
11fdf7f2 TL |
9062 | ASSERT_EQ(bstore->repair(false), 0); |
9063 | ASSERT_EQ(bstore->fsck(false), 0); | |
9064 | ASSERT_EQ(bstore->mount(), 0); | |
9065 | ASSERT_EQ(bstore->statfs(&statfs), 0); | |
f67539c2 | 9066 | // adjust free/internal meta space to success in comparison |
11fdf7f2 | 9067 | statfs0.available = statfs.available; |
f67539c2 | 9068 | statfs0.internal_metadata = statfs.internal_metadata; |
11fdf7f2 TL |
9069 | ASSERT_EQ(statfs0, statfs); |
9070 | ||
9071 | ///////// undecodable shared blob key / stray shared blob records /////// | |
9072 | cerr << "undecodable shared blob key" << std::endl; | |
9073 | bstore->inject_broken_shared_blob_key("undec1", | |
9074 | bufferlist()); | |
9075 | bstore->inject_broken_shared_blob_key("undecodable key 2", | |
9076 | bufferlist()); | |
9077 | bstore->inject_broken_shared_blob_key("undecodable key 3", | |
9078 | bufferlist()); | |
9079 | bstore->umount(); | |
9080 | ASSERT_EQ(bstore->fsck(false), 3); | |
9081 | ASSERT_EQ(bstore->repair(false), 0); | |
9082 | ASSERT_EQ(bstore->fsck(false), 0); | |
9083 | ||
9084 | cerr << "misreferencing" << std::endl; | |
9085 | bstore->mount(); | |
9086 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, 0); | |
9087 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, (offs_base * repeats) / 2); | |
9088 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, offs_base * (repeats -1) ); | |
20effc67 | 9089 | int expected_errors = bstore->has_null_fm() ? 3 : 6; |
11fdf7f2 | 9090 | bstore->umount(); |
20effc67 | 9091 | ASSERT_EQ(bstore->fsck(false), expected_errors); |
11fdf7f2 TL |
9092 | ASSERT_EQ(bstore->repair(false), 0); |
9093 | ||
9094 | ASSERT_EQ(bstore->fsck(true), 0); | |
9095 | ||
9096 | // reproducing issues #21040 & 20983 | |
9097 | SetVal(g_conf(), "bluestore_debug_inject_bug21040", "true"); | |
9098 | g_ceph_context->_conf.apply_changes(nullptr); | |
9099 | bstore->mount(); | |
9100 | ||
9101 | cerr << "repro bug #21040" << std::endl; | |
9102 | { | |
9103 | auto ch = store->open_collection(cid); | |
9104 | { | |
9105 | ObjectStore::Transaction t; | |
9106 | bl.append("0123456789012345"); | |
9107 | t.write(cid, hoid3, offs_base, bl.length(), bl); | |
9108 | bl.clear(); | |
9109 | bl.append('!'); | |
9110 | t.write(cid, hoid3, 0, bl.length(), bl); | |
9111 | ||
9112 | r = queue_transaction(store, ch, std::move(t)); | |
9113 | ASSERT_EQ(r, 0); | |
9114 | } | |
9115 | { | |
9116 | ObjectStore::Transaction t; | |
9117 | t.clone(cid, hoid3, hoid3_cloned); | |
9118 | r = queue_transaction(store, ch, std::move(t)); | |
9119 | ASSERT_EQ(r, 0); | |
9120 | } | |
9121 | ||
9122 | bstore->umount(); | |
20effc67 | 9123 | ASSERT_EQ(bstore->fsck(false), 4); |
11fdf7f2 TL |
9124 | ASSERT_LE(bstore->repair(false), 0); |
9125 | ASSERT_EQ(bstore->fsck(false), 0); | |
9126 | } | |
9127 | ||
adb31ebb TL |
9128 | cerr << "Zombie spanning blob" << std::endl; |
9129 | { | |
9130 | bstore->mount(); | |
9131 | ghobject_t hoid4 = make_object("Object 4", pool); | |
9132 | auto ch = store->open_collection(cid); | |
9133 | { | |
9134 | bufferlist bl; | |
9135 | string s(0x1000, 'a'); | |
9136 | bl.append(s); | |
9137 | ObjectStore::Transaction t; | |
9138 | for(size_t i = 0; i < 0x10; i++) { | |
9139 | t.write(cid, hoid4, i * bl.length(), bl.length(), bl); | |
9140 | } | |
9141 | r = queue_transaction(store, ch, std::move(t)); | |
9142 | ASSERT_EQ(r, 0); | |
9143 | } | |
9144 | sleep(5); | |
9145 | { | |
9146 | bstore->inject_zombie_spanning_blob(cid, hoid4, 12345); | |
9147 | bstore->inject_zombie_spanning_blob(cid, hoid4, 23456); | |
9148 | bstore->inject_zombie_spanning_blob(cid, hoid4, 23457); | |
9149 | } | |
9150 | ||
9151 | bstore->umount(); | |
9152 | ASSERT_EQ(bstore->fsck(false), 1); | |
9153 | ASSERT_LE(bstore->repair(false), 0); | |
9154 | ASSERT_EQ(bstore->fsck(false), 0); | |
9155 | } | |
9156 | ||
eafe8130 TL |
9157 | cerr << "Completing" << std::endl; |
9158 | bstore->mount(); | |
9159 | ||
9160 | } | |
9161 | ||
b3b6e05e TL |
9162 | TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) { |
9163 | if (string(GetParam()) != "bluestore") | |
9164 | return; | |
20effc67 TL |
9165 | if (smr) { |
9166 | cout << "SKIP: smr repair is different" << std::endl; | |
9167 | return; | |
9168 | } | |
b3b6e05e TL |
9169 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); |
9170 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
9171 | ||
9172 | StartDeferred(0x10000); | |
9173 | ||
9174 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9175 | ||
9176 | int r; | |
9177 | ||
9178 | cerr << "initializing" << std::endl; | |
9179 | { | |
9180 | const size_t col_count = 16; | |
9181 | const size_t obj_count = 1024; | |
9182 | ObjectStore::CollectionHandle ch[col_count]; | |
9183 | ghobject_t hoid[col_count][obj_count]; | |
9184 | ||
9185 | unique_ptr<coll_t> cid[col_count]; | |
9186 | ||
9187 | for (size_t i = 0; i < col_count; i++) { | |
9188 | cid[i].reset(new coll_t(spg_t(pg_t(0, i), shard_id_t::NO_SHARD))); | |
9189 | ch[i] = store->create_new_collection(*cid[i]); | |
9190 | for (size_t j = 0; j < obj_count; j++) { | |
9191 | hoid[i][j] = make_object(stringify(j).c_str(), i); | |
9192 | } | |
9193 | } | |
9194 | ||
9195 | for (size_t i = 0; i < col_count; i++) { | |
9196 | ObjectStore::Transaction t; | |
9197 | t.create_collection(*cid[i], 0); | |
9198 | r = queue_transaction(store, ch[i], std::move(t)); | |
9199 | ASSERT_EQ(r, 0); | |
9200 | } | |
9201 | cerr << "onode preparing" << std::endl; | |
9202 | bufferlist bl; | |
9203 | string s(0x1000, 'a'); | |
9204 | bl.append(s); | |
9205 | ||
9206 | for (size_t i = 0; i < col_count; i++) { | |
9207 | for (size_t j = 0; j < obj_count; j++) { | |
9208 | ObjectStore::Transaction t; | |
9209 | t.write(*cid[i], hoid[i][j], bl.length(), bl.length(), bl); | |
9210 | r = queue_transaction(store, ch[i], std::move(t)); | |
9211 | ASSERT_EQ(r, 0); | |
9212 | } | |
9213 | } | |
9214 | cerr << "Zombie spanning blob injection" << std::endl; | |
9215 | ||
9216 | sleep(5); | |
9217 | ||
9218 | for (size_t i = 0; i < col_count; i++) { | |
9219 | for (size_t j = 0; j < obj_count; j++) { | |
9220 | bstore->inject_zombie_spanning_blob(*cid[i], hoid[i][j], 12345); | |
9221 | } | |
9222 | } | |
9223 | ||
9224 | cerr << "fscking/fixing" << std::endl; | |
9225 | bstore->umount(); | |
9226 | ASSERT_EQ(bstore->fsck(false), col_count * obj_count); | |
9227 | ASSERT_LE(bstore->quick_fix(), 0); | |
9228 | ASSERT_EQ(bstore->fsck(false), 0); | |
9229 | } | |
9230 | ||
9231 | cerr << "Completing" << std::endl; | |
9232 | bstore->mount(); | |
9233 | } | |
9234 | ||
20effc67 TL |
9235 | TEST_P(StoreTestSpecificAUSize, BluestoreRepairSharedBlobTest) { |
9236 | if (string(GetParam()) != "bluestore") | |
9237 | return; | |
9238 | if (smr) { | |
9239 | cout << "TODO: repair mismatched write pointer (+ dead bytes mismatch)" << std::endl; | |
9240 | return; | |
9241 | } | |
9242 | ||
9243 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
9244 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
9245 | ||
9246 | const size_t block_size = 0x1000; | |
9247 | StartDeferred(block_size); | |
9248 | ||
9249 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9250 | ||
9251 | // fill the store with some data | |
9252 | const uint64_t pool = 555; | |
9253 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9254 | auto ch = store->create_new_collection(cid); | |
9255 | ||
9256 | ghobject_t hoid = make_object("Object 1", pool); | |
9257 | ghobject_t hoid_cloned = hoid; | |
9258 | hoid_cloned.hobj.snap = 1; | |
9259 | ghobject_t hoid2 = make_object("Object 2", pool); | |
9260 | ||
9261 | string s(block_size, 1); | |
9262 | bufferlist bl; | |
9263 | bl.append(s); | |
9264 | int r; | |
9265 | { | |
9266 | ObjectStore::Transaction t; | |
9267 | t.create_collection(cid, 0); | |
9268 | r = queue_transaction(store, ch, std::move(t)); | |
9269 | ASSERT_EQ(r, 0); | |
9270 | } | |
9271 | ||
9272 | // check the scenario when shared blob contains | |
9273 | // references to extents from two objects which don't overlapp | |
9274 | // o1 -> 0x2000~1K | |
9275 | // o2 -> 0x4000~1k | |
9276 | cerr << "introduce 2 non-overlapped extents in a shared blob" | |
9277 | << std::endl; | |
9278 | { | |
9279 | ObjectStore::Transaction t; | |
9280 | t.write(cid, hoid, 0, bl.length(), bl); | |
9281 | t.write(cid, hoid2, 0, bl.length(), bl); // to make a gap in allocations | |
9282 | t.write(cid, hoid, block_size * 2 , bl.length(), bl); | |
9283 | t.clone(cid, hoid, hoid_cloned); | |
9284 | t.zero(cid, hoid, 0, bl.length()); | |
9285 | t.zero(cid, hoid_cloned, block_size * 2, bl.length()); | |
9286 | r = queue_transaction(store, ch, std::move(t)); | |
9287 | ASSERT_EQ(r, 0); | |
9288 | } | |
9289 | bstore->umount(); | |
9290 | bstore->mount(); | |
9291 | { | |
9292 | string key; | |
9293 | _key_encode_u64(1, &key); | |
9294 | bluestore_shared_blob_t sb(1); | |
9295 | sb.ref_map.get(0x2000, block_size); | |
9296 | sb.ref_map.get(0x4000, block_size); | |
9297 | sb.ref_map.get(0x4000, block_size); | |
9298 | bufferlist bl; | |
9299 | encode(sb, bl); | |
9300 | bstore->inject_broken_shared_blob_key(key, bl); | |
9301 | } | |
9302 | bstore->umount(); | |
9303 | ASSERT_EQ(bstore->fsck(false), 2); | |
9304 | ASSERT_EQ(bstore->repair(false), 0); | |
9305 | ASSERT_EQ(bstore->fsck(false), 0); | |
9306 | ||
9307 | cerr << "Completing" << std::endl; | |
9308 | bstore->mount(); | |
9309 | } | |
9310 | ||
a4b75251 TL |
9311 | TEST_P(StoreTestSpecificAUSize, BluestoreBrokenNoSharedBlobRepairTest) { |
9312 | if (string(GetParam()) != "bluestore") | |
9313 | return; | |
20effc67 TL |
9314 | if (smr) { |
9315 | cout << "SKIP: smr repair is different" << std::endl; | |
9316 | return; | |
9317 | } | |
a4b75251 TL |
9318 | |
9319 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
9320 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
9321 | ||
9322 | StartDeferred(0x10000); | |
9323 | ||
9324 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9325 | ||
9326 | int r; | |
9327 | ||
9328 | // initializing | |
9329 | cerr << "initializing" << std::endl; | |
9330 | { | |
9331 | const uint64_t pool = 555; | |
9332 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9333 | auto ch = store->create_new_collection(cid); | |
9334 | ||
9335 | ghobject_t hoid = make_object("Object", pool); | |
9336 | ghobject_t hoid_cloned = hoid; | |
9337 | hoid_cloned.hobj.snap = 1; | |
9338 | ||
9339 | { | |
9340 | ObjectStore::Transaction t; | |
9341 | t.create_collection(cid, 0); | |
9342 | r = queue_transaction(store, ch, std::move(t)); | |
9343 | ASSERT_EQ(r, 0); | |
9344 | } | |
9345 | { | |
9346 | ObjectStore::Transaction t; | |
9347 | bufferlist bl; | |
9348 | bl.append("0123456789012345"); | |
9349 | t.write(cid, hoid, 0, bl.length(), bl); | |
9350 | ||
9351 | r = queue_transaction(store, ch, std::move(t)); | |
9352 | ASSERT_EQ(r, 0); | |
9353 | } | |
9354 | { | |
9355 | ObjectStore::Transaction t; | |
9356 | t.clone(cid, hoid, hoid_cloned); | |
9357 | r = queue_transaction(store, ch, std::move(t)); | |
9358 | ASSERT_EQ(r, 0); | |
9359 | } | |
9360 | } | |
9361 | // injecting an error and checking | |
9362 | cerr << "injecting" << std::endl; | |
9363 | sleep(3); // need some time for the previous write to land | |
9364 | bstore->inject_no_shared_blob_key(); | |
20effc67 | 9365 | bstore->inject_stray_shared_blob_key(12345678); |
a4b75251 TL |
9366 | |
9367 | { | |
9368 | cerr << "fscking/fixing" << std::endl; | |
9369 | bstore->umount(); | |
20effc67 TL |
9370 | // depending on the allocation map's source we can |
9371 | // either observe or don't observe an additional | |
9372 | // extent leak detection. Hence adjusting the expected | |
9373 | // value | |
9374 | size_t expected_error_count = | |
9375 | g_ceph_context->_conf->bluestore_allocation_from_file ? | |
9376 | 5: // 4 sb ref mismatch errors + 1 statfs mismatch | |
9377 | 7; // 4 sb ref mismatch errors + 1 statfs + 1 block leak + 1 non-free | |
9378 | ASSERT_EQ(bstore->fsck(false), expected_error_count); | |
9379 | // repair might report less errors than fsck above showed | |
9380 | // as some errors, e.g. statfs mismatch, are implicitly fixed | |
9381 | // before the detection during the previous repair steps... | |
9382 | ASSERT_LE(bstore->repair(false), expected_error_count); | |
a4b75251 TL |
9383 | ASSERT_EQ(bstore->fsck(false), 0); |
9384 | } | |
9385 | ||
9386 | cerr << "Completing" << std::endl; | |
9387 | bstore->mount(); | |
9388 | } | |
9389 | ||
adb31ebb | 9390 | TEST_P(StoreTest, BluestoreRepairGlobalStats) { |
eafe8130 TL |
9391 | if (string(GetParam()) != "bluestore") |
9392 | return; | |
9393 | const size_t offs_base = 65536 / 2; | |
9394 | ||
9395 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9396 | ||
9397 | // start with global stats | |
9398 | bstore->inject_global_statfs({}); | |
9399 | bstore->umount(); | |
9400 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
9401 | bstore->mount(); | |
9402 | ||
9403 | // fill the store with some data | |
9404 | const uint64_t pool = 555; | |
9405 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9406 | auto ch = store->create_new_collection(cid); | |
9407 | ||
9408 | ghobject_t hoid = make_object("Object 1", pool); | |
9409 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
9410 | ghobject_t hoid2 = make_object("Object 2", pool); | |
9411 | ghobject_t hoid_cloned = hoid2; | |
9412 | hoid_cloned.hobj.snap = 1; | |
9413 | ghobject_t hoid3 = make_object("Object 3", pool); | |
9414 | ghobject_t hoid3_cloned = hoid3; | |
9415 | hoid3_cloned.hobj.snap = 1; | |
9416 | bufferlist bl; | |
9417 | bl.append("1234512345"); | |
9418 | int r; | |
9419 | const size_t repeats = 16; | |
9420 | { | |
9421 | auto ch = store->create_new_collection(cid); | |
9422 | cerr << "create collection + write" << std::endl; | |
9423 | ObjectStore::Transaction t; | |
9424 | t.create_collection(cid, 0); | |
9425 | for( auto i = 0ul; i < repeats; ++i ) { | |
9426 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
9427 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
9428 | } | |
9429 | for( auto i = 0ul; i < repeats; ++i ) { | |
9430 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
9431 | } | |
9432 | t.clone(cid, hoid2, hoid_cloned); | |
9433 | ||
9434 | r = queue_transaction(store, ch, std::move(t)); | |
9435 | ASSERT_EQ(r, 0); | |
9436 | } | |
9437 | ||
9438 | bstore->umount(); | |
9439 | ||
11fdf7f2 TL |
9440 | // enable per-pool stats collection hence causing fsck to fail |
9441 | cerr << "per-pool statfs" << std::endl; | |
eafe8130 | 9442 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); |
11fdf7f2 TL |
9443 | g_ceph_context->_conf.apply_changes(nullptr); |
9444 | ||
eafe8130 | 9445 | ASSERT_EQ(bstore->fsck(false), 1); |
11fdf7f2 TL |
9446 | ASSERT_EQ(bstore->repair(false), 0); |
9447 | ASSERT_EQ(bstore->fsck(false), 0); | |
9448 | ||
11fdf7f2 | 9449 | bstore->mount(); |
eafe8130 TL |
9450 | } |
9451 | ||
adb31ebb | 9452 | TEST_P(StoreTest, BluestoreRepairGlobalStatsFixOnMount) { |
eafe8130 TL |
9453 | if (string(GetParam()) != "bluestore") |
9454 | return; | |
9455 | const size_t offs_base = 65536 / 2; | |
9456 | ||
9457 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9458 | ||
9459 | // start with global stats | |
9460 | bstore->inject_global_statfs({}); | |
9461 | bstore->umount(); | |
9462 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
9463 | bstore->mount(); | |
9464 | ||
9465 | // fill the store with some data | |
9466 | const uint64_t pool = 555; | |
9467 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9468 | auto ch = store->create_new_collection(cid); | |
9469 | ||
9470 | ghobject_t hoid = make_object("Object 1", pool); | |
9471 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
9472 | ghobject_t hoid2 = make_object("Object 2", pool); | |
9473 | ghobject_t hoid_cloned = hoid2; | |
9474 | hoid_cloned.hobj.snap = 1; | |
9475 | ghobject_t hoid3 = make_object("Object 3", pool); | |
9476 | ghobject_t hoid3_cloned = hoid3; | |
9477 | hoid3_cloned.hobj.snap = 1; | |
9478 | bufferlist bl; | |
9479 | bl.append("1234512345"); | |
9480 | int r; | |
9481 | const size_t repeats = 16; | |
9482 | { | |
9483 | auto ch = store->create_new_collection(cid); | |
9484 | cerr << "create collection + write" << std::endl; | |
9485 | ObjectStore::Transaction t; | |
9486 | t.create_collection(cid, 0); | |
9487 | for( auto i = 0ul; i < repeats; ++i ) { | |
9488 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
9489 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
9490 | } | |
9491 | for( auto i = 0ul; i < repeats; ++i ) { | |
9492 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
9493 | } | |
9494 | t.clone(cid, hoid2, hoid_cloned); | |
9495 | ||
9496 | r = queue_transaction(store, ch, std::move(t)); | |
9497 | ASSERT_EQ(r, 0); | |
9498 | } | |
9499 | ||
9500 | bstore->umount(); | |
9501 | ||
9502 | // enable per-pool stats collection hence causing fsck to fail | |
9503 | cerr << "per-pool statfs" << std::endl; | |
9504 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); | |
9505 | g_ceph_context->_conf.apply_changes(nullptr); | |
9506 | ||
9507 | ASSERT_EQ(bstore->fsck(false), 1); | |
9508 | ||
9509 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
9510 | bstore->mount(); | |
9511 | bstore->umount(); | |
9512 | ASSERT_EQ(bstore->fsck(false), 0); | |
b32b8144 | 9513 | |
eafe8130 | 9514 | bstore->mount(); |
11fdf7f2 TL |
9515 | } |
9516 | ||
9517 | TEST_P(StoreTest, BluestoreStatistics) { | |
9518 | if (string(GetParam()) != "bluestore") | |
9519 | return; | |
9520 | ||
9521 | SetVal(g_conf(), "rocksdb_perf", "true"); | |
9522 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
9523 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
9524 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
9525 | ||
9526 | // disable cache | |
9527 | SetVal(g_conf(), "bluestore_cache_size_ssd", "0"); | |
9528 | SetVal(g_conf(), "bluestore_cache_size_hdd", "0"); | |
9529 | SetVal(g_conf(), "bluestore_cache_size", "0"); | |
9530 | g_ceph_context->_conf.apply_changes(nullptr); | |
9531 | ||
9532 | int r = store->umount(); | |
9533 | ASSERT_EQ(r, 0); | |
9534 | r = store->mount(); | |
9535 | ASSERT_EQ(r, 0); | |
9536 | ||
9537 | BlueStore* bstore = NULL; | |
9538 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
9539 | ||
9540 | coll_t cid; | |
9541 | ghobject_t hoid(hobject_t("test_db_statistics", "", CEPH_NOSNAP, 0, 0, "")); | |
9542 | auto ch = bstore->create_new_collection(cid); | |
9543 | bufferlist bl; | |
9544 | bl.append("0123456789abcdefghi"); | |
9545 | { | |
9546 | ObjectStore::Transaction t; | |
9547 | t.create_collection(cid, 0); | |
9548 | t.touch(cid, hoid); | |
9549 | t.write(cid, hoid, 0, bl.length(), bl); | |
9550 | cerr << "Write object" << std::endl; | |
9551 | r = queue_transaction(bstore, ch, std::move(t)); | |
9552 | ASSERT_EQ(r, 0); | |
9553 | } | |
9554 | { | |
9555 | bufferlist readback; | |
9556 | r = store->read(ch, hoid, 0, bl.length(), readback); | |
9557 | ASSERT_EQ(static_cast<int>(bl.length()), r); | |
9558 | ASSERT_TRUE(bl_eq(bl, readback)); | |
9559 | } | |
f67539c2 TL |
9560 | std::unique_ptr<Formatter> f(Formatter::create("store_test", "json-pretty", "json-pretty")); |
9561 | EXPECT_NO_THROW(store->get_db_statistics(f.get())); | |
11fdf7f2 TL |
9562 | f->flush(cout); |
9563 | cout << std::endl; | |
9564 | } | |
9565 | ||
20effc67 TL |
9566 | TEST_P(StoreTest, BluestoreStrayOmapDetection) |
9567 | { | |
9568 | if (string(GetParam()) != "bluestore") | |
9569 | return; | |
9570 | ||
9571 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9572 | const uint64_t pool = 555; | |
9573 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9574 | ghobject_t oid = make_object("Object 1", pool); | |
9575 | ghobject_t oid2 = make_object("Object 2", pool); | |
9576 | // fill the store with some data | |
9577 | auto ch = store->create_new_collection(cid); | |
9578 | bufferlist h; | |
9579 | h.append("header"); | |
9580 | { | |
9581 | ObjectStore::Transaction t; | |
9582 | t.create_collection(cid, 0); | |
9583 | t.touch(cid, oid); | |
9584 | t.omap_setheader(cid, oid, h); | |
9585 | t.touch(cid, oid2); | |
9586 | t.omap_setheader(cid, oid2, h); | |
9587 | int r = queue_transaction(store, ch, std::move(t)); | |
9588 | ASSERT_EQ(r, 0); | |
9589 | } | |
9590 | ||
9591 | // inject stray omap | |
9592 | bstore->inject_stray_omap(123456, "somename"); | |
9593 | ||
9594 | bstore->umount(); | |
9595 | // check we detect injected stray omap.. | |
9596 | ||
9597 | ASSERT_EQ(bstore->fsck(false), 1); | |
9598 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
9599 | bstore->mount(); | |
9600 | } | |
9601 | ||
9f95a23c TL |
9602 | TEST_P(StoreTest, BluestorePerPoolOmapFixOnMount) |
9603 | { | |
9604 | if (string(GetParam()) != "bluestore") | |
9605 | return; | |
9606 | ||
9607 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
9608 | const uint64_t pool = 555; | |
9609 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
9610 | ghobject_t oid = make_object("Object 1", pool); | |
9611 | ghobject_t oid2 = make_object("Object 2", pool); | |
9612 | // fill the store with some data | |
9613 | auto ch = store->create_new_collection(cid); | |
9614 | map<string, bufferlist> omap; | |
9615 | bufferlist h; | |
9616 | h.append("header"); | |
9617 | { | |
9618 | omap["omap_key"].append("omap value"); | |
9619 | ObjectStore::Transaction t; | |
9620 | t.create_collection(cid, 0); | |
9621 | t.touch(cid, oid); | |
9622 | t.omap_setheader(cid, oid, h); | |
9623 | t.touch(cid, oid2); | |
9624 | t.omap_setheader(cid, oid2, h); | |
9625 | int r = queue_transaction(store, ch, std::move(t)); | |
9626 | ASSERT_EQ(r, 0); | |
9627 | } | |
9628 | ||
9629 | // inject legacy omaps | |
9630 | bstore->inject_legacy_omap(); | |
9631 | bstore->inject_legacy_omap(cid, oid); | |
9632 | bstore->inject_legacy_omap(cid, oid2); | |
9633 | ||
9634 | bstore->umount(); | |
9635 | ||
9636 | // check we injected an issue | |
9637 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
9638 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
9639 | g_ceph_context->_conf.apply_changes(nullptr); | |
9640 | ASSERT_EQ(bstore->fsck(false), 3); | |
9641 | ||
9642 | // set autofix and mount | |
9643 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
9644 | g_ceph_context->_conf.apply_changes(nullptr); | |
9645 | bstore->mount(); | |
9646 | bstore->umount(); | |
9647 | ||
9648 | // check we fixed it.. | |
9649 | ASSERT_EQ(bstore->fsck(false), 0); | |
9650 | bstore->mount(); | |
9651 | ||
9652 | // | |
9653 | // Now repro https://tracker.ceph.com/issues/43824 | |
9654 | // | |
9655 | // inject legacy omaps again | |
9656 | bstore->inject_legacy_omap(); | |
9657 | bstore->inject_legacy_omap(cid, oid); | |
9658 | bstore->inject_legacy_omap(cid, oid2); | |
9659 | bstore->umount(); | |
9660 | ||
9661 | // check we injected an issue | |
9662 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
9663 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
9664 | g_ceph_context->_conf.apply_changes(nullptr); | |
9665 | bstore->mount(); | |
9666 | ch = store->open_collection(cid); | |
9667 | ||
9668 | { | |
9669 | // write to onode which will partiall revert per-pool | |
9670 | // omap repair done on mount due to #43824. | |
9671 | // And object removal will leave stray per-pool omap recs | |
9672 | // | |
9673 | ObjectStore::Transaction t; | |
9674 | bufferlist bl; | |
9675 | bl.append("data"); | |
9676 | //this triggers onode rec update and hence legacy omap | |
9677 | t.write(cid, oid, 0, bl.length(), bl); | |
9678 | t.remove(cid, oid2); // this will trigger stray per-pool omap | |
9679 | int r = queue_transaction(store, ch, std::move(t)); | |
9680 | ASSERT_EQ(r, 0); | |
9681 | } | |
9682 | bstore->umount(); | |
9683 | // check omap's been fixed. | |
9684 | ASSERT_EQ(bstore->fsck(false), 0); // this will fail without fix for #43824 | |
9685 | ||
9686 | bstore->mount(); | |
9687 | } | |
9688 | ||
20effc67 TL |
9689 | class hugepaged_raw; |
9690 | ||
9691 | static bool is_hugepaged(const bufferptr& bp) | |
9692 | { | |
9693 | const auto& ibp = | |
9694 | static_cast<const ceph::buffer_instrumentation::instrumented_bptr&>(bp); | |
9695 | return ibp.is_raw_marked<BlockDevice::hugepaged_raw_marker_t>(); | |
9696 | } | |
9697 | ||
9698 | // disabled by default b/c of the dependency on huge page ssome test | |
9699 | // environments might not offer without extra configuration. | |
9700 | TEST_P(StoreTestDeferredSetup, DISABLED_BluestoreHugeReads) | |
9701 | { | |
9702 | if (string(GetParam()) != "bluestore") { | |
9703 | return; | |
9704 | } | |
9705 | ||
9706 | constexpr static size_t HUGE_BUFFER_SIZE{2_M}; | |
9707 | cout << "Configuring huge page pools" << std::endl; | |
9708 | { | |
9709 | SetVal(g_conf(), "bdev_read_preallocated_huge_buffers", | |
9710 | fmt::format("{}=2", HUGE_BUFFER_SIZE).c_str()); | |
9711 | SetVal(g_conf(), "bluestore_max_blob_size", | |
9712 | std::to_string(HUGE_BUFFER_SIZE).c_str()); | |
9713 | // let's verify the per-IOContext no-cache override | |
9714 | SetVal(g_conf(), "bluestore_default_buffered_read", "true"); | |
9715 | g_ceph_context->_conf.apply_changes(nullptr); | |
9716 | } | |
9717 | DeferredSetup(); | |
9718 | ||
9719 | coll_t cid; | |
9720 | ghobject_t hoid(hobject_t("test_huge_buffers", "", CEPH_NOSNAP, 0, 0, "")); | |
9721 | auto ch = store->create_new_collection(cid); | |
9722 | ||
9723 | bufferlist bl; | |
9724 | { | |
9725 | bufferptr bp{HUGE_BUFFER_SIZE}; | |
9726 | // non-zeros! Otherwise the deduplication will take place. | |
9727 | ::memset(bp.c_str(), 0x42, HUGE_BUFFER_SIZE); | |
9728 | bl.push_back(std::move(bp)); | |
9729 | ASSERT_EQ(bl.get_num_buffers(), 1); | |
9730 | ASSERT_EQ(bl.length(), HUGE_BUFFER_SIZE); | |
9731 | } | |
9732 | ||
9733 | cout << "Write object" << std::endl; | |
9734 | { | |
9735 | ObjectStore::Transaction t; | |
9736 | t.create_collection(cid, 0); | |
9737 | t.touch(cid, hoid); | |
9738 | t.write(cid, hoid, 0, bl.length(), bl); | |
9739 | const auto r = queue_transaction(store, ch, std::move(t)); | |
9740 | ASSERT_EQ(r, 0); | |
9741 | } | |
9742 | ||
9743 | // force cache clear | |
9744 | { | |
9745 | EXPECT_EQ(store->umount(), 0); | |
9746 | EXPECT_EQ(store->mount(), 0); | |
9747 | ch = store->open_collection(cid); | |
9748 | } | |
9749 | ||
9750 | // we want to extend the life-time of all huge paged-backed | |
9751 | // bufferlists to validate the behaviour on pool exhaustion. | |
9752 | bufferlist bl_1_huge, bl_2_huge, bl_3_plain; | |
9753 | ||
9754 | cout << "Read object 1st time" << std::endl; | |
9755 | { | |
9756 | const auto r = store->read(ch, hoid, 0, HUGE_BUFFER_SIZE, bl_1_huge); | |
9757 | ASSERT_EQ(static_cast<int>(HUGE_BUFFER_SIZE), r); | |
9758 | ASSERT_TRUE(bl_eq(bl, bl_1_huge)); | |
9759 | ASSERT_EQ(bl_1_huge.get_num_buffers(), 1); | |
9760 | ASSERT_TRUE(is_hugepaged(bl_1_huge.front())); | |
9761 | } | |
9762 | ||
9763 | cout << "Read object 2nd time" << std::endl; | |
9764 | { | |
9765 | const auto r = store->read(ch, hoid, 0, HUGE_BUFFER_SIZE, bl_2_huge); | |
9766 | ASSERT_EQ(static_cast<int>(HUGE_BUFFER_SIZE), r); | |
9767 | ASSERT_TRUE(bl_eq(bl, bl_2_huge)); | |
9768 | ASSERT_EQ(bl_2_huge.get_num_buffers(), 1); | |
9769 | ASSERT_TRUE(is_hugepaged(bl_2_huge.front())); | |
9770 | } | |
9771 | ||
9772 | cout << "Read object 3rd time" << std::endl; | |
9773 | { | |
9774 | const auto r = store->read(ch, hoid, 0, HUGE_BUFFER_SIZE, bl_3_plain); | |
9775 | ASSERT_EQ(static_cast<int>(HUGE_BUFFER_SIZE), r); | |
9776 | ASSERT_TRUE(bl_eq(bl, bl_3_plain)); | |
9777 | ASSERT_EQ(bl_3_plain.get_num_buffers(), 1); | |
9778 | ASSERT_FALSE(is_hugepaged(bl_3_plain.front())); | |
9779 | } | |
9780 | } | |
9781 | ||
f64942e4 AA |
9782 | TEST_P(StoreTest, SpuriousReadErrorTest) { |
9783 | if (string(GetParam()) != "bluestore") | |
9784 | return; | |
9785 | ||
f64942e4 AA |
9786 | int r; |
9787 | auto logger = store->get_perf_counters(); | |
9788 | coll_t cid; | |
11fdf7f2 | 9789 | auto ch = store->create_new_collection(cid); |
f64942e4 AA |
9790 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
9791 | { | |
9792 | ObjectStore::Transaction t; | |
9793 | t.create_collection(cid, 0); | |
9794 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 9795 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
9796 | ASSERT_EQ(r, 0); |
9797 | } | |
9798 | bufferlist test_data; | |
9799 | bufferptr ap(0x2000); | |
9800 | memset(ap.c_str(), 'a', 0x2000); | |
9801 | test_data.append(ap); | |
9802 | { | |
9803 | ObjectStore::Transaction t; | |
9804 | t.write(cid, hoid, 0, 0x2000, test_data); | |
11fdf7f2 | 9805 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
9806 | ASSERT_EQ(r, 0); |
9807 | // force cache clear | |
9808 | EXPECT_EQ(store->umount(), 0); | |
9809 | EXPECT_EQ(store->mount(), 0); | |
9810 | } | |
f6b5b4d7 | 9811 | ch = store->open_collection(cid); |
f64942e4 AA |
9812 | |
9813 | cerr << "Injecting CRC error with no retry, expecting EIO" << std::endl; | |
11fdf7f2 TL |
9814 | SetVal(g_conf(), "bluestore_retry_disk_reads", "0"); |
9815 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "1"); | |
9816 | g_ceph_context->_conf.apply_changes(nullptr); | |
f64942e4 AA |
9817 | { |
9818 | bufferlist in; | |
11fdf7f2 | 9819 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
9820 | ASSERT_EQ(-EIO, r); |
9821 | ASSERT_EQ(logger->get(l_bluestore_read_eio), 1u); | |
9822 | ASSERT_EQ(logger->get(l_bluestore_reads_with_retries), 0u); | |
9823 | } | |
9824 | ||
9825 | cerr << "Injecting CRC error with retries, expecting success after several retries" << std::endl; | |
11fdf7f2 TL |
9826 | SetVal(g_conf(), "bluestore_retry_disk_reads", "255"); |
9827 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "0.8"); | |
f64942e4 AA |
9828 | /** |
9829 | * Probabilistic test: 25 reads, each has a 80% chance of failing with 255 retries | |
9830 | * Probability of at least one retried read: 1 - (0.2 ** 25) = 100% - 3e-18 | |
9831 | * Probability of a random test failure: 1 - ((1 - (0.8 ** 255)) ** 25) ~= 5e-24 | |
9832 | */ | |
11fdf7f2 | 9833 | g_ceph_context->_conf.apply_changes(nullptr); |
f64942e4 AA |
9834 | { |
9835 | for (int i = 0; i < 25; ++i) { | |
9836 | bufferlist in; | |
11fdf7f2 | 9837 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
9838 | ASSERT_EQ(0x2000, r); |
9839 | ASSERT_TRUE(bl_eq(test_data, in)); | |
9840 | } | |
9841 | ASSERT_GE(logger->get(l_bluestore_reads_with_retries), 1u); | |
9842 | } | |
11fdf7f2 TL |
9843 | } |
9844 | ||
11fdf7f2 TL |
9845 | TEST_P(StoreTest, mergeRegionTest) { |
9846 | if (string(GetParam()) != "bluestore") | |
9847 | return; | |
9848 | ||
9849 | SetVal(g_conf(), "bluestore_fsck_on_mount", "true"); | |
9850 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
9851 | SetVal(g_conf(), "bdev_debug_inflight_ios", "true"); | |
9852 | g_ceph_context->_conf.apply_changes(nullptr); | |
9853 | ||
9854 | uint32_t chunk_size = g_ceph_context->_conf->bdev_block_size; | |
9855 | int r = -1; | |
9856 | coll_t cid; | |
9857 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
9858 | auto ch = store->create_new_collection(cid); | |
9859 | { | |
9860 | ObjectStore::Transaction t; | |
9861 | t.create_collection(cid, 0); | |
9862 | r = queue_transaction(store, ch, std::move(t)); | |
9863 | ASSERT_EQ(r, 0); | |
9864 | } | |
9865 | { | |
9866 | ObjectStore::Transaction t; | |
9867 | t.touch(cid, hoid); | |
9868 | cerr << "Creating object " << hoid << std::endl; | |
9869 | r = queue_transaction(store, ch, std::move(t)); | |
9870 | ASSERT_EQ(r, 0); | |
9871 | } | |
9872 | bufferlist bl5; | |
9873 | bl5.append("abcde"); | |
9874 | uint64_t offset = 0; | |
9875 | { // 1. same region | |
9876 | ObjectStore::Transaction t; | |
9877 | t.write(cid, hoid, offset, 5, bl5); | |
9878 | t.write(cid, hoid, 0xa + offset, 5, bl5); | |
9879 | t.write(cid, hoid, 0x14 + offset, 5, bl5); | |
9880 | r = queue_transaction(store, ch, std::move(t)); | |
9881 | ASSERT_EQ(r, 0); | |
9882 | } | |
9883 | { // 2. adjacent regions | |
9884 | ObjectStore::Transaction t; | |
9885 | offset = chunk_size; | |
9886 | t.write(cid, hoid, offset, 5, bl5); | |
9887 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
9888 | r = queue_transaction(store, ch, std::move(t)); | |
9889 | ASSERT_EQ(r, 0); | |
9890 | } | |
9891 | { // 3. front merge | |
9892 | ObjectStore::Transaction t; | |
9893 | offset = chunk_size * 2; | |
9894 | t.write(cid, hoid, offset, 5, bl5); | |
9895 | t.write(cid, hoid, offset + chunk_size - 2, 5, bl5); | |
9896 | r = queue_transaction(store, ch, std::move(t)); | |
9897 | ASSERT_EQ(r, 0); | |
9898 | } | |
9899 | { // 4. back merge | |
9900 | ObjectStore::Transaction t; | |
9901 | bufferlist blc2; | |
9902 | blc2.append_zero(chunk_size + 2); | |
f64942e4 | 9903 | |
11fdf7f2 TL |
9904 | offset = chunk_size * 3; |
9905 | t.write(cid, hoid, offset, chunk_size + 2, blc2); | |
9906 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
9907 | r = queue_transaction(store, ch, std::move(t)); | |
9908 | ASSERT_EQ(r, 0); | |
9909 | } | |
9910 | { // 5. overlapping | |
9911 | ObjectStore::Transaction t; | |
9912 | uint64_t final_len = 0; | |
9913 | offset = chunk_size * 10; | |
9914 | bufferlist bl2c2; | |
9915 | bl2c2.append_zero(chunk_size * 2); | |
9916 | t.write(cid, hoid, offset + chunk_size * 3 - 3, chunk_size * 2, bl2c2); | |
9917 | bl2c2.append_zero(2); | |
9918 | t.write(cid, hoid, offset + chunk_size - 2, chunk_size * 2 + 2, bl2c2); | |
9919 | r = queue_transaction(store, ch, std::move(t)); | |
9920 | ASSERT_EQ(r, 0); | |
9921 | ||
9922 | final_len = (offset + chunk_size * 3 - 3) + (chunk_size * 2); | |
9923 | bufferlist bl; | |
9924 | r = store->read(ch, hoid, 0, final_len, bl); | |
9925 | ASSERT_EQ(final_len, static_cast<uint64_t>(r)); | |
9926 | } | |
f64942e4 | 9927 | } |
9f95a23c | 9928 | |
20effc67 TL |
9929 | TEST_P(StoreTest, FixSMRWritePointer) { |
9930 | if(string(GetParam()) != "bluestore") | |
9931 | return; | |
9932 | if (!smr) | |
9933 | return; | |
9934 | int r = store->umount(); | |
9935 | ASSERT_EQ(0, r); | |
9936 | ||
9937 | // copied from StoreTestFixture | |
9938 | std::string path = GetParam() + ".test_temp_dir"s; | |
9939 | ||
9940 | std::string p = path + "/block"; | |
9941 | BlockDevice* bdev = BlockDevice::create(g_ceph_context, p, nullptr, nullptr, nullptr, nullptr); | |
9942 | r = bdev->open(p); | |
9943 | ASSERT_EQ(0, r); | |
9944 | ASSERT_EQ(true, bdev->is_smr()); | |
9945 | ||
9946 | std::vector<uint64_t> wp = bdev->get_zones(); | |
9947 | uint64_t first_seq_zone = bdev->get_conventional_region_size() / bdev->get_zone_size(); | |
9948 | ||
9949 | IOContext ioc(g_ceph_context, NULL, true); | |
9950 | bufferlist bl; | |
9951 | bl.append(std::string(1024 * 1024, 'x')); | |
9952 | r = bdev->aio_write(wp[first_seq_zone], bl, &ioc, false); | |
9953 | ASSERT_EQ(0, r); | |
9954 | bdev->aio_submit(&ioc); | |
9955 | ioc.aio_wait(); | |
9956 | bdev->close(); | |
9957 | delete bdev; | |
9958 | ||
9959 | r = store->mount(); | |
9960 | ASSERT_EQ(0, r); | |
9961 | } | |
9962 | ||
9963 | ||
9f95a23c TL |
9964 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { |
9965 | if (string(GetParam()) != "bluestore") | |
9966 | return; | |
9967 | ||
9968 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
9969 | StartDeferred(0x1000); | |
9970 | ||
9971 | int r; | |
9972 | coll_t cid; | |
9973 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
9974 | auto ch = store->create_new_collection(cid); | |
9975 | { | |
9976 | ObjectStore::Transaction t; | |
9977 | t.create_collection(cid, 0); | |
9978 | cerr << "Creating collection " << cid << std::endl; | |
9979 | r = queue_transaction(store, ch, std::move(t)); | |
9980 | ASSERT_EQ(r, 0); | |
9981 | } | |
9982 | { | |
9983 | ObjectStore::Transaction t; | |
9984 | bufferlist bl, orig; | |
9985 | string s(g_ceph_context->_conf->bluestore_max_blob_size_hdd, '0'); | |
9986 | bl.append(s); | |
9987 | t.write(cid, hoid, 0, bl.length(), bl); | |
9988 | cerr << "write" << std::endl; | |
9989 | r = queue_transaction(store, ch, std::move(t)); | |
9990 | ASSERT_EQ(r, 0); | |
9991 | ||
9992 | const PerfCounters* logger = store->get_perf_counters(); | |
9993 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
9994 | } | |
9995 | } | |
9996 | ||
9997 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsSsd) { | |
9998 | if (string(GetParam()) != "bluestore") | |
9999 | return; | |
10000 | ||
10001 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "ssd"); | |
10002 | StartDeferred(0x1000); | |
10003 | ||
10004 | int r; | |
10005 | coll_t cid; | |
10006 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
10007 | auto ch = store->create_new_collection(cid); | |
10008 | { | |
10009 | ObjectStore::Transaction t; | |
10010 | t.create_collection(cid, 0); | |
10011 | cerr << "Creating collection " << cid << std::endl; | |
10012 | r = queue_transaction(store, ch, std::move(t)); | |
10013 | ASSERT_EQ(r, 0); | |
10014 | } | |
10015 | { | |
10016 | ObjectStore::Transaction t; | |
10017 | bufferlist bl, orig; | |
10018 | string s(g_ceph_context->_conf->bluestore_max_blob_size_ssd * 8, '0'); | |
10019 | bl.append(s); | |
10020 | t.write(cid, hoid, 0, bl.length(), bl); | |
10021 | cerr << "write" << std::endl; | |
10022 | r = queue_transaction(store, ch, std::move(t)); | |
10023 | ASSERT_EQ(r, 0); | |
10024 | ||
10025 | const PerfCounters* logger = store->get_perf_counters(); | |
10026 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 8u); | |
10027 | } | |
10028 | } | |
eafe8130 TL |
10029 | |
10030 | TEST_P(StoreTestSpecificAUSize, ReproNoBlobMultiTest) { | |
10031 | ||
10032 | if(string(GetParam()) != "bluestore") | |
10033 | return; | |
20effc67 TL |
10034 | if (smr) { |
10035 | cout << "SKIP (FIXME): bluestore gc does not seem to do the trick here" << std::endl; | |
10036 | return; | |
10037 | } | |
eafe8130 TL |
10038 | |
10039 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10040 | SetVal(g_conf(), "bluestore_block_db_size", "4294967296"); | |
10041 | SetVal(g_conf(), "bluestore_block_size", "12884901888"); | |
10042 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
10043 | ||
10044 | g_conf().apply_changes(nullptr); | |
10045 | ||
10046 | StartDeferred(65536); | |
10047 | ||
10048 | int r; | |
10049 | coll_t cid; | |
10050 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
10051 | ghobject_t hoid2 = hoid; | |
10052 | hoid2.hobj.snap = 1; | |
10053 | ||
10054 | auto ch = store->create_new_collection(cid); | |
10055 | { | |
10056 | ObjectStore::Transaction t; | |
10057 | t.create_collection(cid, 0); | |
10058 | cerr << "Creating collection " << cid << std::endl; | |
10059 | r = queue_transaction(store, ch, std::move(t)); | |
10060 | ASSERT_EQ(r, 0); | |
10061 | } | |
10062 | { | |
10063 | bool exists = store->exists(ch, hoid); | |
10064 | ASSERT_TRUE(!exists); | |
10065 | ||
10066 | ObjectStore::Transaction t; | |
10067 | t.touch(cid, hoid); | |
10068 | cerr << "Creating object " << hoid << std::endl; | |
10069 | r = queue_transaction(store, ch, std::move(t)); | |
10070 | ASSERT_EQ(r, 0); | |
10071 | ||
10072 | exists = store->exists(ch, hoid); | |
10073 | ASSERT_EQ(true, exists); | |
10074 | } | |
10075 | { | |
10076 | uint64_t offs = 0; | |
10077 | bufferlist bl; | |
10078 | const int size = 0x100; | |
10079 | bufferptr ap(size); | |
10080 | memset(ap.c_str(), 'a', size); | |
10081 | bl.append(ap); | |
10082 | int i = 0; | |
10083 | uint64_t blob_size = 524288; | |
10084 | uint64_t total = 0; | |
10085 | for (i = 0; i <= 512; i++) { | |
10086 | offs = 0 + i * size; | |
10087 | ObjectStore::Transaction t; | |
10088 | ghobject_t hoid2 = hoid; | |
10089 | hoid2.hobj.snap = i + 1; | |
10090 | while (offs < 128 * 1024 * 1024) { | |
10091 | ||
10092 | t.write(cid, hoid, offs, ap.length(), bl); | |
10093 | offs += blob_size; | |
10094 | total += ap.length(); | |
10095 | } | |
10096 | t.clone(cid, hoid, hoid2); | |
10097 | r = queue_transaction(store, ch, std::move(t)); | |
10098 | ASSERT_EQ(r, 0); | |
10099 | } | |
10100 | cerr << "Total written = " << total << std::endl; | |
10101 | } | |
10102 | { | |
10103 | cerr << "Finalizing" << std::endl; | |
10104 | const PerfCounters* logger = store->get_perf_counters(); | |
10105 | ASSERT_GE(logger->get(l_bluestore_gc_merged), 1024*1024*1024); | |
10106 | } | |
10107 | } | |
10108 | ||
9f95a23c TL |
10109 | void doManySetAttr(ObjectStore* store, |
10110 | std::function<void(ObjectStore*)> do_check_fn) | |
10111 | { | |
10112 | MixedGenerator gen(447); | |
10113 | gen_type rng(time(NULL)); | |
10114 | coll_t cid(spg_t(pg_t(0, 447), shard_id_t::NO_SHARD)); | |
10115 | ||
f6b5b4d7 | 10116 | SyntheticWorkloadState test_obj(store, &gen, &rng, cid, 0, 0, 0); |
9f95a23c | 10117 | test_obj.init(); |
f6b5b4d7 TL |
10118 | size_t object_count = 256; |
10119 | for (size_t i = 0; i < object_count; ++i) { | |
9f95a23c TL |
10120 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; |
10121 | test_obj.touch(); | |
10122 | } | |
f6b5b4d7 | 10123 | for (size_t i = 0; i < object_count; ++i) { |
9f95a23c TL |
10124 | if (!(i % 100)) { |
10125 | cerr << "Op " << i << std::endl; | |
10126 | test_obj.print_internal_state(); | |
10127 | } | |
f6b5b4d7 | 10128 | test_obj.set_fixed_attrs(1024, 64, 4096); // 1024 attributes, 64 bytes name and 4K value |
9f95a23c TL |
10129 | } |
10130 | test_obj.wait_for_done(); | |
10131 | ||
f6b5b4d7 TL |
10132 | std::cout << "done" << std::endl; |
10133 | do_check_fn(store); | |
9f95a23c TL |
10134 | AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); |
10135 | ceph_assert(admin_socket); | |
10136 | ||
10137 | ceph::bufferlist in, out; | |
10138 | ostringstream err; | |
10139 | ||
f6b5b4d7 TL |
10140 | auto r = admin_socket->execute_command( |
10141 | { "{\"prefix\": \"bluefs stats\"}" }, | |
9f95a23c | 10142 | in, err, &out); |
f6b5b4d7 TL |
10143 | if (r != 0) { |
10144 | cerr << "failure querying: " << cpp_strerror(r) << std::endl; | |
10145 | } else { | |
10146 | std::cout << std::string(out.c_str(), out.length()) << std::endl; | |
9f95a23c | 10147 | } |
9f95a23c TL |
10148 | test_obj.shutdown(); |
10149 | } | |
10150 | ||
10151 | TEST_P(StoreTestSpecificAUSize, SpilloverTest) { | |
10152 | if (string(GetParam()) != "bluestore") | |
10153 | return; | |
20effc67 TL |
10154 | if (smr) { |
10155 | cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; | |
10156 | return; | |
10157 | } | |
9f95a23c TL |
10158 | |
10159 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10160 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
10161 | SetVal(g_conf(), "bluestore_volume_selection_policy", "rocksdb_original"); | |
10162 | ||
10163 | g_conf().apply_changes(nullptr); | |
10164 | ||
10165 | StartDeferred(65536); | |
10166 | doManySetAttr(store.get(), | |
10167 | [&](ObjectStore* _store) { | |
10168 | ||
10169 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
10170 | ceph_assert(bstore); | |
f6b5b4d7 | 10171 | bstore->compact(); |
9f95a23c TL |
10172 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
10173 | //experimentally it was discovered that this case results in 400+MB spillover | |
10174 | //using lower 300MB threshold just to be safe enough | |
f6b5b4d7 TL |
10175 | std::cout << "db_used:" << logger->get(l_bluefs_db_used_bytes) << std::endl; |
10176 | std::cout << "slow_used:" << logger->get(l_bluefs_slow_used_bytes) << std::endl; | |
f67539c2 | 10177 | ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 16 * 1024 * 1024); |
9f95a23c TL |
10178 | |
10179 | } | |
10180 | ); | |
10181 | } | |
10182 | ||
10183 | TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) { | |
10184 | if (string(GetParam()) != "bluestore") | |
10185 | return; | |
20effc67 TL |
10186 | if (smr) { |
10187 | cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; | |
10188 | return; | |
10189 | } | |
9f95a23c TL |
10190 | |
10191 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10192 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
10193 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
10194 | SetVal(g_conf(), "bluestore_volume_selection_reserved", "1"); // just use non-zero to enable | |
10195 | ||
10196 | g_conf().apply_changes(nullptr); | |
10197 | ||
10198 | StartDeferred(65536); | |
10199 | doManySetAttr(store.get(), | |
10200 | [&](ObjectStore* _store) { | |
10201 | ||
10202 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
10203 | ceph_assert(bstore); | |
f6b5b4d7 | 10204 | bstore->compact(); |
9f95a23c TL |
10205 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
10206 | ASSERT_EQ(0, logger->get(l_bluefs_slow_used_bytes)); | |
10207 | } | |
10208 | ); | |
10209 | } | |
10210 | ||
10211 | TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) { | |
10212 | if (string(GetParam()) != "bluestore") | |
10213 | return; | |
20effc67 TL |
10214 | if (smr) { |
10215 | cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; | |
10216 | return; | |
10217 | } | |
9f95a23c TL |
10218 | |
10219 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10220 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
10221 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
10222 | //default 2.0 factor results in too high threshold, using less value | |
10223 | // that results in less but still present spillover. | |
10224 | SetVal(g_conf(), "bluestore_volume_selection_reserved_factor", "0.5"); | |
10225 | ||
10226 | g_conf().apply_changes(nullptr); | |
10227 | ||
10228 | StartDeferred(65536); | |
10229 | doManySetAttr(store.get(), | |
10230 | [&](ObjectStore* _store) { | |
10231 | ||
10232 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
10233 | ceph_assert(bstore); | |
f6b5b4d7 | 10234 | bstore->compact(); |
9f95a23c TL |
10235 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
10236 | ASSERT_LE(logger->get(l_bluefs_slow_used_bytes), 300 * 1024 * 1024); // see SpilloverTest for 300MB choice rationale | |
10237 | } | |
10238 | ); | |
10239 | } | |
10240 | ||
f67539c2 TL |
10241 | TEST_P(StoreTestSpecificAUSize, SpilloverFixed3Test) { |
10242 | if (string(GetParam()) != "bluestore") | |
10243 | return; | |
20effc67 TL |
10244 | if (smr) { |
10245 | cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; | |
10246 | return; | |
10247 | } | |
f67539c2 TL |
10248 | |
10249 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10250 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
10251 | SetVal(g_conf(), "bluestore_volume_selection_policy", "fit_to_fast"); | |
10252 | ||
10253 | g_conf().apply_changes(nullptr); | |
10254 | ||
10255 | StartDeferred(65536); | |
10256 | doManySetAttr(store.get(), | |
10257 | [&](ObjectStore* _store) { | |
10258 | ||
10259 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
10260 | ceph_assert(bstore); | |
10261 | bstore->compact(); | |
10262 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); | |
10263 | ASSERT_EQ(logger->get(l_bluefs_slow_used_bytes), 0); // reffering to SpilloverFixedTest | |
10264 | } | |
10265 | ); | |
10266 | } | |
10267 | ||
10268 | TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) { | |
10269 | if (string(GetParam()) != "bluestore") | |
10270 | return; | |
20effc67 TL |
10271 | if (smr) { |
10272 | return; | |
10273 | } | |
f67539c2 TL |
10274 | |
10275 | SetVal(g_conf(), "bluestore_default_buffered_write", "true"); | |
10276 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); | |
10277 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
10278 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
10279 | g_conf().apply_changes(nullptr); | |
10280 | ||
10281 | StartDeferred(0x1000); | |
10282 | ||
10283 | int r; | |
10284 | coll_t cid; | |
10285 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
10286 | auto ch = store->create_new_collection(cid); | |
10287 | { | |
10288 | ObjectStore::Transaction t; | |
10289 | t.create_collection(cid, 0); | |
10290 | cerr << "Creating collection " << cid << std::endl; | |
10291 | r = queue_transaction(store, ch, std::move(t)); | |
10292 | ASSERT_EQ(r, 0); | |
10293 | } | |
10294 | { | |
10295 | size_t large_object_size = 1 * 1024 * 1024; | |
10296 | size_t expected_write_size = 0x8000; | |
10297 | ObjectStore::Transaction t; | |
10298 | t.touch(cid, hoid); | |
10299 | t.set_alloc_hint(cid, hoid, large_object_size, expected_write_size, | |
10300 | CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ | | |
10301 | CEPH_OSD_ALLOC_HINT_FLAG_APPEND_ONLY); | |
10302 | r = queue_transaction(store, ch, std::move(t)); | |
10303 | ASSERT_EQ(r, 0); | |
10304 | } | |
10305 | { | |
10306 | ObjectStore::Transaction t; | |
10307 | bufferlist bl, orig; | |
10308 | string s(0xc000, '0'); | |
10309 | bl.append(s); | |
10310 | t.write(cid, hoid, 0xb000, bl.length(), bl); | |
10311 | r = queue_transaction(store, ch, std::move(t)); | |
10312 | ASSERT_EQ(r, 0); | |
10313 | } | |
10314 | { | |
10315 | ObjectStore::Transaction t; | |
10316 | bufferlist bl, orig; | |
10317 | string s(0x10000, '1'); | |
10318 | bl.append(s); | |
10319 | t.write(cid, hoid, 0x16000, bl.length(), bl); | |
10320 | r = queue_transaction(store, ch, std::move(t)); | |
10321 | ASSERT_EQ(r, 0); | |
10322 | } | |
10323 | { | |
10324 | ObjectStore::Transaction t; | |
10325 | bufferlist bl, orig; | |
10326 | string s(0x4000, '1'); | |
10327 | bl.append(s); | |
10328 | t.write(cid, hoid, 0x1b000, bl.length(), bl); | |
10329 | r = queue_transaction(store, ch, std::move(t)); | |
10330 | ASSERT_EQ(r, 0); | |
10331 | } | |
10332 | bufferlist bl; | |
10333 | r = store->read(ch, hoid, 0xb000, 0xb000, bl); | |
10334 | ASSERT_EQ(r, 0xb000); | |
10335 | ||
10336 | store->umount(); | |
10337 | store->mount(); | |
10338 | ||
10339 | ch = store->open_collection(cid); | |
10340 | { | |
10341 | ObjectStore::Transaction t; | |
10342 | bufferlist bl, orig; | |
10343 | string s(0xf000, '3'); | |
10344 | bl.append(s); | |
10345 | t.write(cid, hoid, 0xf000, bl.length(), bl); | |
10346 | cerr << "write4" << std::endl; | |
10347 | r = queue_transaction(store, ch, std::move(t)); | |
10348 | ASSERT_EQ(r, 0); | |
10349 | } | |
10350 | ||
10351 | r = store->read(ch, hoid, 0xb000, 0x10000, bl); | |
10352 | ASSERT_EQ(r, 0x10000); | |
10353 | } | |
10354 | ||
a4b75251 TL |
10355 | TEST_P(StoreTestOmapUpgrade, WithOmapHeader) { |
10356 | if (string(GetParam()) != "bluestore") | |
10357 | return; | |
10358 | ||
10359 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "true"); | |
10360 | g_conf().apply_changes(nullptr); | |
10361 | ||
10362 | StartDeferred(); | |
10363 | int64_t poolid = 11; | |
10364 | coll_t cid(spg_t(pg_t(1, poolid), shard_id_t::NO_SHARD)); | |
10365 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, poolid, "")); | |
10366 | auto ch = store->create_new_collection(cid); | |
10367 | int r; | |
10368 | { | |
10369 | ObjectStore::Transaction t; | |
10370 | t.create_collection(cid, 0); | |
10371 | r = queue_transaction(store, ch, std::move(t)); | |
10372 | ASSERT_EQ(r, 0); | |
10373 | } | |
10374 | ||
10375 | map<string, bufferlist> attrs; | |
10376 | bufferlist expected_header; | |
10377 | expected_header.append("this is a header"); | |
10378 | { | |
10379 | ObjectStore::Transaction t; | |
10380 | t.touch(cid, hoid); | |
10381 | bufferlist header; | |
10382 | header.append(expected_header); | |
10383 | t.omap_setheader(cid, hoid, header); | |
10384 | map<string, bufferlist> start_set; | |
10385 | bufferlist bl; | |
10386 | bl.append(string("value")); | |
10387 | start_set.emplace(string("key1"), bl); | |
10388 | t.omap_setkeys(cid, hoid, start_set); | |
10389 | r = queue_transaction(store, ch, std::move(t)); | |
10390 | } | |
10391 | { | |
10392 | map<string,bufferlist> res; | |
10393 | bufferlist h; | |
10394 | r = store->omap_get(ch, hoid, &h, &res); | |
10395 | ASSERT_EQ(r, 0); | |
10396 | ASSERT_TRUE(bl_eq(h, expected_header)); | |
10397 | ASSERT_EQ(res.size(), 1); | |
10398 | ASSERT_EQ(res.begin()->first, "key1"); | |
10399 | } | |
10400 | store->umount(); | |
10401 | ASSERT_EQ(store->fsck(false), 0); | |
10402 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "false"); | |
10403 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
10404 | g_conf().apply_changes(nullptr); | |
10405 | ASSERT_EQ(store->fsck(false), 2); | |
10406 | ASSERT_EQ(store->quick_fix(), 0); | |
10407 | store->mount(); | |
10408 | ch = store->open_collection(cid); | |
10409 | { | |
10410 | map<string,bufferlist> res; | |
10411 | bufferlist h; | |
10412 | r = store->omap_get(ch, hoid, &h, &res); | |
10413 | ASSERT_EQ(r, 0); | |
10414 | ASSERT_EQ(res.size(), 1); | |
10415 | ASSERT_EQ(res.begin()->first, "key1"); | |
10416 | } | |
10417 | { | |
10418 | ObjectStore::Transaction t; | |
10419 | t.remove(cid, hoid); | |
10420 | t.remove_collection(cid); | |
10421 | r = queue_transaction(store, ch, std::move(t)); | |
10422 | ASSERT_EQ(r, 0); | |
10423 | } | |
10424 | } | |
10425 | ||
10426 | TEST_P(StoreTestSpecificAUSize, BluefsWriteInSingleDiskEnvTest) { | |
10427 | if (string(GetParam()) != "bluestore") | |
10428 | return; | |
10429 | ||
10430 | g_conf().apply_changes(nullptr); | |
10431 | ||
10432 | StartDeferred(0x1000); | |
10433 | ||
10434 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
10435 | ceph_assert(bstore); | |
10436 | bstore->inject_bluefs_file("db.slow", "store_test_injection_slow", 1 << 20ul); | |
10437 | bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul); | |
10438 | bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul); | |
10439 | ||
10440 | AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); | |
10441 | ceph_assert(admin_socket); | |
10442 | ||
10443 | ceph::bufferlist in, out; | |
10444 | ostringstream err; | |
10445 | auto r = admin_socket->execute_command( | |
10446 | { "{\"prefix\": \"bluefs stats\"}" }, | |
10447 | in, err, &out); | |
10448 | if (r != 0) { | |
10449 | cerr << "failure querying: " << cpp_strerror(r) << std::endl; | |
10450 | } else { | |
10451 | std::cout << std::string(out.c_str(), out.length()) << std::endl; | |
10452 | } | |
10453 | } | |
10454 | ||
10455 | TEST_P(StoreTestSpecificAUSize, BluefsWriteInNoWalDiskEnvTest) { | |
10456 | if (string(GetParam()) != "bluestore") | |
10457 | return; | |
10458 | ||
10459 | SetVal(g_conf(), "bluestore_block_db_path", "db"); | |
10460 | SetVal(g_conf(), "bluestore_block_db_size", stringify(1ull << 31).c_str()); | |
10461 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
10462 | ||
10463 | g_conf().apply_changes(nullptr); | |
10464 | ||
10465 | StartDeferred(0x1000); | |
10466 | ||
10467 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
10468 | ceph_assert(bstore); | |
10469 | bstore->inject_bluefs_file("db.slow", "store_test_injection_slow", 1 << 20ul); | |
10470 | bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul); | |
10471 | bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul); | |
10472 | ||
10473 | AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); | |
10474 | ceph_assert(admin_socket); | |
10475 | ||
10476 | ceph::bufferlist in, out; | |
10477 | ostringstream err; | |
10478 | auto r = admin_socket->execute_command( | |
10479 | { "{\"prefix\": \"bluefs stats\"}" }, | |
10480 | in, err, &out); | |
10481 | if (r != 0) { | |
10482 | cerr << "failure querying: " << cpp_strerror(r) << std::endl; | |
10483 | } | |
10484 | else { | |
10485 | std::cout << std::string(out.c_str(), out.length()) << std::endl; | |
10486 | } | |
10487 | } | |
10488 | ||
10489 | TEST_P(StoreTestOmapUpgrade, NoOmapHeader) { | |
10490 | if (string(GetParam()) != "bluestore") | |
10491 | return; | |
10492 | ||
10493 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "true"); | |
10494 | g_conf().apply_changes(nullptr); | |
10495 | ||
10496 | StartDeferred(); | |
10497 | int64_t poolid = 11; | |
10498 | coll_t cid(spg_t(pg_t(1, poolid), shard_id_t::NO_SHARD)); | |
10499 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, poolid, "")); | |
10500 | auto ch = store->create_new_collection(cid); | |
10501 | int r; | |
10502 | { | |
10503 | ObjectStore::Transaction t; | |
10504 | t.create_collection(cid, 0); | |
10505 | r = queue_transaction(store, ch, std::move(t)); | |
10506 | ASSERT_EQ(r, 0); | |
10507 | } | |
10508 | ||
10509 | map<string, bufferlist> attrs; | |
10510 | { | |
10511 | ObjectStore::Transaction t; | |
10512 | t.touch(cid, hoid); | |
10513 | map<string, bufferlist> start_set; | |
10514 | bufferlist bl; | |
10515 | bl.append(string("value")); | |
10516 | start_set.emplace(string("key1"), bl); | |
10517 | t.omap_setkeys(cid, hoid, start_set); | |
10518 | r = queue_transaction(store, ch, std::move(t)); | |
10519 | } | |
10520 | { | |
10521 | map<string,bufferlist> res; | |
10522 | bufferlist h; | |
10523 | r = store->omap_get(ch, hoid, &h, &res); | |
10524 | ASSERT_EQ(r, 0); | |
10525 | ASSERT_EQ(h.length(), 0); | |
10526 | ASSERT_EQ(res.size(), 1); | |
10527 | ASSERT_EQ(res.begin()->first, "key1"); | |
10528 | } | |
10529 | store->umount(); | |
10530 | ASSERT_EQ(store->fsck(false), 0); | |
10531 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "false"); | |
10532 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
10533 | g_conf().apply_changes(nullptr); | |
10534 | ASSERT_EQ(store->fsck(false), 2); | |
10535 | ASSERT_EQ(store->quick_fix(), 0); | |
10536 | store->mount(); | |
10537 | ch = store->open_collection(cid); | |
10538 | { | |
10539 | map<string,bufferlist> res; | |
10540 | bufferlist h; | |
10541 | r = store->omap_get(ch, hoid, &h, &res); | |
10542 | ASSERT_EQ(r, 0); | |
10543 | ASSERT_EQ(res.size(), 1); | |
10544 | ASSERT_EQ(res.begin()->first, "key1"); | |
10545 | } | |
10546 | { | |
10547 | ObjectStore::Transaction t; | |
10548 | t.remove(cid, hoid); | |
10549 | t.remove_collection(cid); | |
10550 | r = queue_transaction(store, ch, std::move(t)); | |
10551 | ASSERT_EQ(r, 0); | |
10552 | } | |
10553 | } | |
10554 | ||
10555 | TEST_P(StoreTestOmapUpgrade, LargeLegacyToPG) { | |
10556 | if (string(GetParam()) != "bluestore") | |
10557 | return; | |
10558 | ||
10559 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "true"); | |
10560 | g_conf().apply_changes(nullptr); | |
10561 | ||
10562 | int64_t poolid; | |
10563 | coll_t cid; | |
10564 | ghobject_t hoid; | |
10565 | ObjectStore::CollectionHandle ch; | |
10566 | StartDeferred(); | |
10567 | poolid = 11; | |
10568 | cid = coll_t(spg_t(pg_t(1, poolid), shard_id_t::NO_SHARD)); | |
10569 | ch = store->create_new_collection(cid); | |
10570 | int r; | |
10571 | { | |
10572 | ObjectStore::Transaction t; | |
10573 | t.create_collection(cid, 0); | |
10574 | r = queue_transaction(store, ch, std::move(t)); | |
10575 | ASSERT_EQ(r, 0); | |
10576 | } | |
10577 | //ASSERT_EQ(false, g_conf().get_val<bool>("bluestore_debug_inject_upgrade_bug53062")); | |
10578 | map<string, bufferlist> attrs; | |
10579 | bufferlist expected_header; | |
10580 | expected_header.append("this is a header"); | |
10581 | ||
10582 | size_t object_count = 1000; | |
10583 | make_omap_data(object_count, poolid, cid); | |
10584 | //checking just written data | |
10585 | check_omap_data(object_count, poolid, cid); | |
10586 | ||
10587 | store->umount(); | |
10588 | ASSERT_EQ(store->fsck(false), 0); | |
10589 | SetVal(g_conf(), "bluestore_debug_legacy_omap", "false"); | |
10590 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
10591 | g_conf().apply_changes(nullptr); | |
10592 | ASSERT_EQ(store->fsck(false), 1001); | |
10593 | ASSERT_EQ(store->quick_fix(), 0); | |
10594 | store->mount(); | |
10595 | ch = store->open_collection(cid); | |
10596 | ||
10597 | //checking quick_fix() data | |
10598 | check_omap_data(object_count, poolid, cid); | |
10599 | ||
10600 | { | |
10601 | ObjectStore::Transaction t; | |
10602 | for (size_t o = 0; o < object_count; o++) | |
10603 | { | |
10604 | std::string oid = generate_monotonic_name(object_count, o, 3.71, 0.5); | |
10605 | ghobject_t hoid(hobject_t(oid, "", CEPH_NOSNAP, 0, poolid, "")); | |
10606 | t.remove(cid, hoid); | |
10607 | } | |
10608 | t.remove_collection(cid); | |
10609 | r = queue_transaction(store, ch, std::move(t)); | |
10610 | ASSERT_EQ(r, 0); | |
10611 | } | |
10612 | } | |
10613 | ||
11fdf7f2 | 10614 | #endif // WITH_BLUESTORE |
f64942e4 | 10615 | |
7c673cae | 10616 | int main(int argc, char **argv) { |
20effc67 | 10617 | auto args = argv_to_vec(argc, argv); |
7c673cae | 10618 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, |
11fdf7f2 TL |
10619 | CODE_ENVIRONMENT_UTILITY, |
10620 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae FG |
10621 | common_init_finish(g_ceph_context); |
10622 | ||
20effc67 TL |
10623 | for (auto& i : args) { |
10624 | if (i == "--smr"s) { | |
10625 | #if defined(HAVE_LIBZBD) | |
10626 | derr << "Adjusting tests for smr mode." << dendl; | |
10627 | smr = true; | |
10628 | #else | |
10629 | derr << "smr mode selected, but support not compiled in" << dendl; | |
10630 | return 1; | |
10631 | #endif | |
10632 | } | |
10633 | } | |
10634 | ||
11fdf7f2 TL |
10635 | // make sure we can adjust any config settings |
10636 | g_ceph_context->_conf._clear_safe_to_start_threads(); | |
10637 | ||
10638 | g_ceph_context->_conf.set_val_or_die("osd_journal_size", "400"); | |
10639 | g_ceph_context->_conf.set_val_or_die("filestore_index_retry_probability", "0.5"); | |
10640 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_timeout", "1000"); | |
10641 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_suicide_timeout", "10000"); | |
10642 | //g_ceph_context->_conf.set_val_or_die("filestore_fiemap", "true"); | |
10643 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mkfs", "false"); | |
10644 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mount", "false"); | |
10645 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_umount", "false"); | |
11fdf7f2 TL |
10646 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_small_allocations", "4"); |
10647 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_freelist", "true"); | |
10648 | g_ceph_context->_conf.set_val_or_die("bluestore_clone_cow", "true"); | |
10649 | g_ceph_context->_conf.set_val_or_die("bluestore_max_alloc_size", "196608"); | |
7c673cae FG |
10650 | |
10651 | // set small cache sizes so we see trimming during Synthetic tests | |
11fdf7f2 TL |
10652 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_hdd", "4000000"); |
10653 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_ssd", "4000000"); | |
7c673cae FG |
10654 | |
10655 | // very short *_max prealloc so that we fall back to async submits | |
11fdf7f2 TL |
10656 | g_ceph_context->_conf.set_val_or_die("bluestore_blobid_prealloc", "10"); |
10657 | g_ceph_context->_conf.set_val_or_die("bluestore_nid_prealloc", "10"); | |
10658 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_randomize_serial_transaction", | |
7c673cae FG |
10659 | "10"); |
10660 | ||
11fdf7f2 | 10661 | g_ceph_context->_conf.set_val_or_die("bdev_debug_aio", "true"); |
7c673cae FG |
10662 | |
10663 | // specify device size | |
11fdf7f2 | 10664 | g_ceph_context->_conf.set_val_or_die("bluestore_block_size", |
94b18763 | 10665 | stringify(DEF_STORE_TEST_BLOCKDEV_SIZE)); |
7c673cae | 10666 | |
11fdf7f2 | 10667 | g_ceph_context->_conf.set_val_or_die( |
7c673cae | 10668 | "enable_experimental_unrecoverable_data_corrupting_features", "*"); |
11fdf7f2 | 10669 | g_ceph_context->_conf.apply_changes(nullptr); |
7c673cae FG |
10670 | |
10671 | ::testing::InitGoogleTest(&argc, argv); | |
10672 | return RUN_ALL_TESTS(); | |
10673 | } | |
10674 | ||
10675 | /* | |
10676 | * Local Variables: | |
10677 | * compile-command: "cd ../.. ; make ceph_test_objectstore && | |
10678 | * ./ceph_test_objectstore \ | |
10679 | * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20 | |
10680 | * " | |
10681 | * End: | |
10682 | */ |