]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <glob.h> | |
16 | #include <stdio.h> | |
17 | #include <string.h> | |
18 | #include <iostream> | |
19 | #include <time.h> | |
20 | #include <sys/mount.h> | |
21 | #include <boost/scoped_ptr.hpp> | |
22 | #include <boost/random/mersenne_twister.hpp> | |
23 | #include <boost/random/uniform_int.hpp> | |
24 | #include <boost/random/binomial_distribution.hpp> | |
25 | #include <gtest/gtest.h> | |
26 | ||
27 | #include "os/ObjectStore.h" | |
28 | #include "os/filestore/FileStore.h" | |
11fdf7f2 | 29 | #if defined(WITH_BLUESTORE) |
7c673cae | 30 | #include "os/bluestore/BlueStore.h" |
9f95a23c | 31 | #include "os/bluestore/BlueFS.h" |
7c673cae FG |
32 | #endif |
33 | #include "include/Context.h" | |
34 | #include "common/ceph_argparse.h" | |
9f95a23c | 35 | #include "common/admin_socket.h" |
7c673cae | 36 | #include "global/global_init.h" |
9f95a23c | 37 | #include "common/ceph_mutex.h" |
7c673cae FG |
38 | #include "common/Cond.h" |
39 | #include "common/errno.h" | |
40 | #include "include/stringify.h" | |
41 | #include "include/coredumpctl.h" | |
42 | ||
43 | #include "include/unordered_map.h" | |
44 | #include "store_test_fixture.h" | |
45 | ||
11fdf7f2 | 46 | using namespace std::placeholders; |
7c673cae FG |
47 | |
48 | typedef boost::mt11213b gen_type; | |
49 | ||
94b18763 | 50 | const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000; |
7c673cae FG |
51 | #define dout_context g_ceph_context |
52 | ||
7c673cae FG |
53 | static bool bl_eq(bufferlist& expected, bufferlist& actual) |
54 | { | |
55 | if (expected.contents_equal(actual)) | |
56 | return true; | |
57 | ||
58 | unsigned first = 0; | |
59 | if(expected.length() != actual.length()) { | |
60 | cout << "--- buffer lengths mismatch " << std::hex | |
61 | << "expected 0x" << expected.length() << " != actual 0x" | |
62 | << actual.length() << std::dec << std::endl; | |
63 | derr << "--- buffer lengths mismatch " << std::hex | |
64 | << "expected 0x" << expected.length() << " != actual 0x" | |
65 | << actual.length() << std::dec << dendl; | |
66 | } | |
11fdf7f2 | 67 | auto len = std::min(expected.length(), actual.length()); |
7c673cae FG |
68 | while ( first<len && expected[first] == actual[first]) |
69 | ++first; | |
70 | unsigned last = len; | |
71 | while (last > 0 && expected[last-1] == actual[last-1]) | |
72 | --last; | |
73 | if(len > 0) { | |
74 | cout << "--- buffer mismatch between offset 0x" << std::hex << first | |
75 | << " and 0x" << last << ", total 0x" << len << std::dec | |
76 | << std::endl; | |
77 | derr << "--- buffer mismatch between offset 0x" << std::hex << first | |
78 | << " and 0x" << last << ", total 0x" << len << std::dec | |
79 | << dendl; | |
80 | cout << "--- expected:\n"; | |
81 | expected.hexdump(cout); | |
82 | cout << "--- actual:\n"; | |
83 | actual.hexdump(cout); | |
84 | } | |
85 | return false; | |
86 | } | |
87 | ||
88 | ||
89 | ||
90 | template <typename T> | |
11fdf7f2 | 91 | int queue_transaction( |
7c673cae | 92 | T &store, |
11fdf7f2 | 93 | ObjectStore::CollectionHandle ch, |
7c673cae FG |
94 | ObjectStore::Transaction &&t) { |
95 | if (rand() % 2) { | |
96 | ObjectStore::Transaction t2; | |
97 | t2.append(t); | |
11fdf7f2 | 98 | return store->queue_transaction(ch, std::move(t2)); |
7c673cae | 99 | } else { |
11fdf7f2 | 100 | return store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
101 | } |
102 | } | |
103 | ||
f91f0fd5 TL |
104 | template <typename T> |
105 | int collection_list(T &store, ObjectStore::CollectionHandle &c, | |
106 | const ghobject_t& start, const ghobject_t& end, int max, | |
107 | vector<ghobject_t> *ls, ghobject_t *pnext, | |
108 | bool disable_legacy = false) { | |
109 | if (disable_legacy || rand() % 2) { | |
110 | return store->collection_list(c, start, end, max, ls, pnext); | |
111 | } else { | |
112 | return store->collection_list_legacy(c, start, end, max, ls, pnext); | |
113 | } | |
114 | } | |
7c673cae FG |
115 | |
116 | bool sorted(const vector<ghobject_t> &in) { | |
117 | ghobject_t start; | |
118 | for (vector<ghobject_t>::const_iterator i = in.begin(); | |
119 | i != in.end(); | |
120 | ++i) { | |
121 | if (start > *i) { | |
122 | cout << start << " should follow " << *i << std::endl; | |
123 | return false; | |
124 | } | |
125 | start = *i; | |
126 | } | |
127 | return true; | |
128 | } | |
129 | ||
130 | class StoreTest : public StoreTestFixture, | |
131 | public ::testing::WithParamInterface<const char*> { | |
132 | public: | |
133 | StoreTest() | |
134 | : StoreTestFixture(GetParam()) | |
135 | {} | |
11fdf7f2 TL |
136 | void doCompressionTest(); |
137 | void doSyntheticTest( | |
138 | int num_ops, | |
139 | uint64_t max_obj, uint64_t max_wr, uint64_t align); | |
7c673cae FG |
140 | }; |
141 | ||
142 | class StoreTestDeferredSetup : public StoreTest { | |
143 | void SetUp() override { | |
144 | //do nothing | |
145 | } | |
146 | ||
147 | protected: | |
148 | void DeferredSetup() { | |
149 | StoreTest::SetUp(); | |
150 | } | |
151 | ||
152 | public: | |
153 | }; | |
154 | ||
155 | class StoreTestSpecificAUSize : public StoreTestDeferredSetup { | |
156 | ||
157 | public: | |
158 | typedef | |
159 | std::function<void( | |
7c673cae FG |
160 | uint64_t num_ops, |
161 | uint64_t max_obj, | |
162 | uint64_t max_wr, | |
163 | uint64_t align)> MatrixTest; | |
164 | ||
165 | void StartDeferred(size_t min_alloc_size) { | |
11fdf7f2 | 166 | SetVal(g_conf(), "bluestore_min_alloc_size", stringify(min_alloc_size).c_str()); |
7c673cae FG |
167 | DeferredSetup(); |
168 | } | |
7c673cae FG |
169 | |
170 | private: | |
171 | // bluestore matrix testing | |
172 | uint64_t max_write = 40 * 1024; | |
173 | uint64_t max_size = 400 * 1024; | |
174 | uint64_t alignment = 0; | |
175 | uint64_t num_ops = 10000; | |
176 | ||
177 | protected: | |
178 | string matrix_get(const char *k) { | |
179 | if (string(k) == "max_write") { | |
180 | return stringify(max_write); | |
181 | } else if (string(k) == "max_size") { | |
182 | return stringify(max_size); | |
183 | } else if (string(k) == "alignment") { | |
184 | return stringify(alignment); | |
185 | } else if (string(k) == "num_ops") { | |
186 | return stringify(num_ops); | |
187 | } else { | |
188 | char *buf; | |
11fdf7f2 | 189 | g_conf().get_val(k, &buf, -1); |
7c673cae FG |
190 | string v = buf; |
191 | free(buf); | |
192 | return v; | |
193 | } | |
194 | } | |
195 | ||
196 | void matrix_set(const char *k, const char *v) { | |
197 | if (string(k) == "max_write") { | |
198 | max_write = atoll(v); | |
199 | } else if (string(k) == "max_size") { | |
200 | max_size = atoll(v); | |
201 | } else if (string(k) == "alignment") { | |
202 | alignment = atoll(v); | |
203 | } else if (string(k) == "num_ops") { | |
204 | num_ops = atoll(v); | |
205 | } else { | |
11fdf7f2 | 206 | SetVal(g_conf(), k, v); |
7c673cae FG |
207 | } |
208 | } | |
209 | ||
210 | void do_matrix_choose(const char *matrix[][10], | |
211 | int i, int pos, int num, | |
7c673cae FG |
212 | MatrixTest fn) { |
213 | if (matrix[i][0]) { | |
214 | int count; | |
215 | for (count = 0; matrix[i][count+1]; ++count) ; | |
216 | for (int j = 1; matrix[i][j]; ++j) { | |
217 | matrix_set(matrix[i][0], matrix[i][j]); | |
218 | do_matrix_choose(matrix, | |
219 | i + 1, | |
220 | pos * count + j - 1, | |
221 | num * count, | |
7c673cae FG |
222 | fn); |
223 | } | |
224 | } else { | |
225 | cout << "---------------------- " << (pos + 1) << " / " << num | |
226 | << " ----------------------" << std::endl; | |
227 | for (unsigned k=0; matrix[k][0]; ++k) { | |
228 | cout << " " << matrix[k][0] << " = " << matrix_get(matrix[k][0]) | |
229 | << std::endl; | |
230 | } | |
11fdf7f2 TL |
231 | g_ceph_context->_conf.apply_changes(nullptr); |
232 | fn(num_ops, max_size, max_write, alignment); | |
7c673cae FG |
233 | } |
234 | } | |
235 | ||
236 | void do_matrix(const char *matrix[][10], | |
7c673cae | 237 | MatrixTest fn) { |
7c673cae FG |
238 | |
239 | if (strcmp(matrix[0][0], "bluestore_min_alloc_size") == 0) { | |
240 | int count; | |
241 | for (count = 0; matrix[0][count+1]; ++count) ; | |
242 | for (size_t j = 1; matrix[0][j]; ++j) { | |
243 | if (j > 1) { | |
244 | TearDown(); | |
245 | } | |
246 | StartDeferred(strtoll(matrix[0][j], NULL, 10)); | |
11fdf7f2 | 247 | do_matrix_choose(matrix, 1, j - 1, count, fn); |
7c673cae FG |
248 | } |
249 | } else { | |
250 | StartDeferred(0); | |
11fdf7f2 | 251 | do_matrix_choose(matrix, 0, 0, 1, fn); |
7c673cae | 252 | } |
7c673cae FG |
253 | } |
254 | ||
255 | }; | |
256 | ||
257 | TEST_P(StoreTest, collect_metadata) { | |
258 | map<string,string> pm; | |
259 | store->collect_metadata(&pm); | |
260 | if (GetParam() == string("filestore")) { | |
261 | ASSERT_NE(pm.count("filestore_backend"), 0u); | |
262 | ASSERT_NE(pm.count("filestore_f_type"), 0u); | |
263 | ASSERT_NE(pm.count("backend_filestore_partition_path"), 0u); | |
264 | ASSERT_NE(pm.count("backend_filestore_dev_node"), 0u); | |
265 | } | |
266 | } | |
267 | ||
268 | TEST_P(StoreTest, Trivial) { | |
269 | } | |
270 | ||
271 | TEST_P(StoreTest, TrivialRemount) { | |
272 | int r = store->umount(); | |
273 | ASSERT_EQ(0, r); | |
274 | r = store->mount(); | |
275 | ASSERT_EQ(0, r); | |
276 | } | |
277 | ||
278 | TEST_P(StoreTest, SimpleRemount) { | |
7c673cae FG |
279 | coll_t cid; |
280 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
281 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
282 | bufferlist bl; | |
283 | bl.append("1234512345"); | |
284 | int r; | |
11fdf7f2 | 285 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
286 | { |
287 | cerr << "create collection + write" << std::endl; | |
288 | ObjectStore::Transaction t; | |
289 | t.create_collection(cid, 0); | |
290 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 291 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
292 | ASSERT_EQ(r, 0); |
293 | } | |
11fdf7f2 | 294 | ch.reset(); |
7c673cae FG |
295 | r = store->umount(); |
296 | ASSERT_EQ(0, r); | |
297 | r = store->mount(); | |
298 | ASSERT_EQ(0, r); | |
11fdf7f2 | 299 | ch = store->open_collection(cid); |
7c673cae FG |
300 | { |
301 | ObjectStore::Transaction t; | |
302 | t.write(cid, hoid2, 0, bl.length(), bl); | |
11fdf7f2 | 303 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
304 | ASSERT_EQ(r, 0); |
305 | } | |
306 | { | |
307 | ObjectStore::Transaction t; | |
308 | t.remove(cid, hoid); | |
309 | t.remove(cid, hoid2); | |
310 | t.remove_collection(cid); | |
311 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 312 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
313 | ASSERT_EQ(r, 0); |
314 | } | |
11fdf7f2 | 315 | ch.reset(); |
7c673cae FG |
316 | r = store->umount(); |
317 | ASSERT_EQ(0, r); | |
318 | r = store->mount(); | |
319 | ASSERT_EQ(0, r); | |
11fdf7f2 | 320 | ch = store->create_new_collection(cid); |
7c673cae FG |
321 | { |
322 | ObjectStore::Transaction t; | |
323 | t.create_collection(cid, 0); | |
11fdf7f2 | 324 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 325 | ASSERT_EQ(r, 0); |
11fdf7f2 | 326 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
327 | ASSERT_TRUE(!exists); |
328 | } | |
329 | { | |
330 | ObjectStore::Transaction t; | |
331 | t.remove_collection(cid); | |
332 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 333 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
334 | ASSERT_EQ(r, 0); |
335 | } | |
336 | } | |
337 | ||
338 | TEST_P(StoreTest, IORemount) { | |
7c673cae FG |
339 | coll_t cid; |
340 | bufferlist bl; | |
341 | bl.append("1234512345"); | |
342 | int r; | |
11fdf7f2 | 343 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
344 | { |
345 | cerr << "create collection + objects" << std::endl; | |
346 | ObjectStore::Transaction t; | |
347 | t.create_collection(cid, 0); | |
348 | for (int n=1; n<=100; ++n) { | |
349 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
350 | t.write(cid, hoid, 0, bl.length(), bl); | |
351 | } | |
11fdf7f2 | 352 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
353 | ASSERT_EQ(r, 0); |
354 | } | |
355 | // overwrites | |
356 | { | |
357 | cout << "overwrites" << std::endl; | |
358 | for (int n=1; n<=100; ++n) { | |
359 | ObjectStore::Transaction t; | |
360 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
361 | t.write(cid, hoid, 1, bl.length(), bl); | |
11fdf7f2 | 362 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
363 | ASSERT_EQ(r, 0); |
364 | } | |
365 | } | |
11fdf7f2 | 366 | ch.reset(); |
7c673cae FG |
367 | r = store->umount(); |
368 | ASSERT_EQ(0, r); | |
369 | r = store->mount(); | |
370 | ASSERT_EQ(0, r); | |
371 | { | |
372 | ObjectStore::Transaction t; | |
373 | for (int n=1; n<=100; ++n) { | |
374 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
375 | t.remove(cid, hoid); | |
376 | } | |
377 | t.remove_collection(cid); | |
11fdf7f2 TL |
378 | auto ch = store->open_collection(cid); |
379 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
380 | ASSERT_EQ(r, 0); |
381 | } | |
382 | } | |
383 | ||
384 | TEST_P(StoreTest, UnprintableCharsName) { | |
7c673cae FG |
385 | coll_t cid; |
386 | string name = "funnychars_"; | |
387 | for (unsigned i = 0; i < 256; ++i) { | |
388 | name.push_back(i); | |
389 | } | |
390 | ghobject_t oid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
391 | int r; | |
11fdf7f2 | 392 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
393 | { |
394 | cerr << "create collection + object" << std::endl; | |
395 | ObjectStore::Transaction t; | |
396 | t.create_collection(cid, 0); | |
397 | t.touch(cid, oid); | |
11fdf7f2 | 398 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
399 | ASSERT_EQ(r, 0); |
400 | } | |
11fdf7f2 | 401 | ch.reset(); |
7c673cae FG |
402 | r = store->umount(); |
403 | ASSERT_EQ(0, r); | |
404 | r = store->mount(); | |
405 | ASSERT_EQ(0, r); | |
406 | { | |
407 | cout << "removing" << std::endl; | |
408 | ObjectStore::Transaction t; | |
409 | t.remove(cid, oid); | |
410 | t.remove_collection(cid); | |
11fdf7f2 TL |
411 | auto ch = store->open_collection(cid); |
412 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
413 | ASSERT_EQ(r, 0); |
414 | } | |
415 | } | |
416 | ||
417 | TEST_P(StoreTest, FiemapEmpty) { | |
7c673cae FG |
418 | coll_t cid; |
419 | int r = 0; | |
420 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
11fdf7f2 | 421 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
422 | { |
423 | ObjectStore::Transaction t; | |
424 | t.create_collection(cid, 0); | |
425 | t.touch(cid, oid); | |
426 | t.truncate(cid, oid, 100000); | |
11fdf7f2 | 427 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
428 | ASSERT_EQ(r, 0); |
429 | } | |
430 | { | |
431 | bufferlist bl; | |
11fdf7f2 | 432 | store->fiemap(ch, oid, 0, 100000, bl); |
7c673cae | 433 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
434 | auto p = bl.cbegin(); |
435 | decode(m, p); | |
7c673cae FG |
436 | cout << " got " << m << std::endl; |
437 | e[0] = 100000; | |
438 | EXPECT_TRUE(m == e || m.empty()); | |
439 | } | |
440 | { | |
441 | ObjectStore::Transaction t; | |
442 | t.remove(cid, oid); | |
443 | t.remove_collection(cid); | |
444 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 445 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
446 | ASSERT_EQ(r, 0); |
447 | } | |
448 | } | |
449 | ||
450 | TEST_P(StoreTest, FiemapHoles) { | |
7c673cae FG |
451 | const uint64_t MAX_EXTENTS = 4000; |
452 | const uint64_t SKIP_STEP = 65536; | |
453 | coll_t cid; | |
454 | int r = 0; | |
455 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
456 | bufferlist bl; | |
457 | bl.append("foo"); | |
11fdf7f2 | 458 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
459 | { |
460 | ObjectStore::Transaction t; | |
461 | t.create_collection(cid, 0); | |
462 | t.touch(cid, oid); | |
463 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) | |
464 | t.write(cid, oid, SKIP_STEP * i, 3, bl); | |
11fdf7f2 | 465 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
466 | ASSERT_EQ(r, 0); |
467 | } | |
468 | { | |
469 | //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3 | |
470 | bufferlist bl; | |
11fdf7f2 | 471 | store->fiemap(ch, oid, 0, SKIP_STEP * (MAX_EXTENTS - 1) + 3, bl); |
7c673cae | 472 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
473 | auto p = bl.cbegin(); |
474 | decode(m, p); | |
7c673cae FG |
475 | cout << " got " << m << std::endl; |
476 | ASSERT_TRUE(!m.empty()); | |
477 | ASSERT_GE(m[0], 3u); | |
11fdf7f2 TL |
478 | auto last = m.crbegin(); |
479 | if (m.size() == 1) { | |
480 | ASSERT_EQ(0u, last->first); | |
481 | } else if (m.size() == MAX_EXTENTS) { | |
482 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) { | |
483 | ASSERT_TRUE(m.count(SKIP_STEP * i)); | |
484 | } | |
7c673cae | 485 | } |
11fdf7f2 TL |
486 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); |
487 | } | |
488 | { | |
7c673cae | 489 | // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3 |
11fdf7f2 TL |
490 | bufferlist bl; |
491 | store->fiemap(ch, oid, SKIP_STEP, SKIP_STEP * (MAX_EXTENTS - 2) + 3, bl); | |
492 | map<uint64_t,uint64_t> m, e; | |
493 | auto p = bl.cbegin(); | |
494 | decode(m, p); | |
7c673cae FG |
495 | cout << " got " << m << std::endl; |
496 | ASSERT_TRUE(!m.empty()); | |
11fdf7f2 TL |
497 | // kstore always returns [0, object_size] regardless of offset and length |
498 | // FIXME: if fiemap logic in kstore is refined | |
499 | if (string(GetParam()) != "kstore") { | |
500 | ASSERT_GE(m[SKIP_STEP], 3u); | |
501 | auto last = m.crbegin(); | |
502 | if (m.size() == 1) { | |
503 | ASSERT_EQ(SKIP_STEP, last->first); | |
504 | } else if (m.size() == MAX_EXTENTS - 2) { | |
505 | for (uint64_t i = 1; i < MAX_EXTENTS - 1; i++) { | |
506 | ASSERT_TRUE(m.count(SKIP_STEP*i)); | |
507 | } | |
508 | } | |
509 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); | |
7c673cae | 510 | } |
7c673cae FG |
511 | } |
512 | { | |
513 | ObjectStore::Transaction t; | |
514 | t.remove(cid, oid); | |
515 | t.remove_collection(cid); | |
516 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 517 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
518 | ASSERT_EQ(r, 0); |
519 | } | |
520 | } | |
521 | ||
522 | TEST_P(StoreTest, SimpleMetaColTest) { | |
7c673cae FG |
523 | coll_t cid; |
524 | int r = 0; | |
525 | { | |
11fdf7f2 | 526 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
527 | ObjectStore::Transaction t; |
528 | t.create_collection(cid, 0); | |
529 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 530 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
531 | ASSERT_EQ(r, 0); |
532 | } | |
533 | { | |
534 | ObjectStore::Transaction t; | |
535 | t.remove_collection(cid); | |
536 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
537 | auto ch = store->open_collection(cid); |
538 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
539 | ASSERT_EQ(r, 0); |
540 | } | |
541 | { | |
11fdf7f2 | 542 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
543 | ObjectStore::Transaction t; |
544 | t.create_collection(cid, 0); | |
545 | cerr << "add collection" << std::endl; | |
11fdf7f2 | 546 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
547 | ASSERT_EQ(r, 0); |
548 | } | |
549 | { | |
550 | ObjectStore::Transaction t; | |
551 | t.remove_collection(cid); | |
552 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
553 | auto ch = store->open_collection(cid); |
554 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
555 | ASSERT_EQ(r, 0); |
556 | } | |
557 | } | |
558 | ||
559 | TEST_P(StoreTest, SimplePGColTest) { | |
7c673cae FG |
560 | coll_t cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD)); |
561 | int r = 0; | |
562 | { | |
563 | ObjectStore::Transaction t; | |
11fdf7f2 | 564 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
565 | t.create_collection(cid, 4); |
566 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 567 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
568 | ASSERT_EQ(r, 0); |
569 | } | |
570 | { | |
571 | ObjectStore::Transaction t; | |
572 | t.remove_collection(cid); | |
573 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
574 | auto ch = store->open_collection(cid); |
575 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
576 | ASSERT_EQ(r, 0); |
577 | } | |
578 | { | |
579 | ObjectStore::Transaction t; | |
580 | t.create_collection(cid, 4); | |
581 | cerr << "add collection" << std::endl; | |
11fdf7f2 TL |
582 | auto ch = store->create_new_collection(cid); |
583 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
584 | ASSERT_EQ(r, 0); |
585 | } | |
586 | { | |
587 | ObjectStore::Transaction t; | |
588 | t.remove_collection(cid); | |
589 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
590 | auto ch = store->open_collection(cid); |
591 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
592 | ASSERT_EQ(r, 0); |
593 | } | |
594 | } | |
595 | ||
596 | TEST_P(StoreTest, SimpleColPreHashTest) { | |
7c673cae FG |
597 | // Firstly we will need to revert the value making sure |
598 | // collection hint actually works | |
599 | int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold; | |
600 | std::ostringstream oss; | |
601 | if (merge_threshold > 0) { | |
602 | oss << "-" << merge_threshold; | |
11fdf7f2 | 603 | SetVal(g_conf(), "filestore_merge_threshold", oss.str().c_str()); |
7c673cae FG |
604 | } |
605 | ||
606 | uint32_t pg_num = 128; | |
607 | ||
608 | boost::uniform_int<> pg_id_range(0, pg_num); | |
609 | gen_type rng(time(NULL)); | |
610 | int pg_id = pg_id_range(rng); | |
611 | ||
612 | int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple; | |
613 | boost::uniform_int<> folders_range(5, 256); | |
614 | uint64_t expected_num_objs = (uint64_t)objs_per_folder * (uint64_t)folders_range(rng); | |
615 | ||
616 | coll_t cid(spg_t(pg_t(pg_id, 15), shard_id_t::NO_SHARD)); | |
617 | int r; | |
11fdf7f2 | 618 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
619 | { |
620 | // Create a collection along with a hint | |
621 | ObjectStore::Transaction t; | |
622 | t.create_collection(cid, 5); | |
623 | cerr << "create collection" << std::endl; | |
624 | bufferlist hint; | |
11fdf7f2 TL |
625 | encode(pg_num, hint); |
626 | encode(expected_num_objs, hint); | |
7c673cae FG |
627 | t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); |
628 | cerr << "collection hint" << std::endl; | |
11fdf7f2 | 629 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
630 | ASSERT_EQ(r, 0); |
631 | } | |
632 | { | |
633 | // Remove the collection | |
634 | ObjectStore::Transaction t; | |
635 | t.remove_collection(cid); | |
636 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 637 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
638 | ASSERT_EQ(r, 0); |
639 | } | |
7c673cae FG |
640 | } |
641 | ||
642 | TEST_P(StoreTest, SmallBlockWrites) { | |
7c673cae FG |
643 | int r; |
644 | coll_t cid; | |
11fdf7f2 | 645 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
646 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
647 | { | |
648 | ObjectStore::Transaction t; | |
649 | t.create_collection(cid, 0); | |
650 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 651 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
652 | ASSERT_EQ(r, 0); |
653 | } | |
654 | bufferlist a; | |
655 | bufferptr ap(0x1000); | |
656 | memset(ap.c_str(), 'a', 0x1000); | |
657 | a.append(ap); | |
658 | bufferlist b; | |
659 | bufferptr bp(0x1000); | |
660 | memset(bp.c_str(), 'b', 0x1000); | |
661 | b.append(bp); | |
662 | bufferlist c; | |
663 | bufferptr cp(0x1000); | |
664 | memset(cp.c_str(), 'c', 0x1000); | |
665 | c.append(cp); | |
666 | bufferptr zp(0x1000); | |
667 | zp.zero(); | |
668 | bufferlist z; | |
669 | z.append(zp); | |
670 | { | |
671 | ObjectStore::Transaction t; | |
672 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 673 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
674 | ASSERT_EQ(r, 0); |
675 | ||
676 | bufferlist in, exp; | |
11fdf7f2 | 677 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
678 | ASSERT_EQ(0x1000, r); |
679 | exp.append(a); | |
680 | ASSERT_TRUE(bl_eq(exp, in)); | |
681 | } | |
682 | { | |
683 | ObjectStore::Transaction t; | |
684 | t.write(cid, hoid, 0x1000, 0x1000, b); | |
11fdf7f2 | 685 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
686 | ASSERT_EQ(r, 0); |
687 | ||
688 | bufferlist in, exp; | |
11fdf7f2 | 689 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
690 | ASSERT_EQ(0x2000, r); |
691 | exp.append(a); | |
692 | exp.append(b); | |
693 | ASSERT_TRUE(bl_eq(exp, in)); | |
694 | } | |
695 | { | |
696 | ObjectStore::Transaction t; | |
697 | t.write(cid, hoid, 0x3000, 0x1000, c); | |
11fdf7f2 | 698 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
699 | ASSERT_EQ(r, 0); |
700 | ||
701 | bufferlist in, exp; | |
11fdf7f2 | 702 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
703 | ASSERT_EQ(0x4000, r); |
704 | exp.append(a); | |
705 | exp.append(b); | |
706 | exp.append(z); | |
707 | exp.append(c); | |
708 | ASSERT_TRUE(bl_eq(exp, in)); | |
709 | } | |
710 | { | |
711 | ObjectStore::Transaction t; | |
712 | t.write(cid, hoid, 0x2000, 0x1000, a); | |
11fdf7f2 | 713 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
714 | ASSERT_EQ(r, 0); |
715 | ||
716 | bufferlist in, exp; | |
11fdf7f2 | 717 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
718 | ASSERT_EQ(0x4000, r); |
719 | exp.append(a); | |
720 | exp.append(b); | |
721 | exp.append(a); | |
722 | exp.append(c); | |
723 | ASSERT_TRUE(bl_eq(exp, in)); | |
724 | } | |
725 | { | |
726 | ObjectStore::Transaction t; | |
727 | t.write(cid, hoid, 0, 0x1000, c); | |
11fdf7f2 | 728 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
729 | ASSERT_EQ(r, 0); |
730 | } | |
731 | { | |
732 | bufferlist in, exp; | |
11fdf7f2 | 733 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
734 | ASSERT_EQ(0x4000, r); |
735 | exp.append(c); | |
736 | exp.append(b); | |
737 | exp.append(a); | |
738 | exp.append(c); | |
739 | ASSERT_TRUE(bl_eq(exp, in)); | |
740 | } | |
741 | { | |
742 | ObjectStore::Transaction t; | |
743 | t.remove(cid, hoid); | |
744 | t.remove_collection(cid); | |
745 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 746 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
747 | ASSERT_EQ(r, 0); |
748 | } | |
749 | } | |
750 | ||
751 | TEST_P(StoreTest, BufferCacheReadTest) { | |
7c673cae FG |
752 | int r; |
753 | coll_t cid; | |
754 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
755 | { | |
11fdf7f2 TL |
756 | auto ch = store->open_collection(cid); |
757 | ASSERT_FALSE(ch); | |
7c673cae | 758 | } |
11fdf7f2 | 759 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
760 | { |
761 | ObjectStore::Transaction t; | |
762 | t.create_collection(cid, 0); | |
763 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 764 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
765 | ASSERT_EQ(r, 0); |
766 | } | |
767 | { | |
11fdf7f2 | 768 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
769 | ASSERT_TRUE(!exists); |
770 | ||
771 | ObjectStore::Transaction t; | |
772 | t.touch(cid, hoid); | |
773 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 774 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
775 | ASSERT_EQ(r, 0); |
776 | ||
11fdf7f2 | 777 | exists = store->exists(ch, hoid); |
7c673cae FG |
778 | ASSERT_EQ(true, exists); |
779 | } | |
780 | { | |
781 | ObjectStore::Transaction t; | |
782 | bufferlist bl, newdata; | |
783 | bl.append("abcde"); | |
784 | t.write(cid, hoid, 0, 5, bl); | |
785 | t.write(cid, hoid, 10, 5, bl); | |
786 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 787 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
788 | ASSERT_EQ(r, 0); |
789 | ||
11fdf7f2 | 790 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
791 | ASSERT_EQ(r, 15); |
792 | { | |
793 | bufferlist expected; | |
794 | expected.append(bl); | |
795 | expected.append_zero(5); | |
796 | expected.append(bl); | |
797 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
798 | } | |
799 | } | |
800 | //overwrite over the same extents | |
801 | { | |
802 | ObjectStore::Transaction t; | |
803 | bufferlist bl, newdata; | |
804 | bl.append("edcba"); | |
805 | t.write(cid, hoid, 0, 5, bl); | |
806 | t.write(cid, hoid, 10, 5, bl); | |
807 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 808 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
809 | ASSERT_EQ(r, 0); |
810 | ||
11fdf7f2 | 811 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
812 | ASSERT_EQ(r, 15); |
813 | { | |
814 | bufferlist expected; | |
815 | expected.append(bl); | |
816 | expected.append_zero(5); | |
817 | expected.append(bl); | |
818 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
819 | } | |
820 | } | |
821 | //additional write to an unused region of some blob | |
822 | { | |
823 | ObjectStore::Transaction t; | |
824 | bufferlist bl2, newdata; | |
825 | bl2.append("1234567890"); | |
826 | ||
827 | t.write(cid, hoid, 20, bl2.length(), bl2); | |
828 | cerr << "Append" << std::endl; | |
11fdf7f2 | 829 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
830 | ASSERT_EQ(r, 0); |
831 | ||
11fdf7f2 | 832 | r = store->read(ch, hoid, 0, 30, newdata); |
7c673cae FG |
833 | ASSERT_EQ(r, 30); |
834 | { | |
835 | bufferlist expected; | |
836 | expected.append("edcba"); | |
837 | expected.append_zero(5); | |
838 | expected.append("edcba"); | |
839 | expected.append_zero(5); | |
840 | expected.append(bl2); | |
841 | ||
842 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
843 | } | |
844 | } | |
845 | //additional write to an unused region of some blob and partial owerite over existing extents | |
846 | { | |
847 | ObjectStore::Transaction t; | |
848 | bufferlist bl, bl2, bl3, newdata; | |
849 | bl.append("DCB"); | |
850 | bl2.append("1234567890"); | |
851 | bl3.append("BA"); | |
852 | ||
853 | t.write(cid, hoid, 30, bl2.length(), bl2); | |
854 | t.write(cid, hoid, 1, bl.length(), bl); | |
855 | t.write(cid, hoid, 13, bl3.length(), bl3); | |
856 | cerr << "TripleWrite" << std::endl; | |
11fdf7f2 | 857 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
858 | ASSERT_EQ(r, 0); |
859 | ||
11fdf7f2 | 860 | r = store->read(ch, hoid, 0, 40, newdata); |
7c673cae FG |
861 | ASSERT_EQ(r, 40); |
862 | { | |
863 | bufferlist expected; | |
864 | expected.append("eDCBa"); | |
865 | expected.append_zero(5); | |
866 | expected.append("edcBA"); | |
867 | expected.append_zero(5); | |
868 | expected.append(bl2); | |
869 | expected.append(bl2); | |
870 | ||
871 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
872 | } | |
873 | } | |
874 | } | |
875 | ||
11fdf7f2 | 876 | void StoreTest::doCompressionTest() |
7c673cae | 877 | { |
7c673cae FG |
878 | int r; |
879 | coll_t cid; | |
880 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
881 | { | |
11fdf7f2 TL |
882 | auto ch = store->open_collection(cid); |
883 | ASSERT_FALSE(ch); | |
7c673cae | 884 | } |
11fdf7f2 | 885 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
886 | { |
887 | ObjectStore::Transaction t; | |
888 | t.create_collection(cid, 0); | |
889 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 890 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
891 | ASSERT_EQ(r, 0); |
892 | } | |
893 | { | |
11fdf7f2 | 894 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
895 | ASSERT_TRUE(!exists); |
896 | ||
897 | ObjectStore::Transaction t; | |
898 | t.touch(cid, hoid); | |
899 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 900 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
901 | ASSERT_EQ(r, 0); |
902 | ||
11fdf7f2 | 903 | exists = store->exists(ch, hoid); |
7c673cae FG |
904 | ASSERT_EQ(true, exists); |
905 | } | |
906 | std::string data; | |
907 | data.resize(0x10000 * 4); | |
908 | for(size_t i = 0;i < data.size(); i++) | |
909 | data[i] = i / 256; | |
910 | { | |
911 | ObjectStore::Transaction t; | |
912 | bufferlist bl, newdata; | |
913 | bl.append(data); | |
914 | t.write(cid, hoid, 0, bl.length(), bl); | |
915 | cerr << "CompressibleData (4xAU) Write" << std::endl; | |
11fdf7f2 | 916 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
917 | ASSERT_EQ(r, 0); |
918 | ||
11fdf7f2 | 919 | r = store->read(ch, hoid, 0, data.size() , newdata); |
7c673cae FG |
920 | |
921 | ASSERT_EQ(r, (int)data.size()); | |
922 | { | |
923 | bufferlist expected; | |
924 | expected.append(data); | |
925 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
926 | } | |
927 | newdata.clear(); | |
11fdf7f2 | 928 | r = store->read(ch, hoid, 0, 711 , newdata); |
7c673cae FG |
929 | ASSERT_EQ(r, 711); |
930 | { | |
931 | bufferlist expected; | |
932 | expected.append(data.substr(0,711)); | |
933 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
934 | } | |
935 | newdata.clear(); | |
11fdf7f2 | 936 | r = store->read(ch, hoid, 0xf00f, data.size(), newdata); |
7c673cae FG |
937 | ASSERT_EQ(r, int(data.size() - 0xf00f) ); |
938 | { | |
939 | bufferlist expected; | |
940 | expected.append(data.substr(0xf00f)); | |
941 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
942 | } | |
943 | { | |
944 | struct store_statfs_t statfs; | |
945 | int r = store->statfs(&statfs); | |
946 | ASSERT_EQ(r, 0); | |
11fdf7f2 TL |
947 | ASSERT_EQ(statfs.data_stored, (unsigned)data.size()); |
948 | ASSERT_LE(statfs.data_compressed, (unsigned)data.size()); | |
949 | ASSERT_EQ(statfs.data_compressed_original, (unsigned)data.size()); | |
950 | ASSERT_LE(statfs.data_compressed_allocated, (unsigned)data.size()); | |
7c673cae FG |
951 | } |
952 | } | |
953 | std::string data2; | |
954 | data2.resize(0x10000 * 4 - 0x9000); | |
955 | for(size_t i = 0;i < data2.size(); i++) | |
956 | data2[i] = (i+1) / 256; | |
957 | { | |
958 | ObjectStore::Transaction t; | |
959 | bufferlist bl, newdata; | |
960 | bl.append(data2); | |
961 | t.write(cid, hoid, 0x8000, bl.length(), bl); | |
962 | cerr << "CompressibleData partial overwrite" << std::endl; | |
11fdf7f2 | 963 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
964 | ASSERT_EQ(r, 0); |
965 | ||
11fdf7f2 | 966 | r = store->read(ch, hoid, 0, 0x10000, newdata); |
7c673cae FG |
967 | ASSERT_EQ(r, (int)0x10000); |
968 | { | |
969 | bufferlist expected; | |
970 | expected.append(data.substr(0, 0x8000)); | |
971 | expected.append(data2.substr(0, 0x8000)); | |
972 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
973 | } | |
974 | newdata.clear(); | |
11fdf7f2 | 975 | r = store->read(ch, hoid, 0x9000, 711 , newdata); |
7c673cae FG |
976 | ASSERT_EQ(r, 711); |
977 | { | |
978 | bufferlist expected; | |
979 | expected.append(data2.substr(0x1000,711)); | |
980 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
981 | } | |
982 | newdata.clear(); | |
11fdf7f2 | 983 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
984 | ASSERT_EQ(r, int(0x40000) ); |
985 | { | |
986 | bufferlist expected; | |
987 | expected.append(data.substr(0, 0x8000)); | |
988 | expected.append(data2.substr(0, 0x37000)); | |
989 | expected.append(data.substr(0x3f000, 0x1000)); | |
990 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
991 | } | |
992 | } | |
993 | data2.resize(0x3f000); | |
994 | for(size_t i = 0;i < data2.size(); i++) | |
995 | data2[i] = (i+2) / 256; | |
996 | { | |
997 | ObjectStore::Transaction t; | |
998 | bufferlist bl, newdata; | |
999 | bl.append(data2); | |
1000 | t.write(cid, hoid, 0, bl.length(), bl); | |
1001 | cerr << "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 1002 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1003 | ASSERT_EQ(r, 0); |
1004 | ||
11fdf7f2 | 1005 | r = store->read(ch, hoid, 0, 0x3e000 - 1, newdata); |
7c673cae FG |
1006 | ASSERT_EQ(r, (int)0x3e000 - 1); |
1007 | { | |
1008 | bufferlist expected; | |
1009 | expected.append(data2.substr(0, 0x3e000 - 1)); | |
1010 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1011 | } | |
1012 | newdata.clear(); | |
11fdf7f2 | 1013 | r = store->read(ch, hoid, 0x3e000-1, 0x2001, newdata); |
7c673cae FG |
1014 | ASSERT_EQ(r, 0x2001); |
1015 | { | |
1016 | bufferlist expected; | |
1017 | expected.append(data2.substr(0x3e000-1, 0x1001)); | |
1018 | expected.append(data.substr(0x3f000, 0x1000)); | |
1019 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1020 | } | |
1021 | newdata.clear(); | |
11fdf7f2 | 1022 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
1023 | ASSERT_EQ(r, int(0x40000) ); |
1024 | { | |
1025 | bufferlist expected; | |
1026 | expected.append(data2.substr(0, 0x3f000)); | |
1027 | expected.append(data.substr(0x3f000, 0x1000)); | |
1028 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1029 | } | |
1030 | } | |
1031 | data.resize(0x1001); | |
1032 | for(size_t i = 0;i < data.size(); i++) | |
1033 | data[i] = (i+3) / 256; | |
1034 | { | |
1035 | ObjectStore::Transaction t; | |
1036 | bufferlist bl, newdata; | |
1037 | bl.append(data); | |
1038 | t.write(cid, hoid, 0x3f000-1, bl.length(), bl); | |
1039 | cerr << "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 1040 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1041 | ASSERT_EQ(r, 0); |
1042 | ||
11fdf7f2 | 1043 | r = store->read(ch, hoid, 0x3e000, 0x2000, newdata); |
7c673cae FG |
1044 | ASSERT_EQ(r, (int)0x2000); |
1045 | { | |
1046 | bufferlist expected; | |
1047 | expected.append(data2.substr(0x3e000, 0x1000 - 1)); | |
1048 | expected.append(data.substr(0, 0x1001)); | |
1049 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1050 | } | |
1051 | } | |
1052 | { | |
1053 | ObjectStore::Transaction t; | |
1054 | t.remove(cid, hoid); | |
1055 | cerr << "Cleaning object" << std::endl; | |
11fdf7f2 | 1056 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1057 | ASSERT_EQ(r, 0); |
1058 | } | |
1059 | //force fsck | |
11fdf7f2 | 1060 | ch.reset(); |
7c673cae | 1061 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1062 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1063 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 TL |
1064 | ch = store->open_collection(cid); |
1065 | auto settingsBookmark = BookmarkSettings(); | |
1066 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
1067 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 1068 | { |
7c673cae FG |
1069 | data.resize(0x10000*6); |
1070 | ||
1071 | for(size_t i = 0;i < data.size(); i++) | |
1072 | data[i] = i / 256; | |
1073 | ObjectStore::Transaction t; | |
1074 | bufferlist bl, newdata; | |
1075 | bl.append(data); | |
1076 | t.write(cid, hoid, 0, bl.length(), bl); | |
1077 | cerr << "CompressibleData large blob" << std::endl; | |
11fdf7f2 | 1078 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1079 | ASSERT_EQ(r, 0); |
1080 | } | |
1081 | //force fsck | |
11fdf7f2 | 1082 | ch.reset(); |
7c673cae | 1083 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1084 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1085 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1086 | ch = store->open_collection(cid); |
7c673cae FG |
1087 | { |
1088 | ObjectStore::Transaction t; | |
1089 | t.remove(cid, hoid); | |
1090 | t.remove_collection(cid); | |
1091 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1092 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1093 | ASSERT_EQ(r, 0); |
1094 | } | |
7c673cae FG |
1095 | } |
1096 | ||
1097 | TEST_P(StoreTest, CompressionTest) { | |
1098 | if (string(GetParam()) != "bluestore") | |
1099 | return; | |
1100 | ||
11fdf7f2 TL |
1101 | SetVal(g_conf(), "bluestore_compression_algorithm", "snappy"); |
1102 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
1103 | g_ceph_context->_conf.apply_changes(nullptr); | |
1104 | doCompressionTest(); | |
7c673cae | 1105 | |
11fdf7f2 TL |
1106 | SetVal(g_conf(), "bluestore_compression_algorithm", "zlib"); |
1107 | SetVal(g_conf(), "bluestore_compression_mode", "aggressive"); | |
1108 | g_ceph_context->_conf.apply_changes(nullptr); | |
1109 | doCompressionTest(); | |
7c673cae FG |
1110 | } |
1111 | ||
1112 | TEST_P(StoreTest, SimpleObjectTest) { | |
7c673cae FG |
1113 | int r; |
1114 | coll_t cid; | |
1115 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1116 | { | |
11fdf7f2 TL |
1117 | auto ch = store->open_collection(cid); |
1118 | ASSERT_FALSE(ch); | |
7c673cae | 1119 | } |
11fdf7f2 | 1120 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1121 | { |
1122 | ObjectStore::Transaction t; | |
1123 | t.create_collection(cid, 0); | |
1124 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1125 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1126 | ASSERT_EQ(r, 0); |
1127 | } | |
1128 | { | |
11fdf7f2 | 1129 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1130 | ASSERT_TRUE(!exists); |
1131 | ||
1132 | ObjectStore::Transaction t; | |
1133 | t.touch(cid, hoid); | |
1134 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1135 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1136 | ASSERT_EQ(r, 0); |
1137 | ||
11fdf7f2 | 1138 | exists = store->exists(ch, hoid); |
7c673cae FG |
1139 | ASSERT_EQ(true, exists); |
1140 | } | |
1141 | { | |
1142 | ObjectStore::Transaction t; | |
1143 | t.remove(cid, hoid); | |
1144 | t.touch(cid, hoid); | |
1145 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1146 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1147 | ASSERT_EQ(r, 0); |
1148 | } | |
1149 | { | |
1150 | ObjectStore::Transaction t; | |
1151 | bufferlist bl, orig; | |
1152 | bl.append("abcde"); | |
1153 | orig = bl; | |
1154 | t.remove(cid, hoid); | |
1155 | t.write(cid, hoid, 0, 5, bl); | |
1156 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1157 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1158 | ASSERT_EQ(r, 0); |
1159 | ||
1160 | bufferlist in; | |
11fdf7f2 | 1161 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
1162 | ASSERT_EQ(5, r); |
1163 | ASSERT_TRUE(bl_eq(orig, in)); | |
1164 | } | |
1165 | { | |
1166 | ObjectStore::Transaction t; | |
1167 | bufferlist bl, exp; | |
1168 | bl.append("abcde"); | |
1169 | exp = bl; | |
1170 | exp.append(bl); | |
1171 | t.write(cid, hoid, 5, 5, bl); | |
1172 | cerr << "Append" << std::endl; | |
11fdf7f2 | 1173 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1174 | ASSERT_EQ(r, 0); |
1175 | ||
1176 | bufferlist in; | |
11fdf7f2 | 1177 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1178 | ASSERT_EQ(10, r); |
1179 | ASSERT_TRUE(bl_eq(exp, in)); | |
1180 | } | |
1181 | { | |
1182 | ObjectStore::Transaction t; | |
1183 | bufferlist bl, exp; | |
1184 | bl.append("abcdeabcde"); | |
1185 | exp = bl; | |
1186 | t.write(cid, hoid, 0, 10, bl); | |
1187 | cerr << "Full overwrite" << std::endl; | |
11fdf7f2 | 1188 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1189 | ASSERT_EQ(r, 0); |
1190 | ||
1191 | bufferlist in; | |
11fdf7f2 | 1192 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1193 | ASSERT_EQ(10, r); |
1194 | ASSERT_TRUE(bl_eq(exp, in)); | |
1195 | } | |
1196 | { | |
1197 | ObjectStore::Transaction t; | |
1198 | bufferlist bl; | |
1199 | bl.append("abcde"); | |
1200 | t.write(cid, hoid, 3, 5, bl); | |
1201 | cerr << "Partial overwrite" << std::endl; | |
11fdf7f2 | 1202 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1203 | ASSERT_EQ(r, 0); |
1204 | ||
1205 | bufferlist in, exp; | |
1206 | exp.append("abcabcdede"); | |
11fdf7f2 | 1207 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1208 | ASSERT_EQ(10, r); |
1209 | in.hexdump(cout); | |
1210 | ASSERT_TRUE(bl_eq(exp, in)); | |
1211 | } | |
1212 | { | |
1213 | { | |
1214 | ObjectStore::Transaction t; | |
1215 | bufferlist bl; | |
1216 | bl.append("fghij"); | |
1217 | t.truncate(cid, hoid, 0); | |
1218 | t.write(cid, hoid, 5, 5, bl); | |
1219 | cerr << "Truncate + hole" << std::endl; | |
11fdf7f2 | 1220 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1221 | ASSERT_EQ(r, 0); |
1222 | } | |
1223 | { | |
1224 | ObjectStore::Transaction t; | |
1225 | bufferlist bl; | |
1226 | bl.append("abcde"); | |
1227 | t.write(cid, hoid, 0, 5, bl); | |
1228 | cerr << "Reverse fill-in" << std::endl; | |
11fdf7f2 | 1229 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1230 | ASSERT_EQ(r, 0); |
1231 | } | |
1232 | ||
1233 | bufferlist in, exp; | |
1234 | exp.append("abcdefghij"); | |
11fdf7f2 | 1235 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1236 | ASSERT_EQ(10, r); |
1237 | in.hexdump(cout); | |
1238 | ASSERT_TRUE(bl_eq(exp, in)); | |
1239 | } | |
1240 | { | |
1241 | ObjectStore::Transaction t; | |
1242 | bufferlist bl; | |
1243 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1244 | t.write(cid, hoid, 0, bl.length(), bl); | |
1245 | cerr << "larger overwrite" << std::endl; | |
11fdf7f2 | 1246 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1247 | ASSERT_EQ(r, 0); |
1248 | ||
1249 | bufferlist in; | |
11fdf7f2 | 1250 | r = store->read(ch, hoid, 0, bl.length(), in); |
7c673cae FG |
1251 | ASSERT_EQ((int)bl.length(), r); |
1252 | in.hexdump(cout); | |
1253 | ASSERT_TRUE(bl_eq(bl, in)); | |
1254 | } | |
1255 | { | |
1256 | bufferlist bl; | |
1257 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1258 | ||
1259 | //test: offset=len=0 mean read all data | |
1260 | bufferlist in; | |
11fdf7f2 | 1261 | r = store->read(ch, hoid, 0, 0, in); |
7c673cae FG |
1262 | ASSERT_EQ((int)bl.length(), r); |
1263 | in.hexdump(cout); | |
1264 | ASSERT_TRUE(bl_eq(bl, in)); | |
1265 | } | |
1266 | { | |
1267 | //verifying unaligned csums | |
1268 | std::string s1("1"), s2(0x1000, '2'), s3("00"); | |
1269 | { | |
1270 | ObjectStore::Transaction t; | |
1271 | bufferlist bl; | |
1272 | bl.append(s1); | |
1273 | bl.append(s2); | |
1274 | t.truncate(cid, hoid, 0); | |
1275 | t.write(cid, hoid, 0x1000-1, bl.length(), bl); | |
1276 | cerr << "Write unaligned csum, stage 1" << std::endl; | |
11fdf7f2 | 1277 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1278 | ASSERT_EQ(r, 0); |
1279 | } | |
1280 | ||
1281 | bufferlist in, exp1, exp2, exp3; | |
1282 | exp1.append(s1); | |
1283 | exp2.append(s2); | |
1284 | exp3.append(s3); | |
11fdf7f2 | 1285 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1286 | ASSERT_EQ(1, r); |
1287 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1288 | in.clear(); | |
11fdf7f2 | 1289 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1290 | ASSERT_EQ(0x1000, r); |
1291 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1292 | ||
1293 | { | |
1294 | ObjectStore::Transaction t; | |
1295 | bufferlist bl; | |
1296 | bl.append(s3); | |
1297 | t.write(cid, hoid, 1, bl.length(), bl); | |
1298 | cerr << "Write unaligned csum, stage 2" << std::endl; | |
11fdf7f2 | 1299 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1300 | ASSERT_EQ(r, 0); |
1301 | } | |
1302 | in.clear(); | |
11fdf7f2 | 1303 | r = store->read(ch, hoid, 1, 2, in); |
7c673cae FG |
1304 | ASSERT_EQ(2, r); |
1305 | ASSERT_TRUE(bl_eq(exp3, in)); | |
1306 | in.clear(); | |
11fdf7f2 | 1307 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1308 | ASSERT_EQ(1, r); |
1309 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1310 | in.clear(); | |
11fdf7f2 | 1311 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1312 | ASSERT_EQ(0x1000, r); |
1313 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1314 | ||
1315 | } | |
1316 | ||
1317 | { | |
1318 | ObjectStore::Transaction t; | |
1319 | t.remove(cid, hoid); | |
1320 | t.remove_collection(cid); | |
1321 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1322 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1323 | ASSERT_EQ(r, 0); |
1324 | } | |
1325 | } | |
1326 | ||
11fdf7f2 TL |
1327 | #if defined(WITH_BLUESTORE) |
1328 | ||
1911f103 TL |
1329 | TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { |
1330 | if(string(GetParam()) != "bluestore") | |
1331 | return; | |
1332 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
1333 | g_conf().apply_changes(nullptr); | |
1334 | StartDeferred(65536); | |
1335 | ||
1336 | int r; | |
1337 | coll_t cid; | |
1338 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1339 | const PerfCounters* logger = store->get_perf_counters(); | |
1340 | auto ch = store->create_new_collection(cid); | |
1341 | { | |
1342 | ObjectStore::Transaction t; | |
1343 | t.create_collection(cid, 0); | |
1344 | cerr << "Creating collection " << cid << std::endl; | |
1345 | r = queue_transaction(store, ch, std::move(t)); | |
1346 | ASSERT_EQ(r, 0); | |
1347 | } | |
1348 | { | |
1349 | bool exists = store->exists(ch, hoid); | |
1350 | ASSERT_TRUE(!exists); | |
1351 | ||
1352 | ObjectStore::Transaction t; | |
1353 | t.touch(cid, hoid); | |
1354 | cerr << "Creating object " << hoid << std::endl; | |
1355 | r = queue_transaction(store, ch, std::move(t)); | |
1356 | ASSERT_EQ(r, 0); | |
1357 | ||
1358 | exists = store->exists(ch, hoid); | |
1359 | ASSERT_EQ(true, exists); | |
1360 | } | |
1361 | { | |
1362 | ObjectStore::Transaction t; | |
1363 | bufferlist bl, orig; | |
1364 | string s(4096, 'a'); | |
1365 | bl.append(s); | |
1366 | t.write(cid, hoid, 0x11000, bl.length(), bl); | |
1367 | cerr << "write1" << std::endl; | |
1368 | r = queue_transaction(store, ch, std::move(t)); | |
1369 | ASSERT_EQ(r, 0); | |
1370 | } | |
1371 | { | |
1372 | ObjectStore::Transaction t; | |
1373 | bufferlist bl, orig; | |
1374 | string s(4096 * 3, 'a'); | |
1375 | bl.append(s); | |
1376 | t.write(cid, hoid, 0x15000, bl.length(), bl); | |
1377 | cerr << "write2" << std::endl; | |
1378 | r = queue_transaction(store, ch, std::move(t)); | |
1379 | ASSERT_EQ(r, 0); | |
1380 | } | |
1381 | ASSERT_EQ(logger->get(l_bluestore_write_small), 2u); | |
1382 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 1u); | |
1383 | ||
1384 | { | |
1385 | ObjectStore::Transaction t; | |
1386 | bufferlist bl, orig; | |
1387 | string s(4096 * 2, 'a'); | |
1388 | bl.append(s); | |
1389 | t.write(cid, hoid, 0xe000, bl.length(), bl); | |
1390 | cerr << "write3" << std::endl; | |
1391 | r = queue_transaction(store, ch, std::move(t)); | |
1392 | ASSERT_EQ(r, 0); | |
1393 | } | |
1394 | ASSERT_EQ(logger->get(l_bluestore_write_small), 3u); | |
1395 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1396 | ||
1397 | ||
1398 | { | |
1399 | ObjectStore::Transaction t; | |
1400 | bufferlist bl, orig; | |
1401 | string s(4096, 'a'); | |
1402 | bl.append(s); | |
1403 | t.write(cid, hoid, 0xf000, bl.length(), bl); | |
1404 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1405 | cerr << "write3" << std::endl; | |
1406 | r = queue_transaction(store, ch, std::move(t)); | |
1407 | ASSERT_EQ(r, 0); | |
1408 | } | |
1409 | ASSERT_EQ(logger->get(l_bluestore_write_small), 5u); | |
1410 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1411 | { | |
1412 | ObjectStore::Transaction t; | |
1413 | t.remove(cid, hoid); | |
1414 | t.remove_collection(cid); | |
1415 | cerr << "Cleaning" << std::endl; | |
1416 | r = queue_transaction(store, ch, std::move(t)); | |
1417 | ASSERT_EQ(r, 0); | |
1418 | } | |
1419 | } | |
1420 | ||
1421 | ||
7c673cae FG |
1422 | TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { |
1423 | if(string(GetParam()) != "bluestore") | |
1424 | return; | |
1425 | StartDeferred(65536); | |
11fdf7f2 | 1426 | SetVal(g_conf(), "bluestore_compression_mode", "force"); |
eafe8130 | 1427 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); |
7c673cae | 1428 | // just a big number to disble gc |
11fdf7f2 TL |
1429 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "100000"); |
1430 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
1431 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
1432 | int r; |
1433 | ||
11fdf7f2 TL |
1434 | int poolid = 4373; |
1435 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
1436 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
1437 | string(), | |
1438 | 0, | |
1439 | poolid, | |
1440 | string())); | |
7c673cae FG |
1441 | ghobject_t hoid2 = hoid; |
1442 | hoid2.hobj.snap = 1; | |
1443 | { | |
11fdf7f2 TL |
1444 | auto ch = store->open_collection(cid); |
1445 | ASSERT_FALSE(ch); | |
7c673cae | 1446 | } |
11fdf7f2 | 1447 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1448 | { |
1449 | ObjectStore::Transaction t; | |
1450 | t.create_collection(cid, 0); | |
1451 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1452 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1453 | ASSERT_EQ(r, 0); |
1454 | } | |
1455 | { | |
11fdf7f2 | 1456 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1457 | ASSERT_TRUE(!exists); |
1458 | ||
1459 | ObjectStore::Transaction t; | |
1460 | t.touch(cid, hoid); | |
1461 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1462 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1463 | ASSERT_EQ(r, 0); |
1464 | ||
11fdf7f2 | 1465 | exists = store->exists(ch, hoid); |
7c673cae FG |
1466 | ASSERT_EQ(true, exists); |
1467 | } | |
1468 | { | |
1469 | struct store_statfs_t statfs; | |
1470 | int r = store->statfs(&statfs); | |
1471 | ASSERT_EQ(r, 0); | |
1472 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
1473 | ASSERT_EQ( 0u, statfs.data_stored); |
1474 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); | |
1475 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
1476 | ||
1477 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1478 | bool per_pool_omap; |
1479 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1480 | ASSERT_EQ(r, 0); |
1481 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
1482 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
1483 | ||
7c673cae | 1484 | //force fsck |
11fdf7f2 | 1485 | ch.reset(); |
7c673cae | 1486 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1487 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1488 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1489 | ch = store->open_collection(cid); |
7c673cae FG |
1490 | } |
1491 | { | |
1492 | ObjectStore::Transaction t; | |
1493 | bufferlist bl; | |
1494 | bl.append("abcde"); | |
1495 | t.write(cid, hoid, 0, 5, bl); | |
1496 | cerr << "Append 5 bytes" << std::endl; | |
11fdf7f2 | 1497 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1498 | ASSERT_EQ(r, 0); |
1499 | ||
1500 | struct store_statfs_t statfs; | |
1501 | int r = store->statfs(&statfs); | |
1502 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1503 | ASSERT_EQ(5, statfs.data_stored); |
7c673cae | 1504 | ASSERT_EQ(0x10000, statfs.allocated); |
11fdf7f2 TL |
1505 | ASSERT_EQ(0, statfs.data_compressed); |
1506 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1507 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1508 | ||
1509 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1510 | bool per_pool_omap; |
1511 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1512 | ASSERT_EQ(r, 0); |
1513 | ASSERT_EQ(5, statfs_pool.data_stored); | |
1514 | ASSERT_EQ(0x10000, statfs_pool.allocated); | |
1515 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1516 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1517 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1518 | ||
1519 | // accessing unknown pool | |
9f95a23c | 1520 | r = store->pool_statfs(poolid + 1, &statfs_pool, &per_pool_omap); |
11fdf7f2 TL |
1521 | ASSERT_EQ(r, 0); |
1522 | ASSERT_EQ(0, statfs_pool.data_stored); | |
1523 | ASSERT_EQ(0, statfs_pool.allocated); | |
1524 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1525 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1526 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1527 | ||
7c673cae | 1528 | //force fsck |
11fdf7f2 | 1529 | ch.reset(); |
7c673cae | 1530 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1531 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1532 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1533 | ch = store->open_collection(cid); |
7c673cae FG |
1534 | } |
1535 | { | |
1536 | ObjectStore::Transaction t; | |
1537 | std::string s(0x30000, 'a'); | |
1538 | bufferlist bl; | |
1539 | bl.append(s); | |
1540 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1541 | cerr << "Append 0x30000 compressible bytes" << std::endl; | |
11fdf7f2 | 1542 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1543 | ASSERT_EQ(r, 0); |
1544 | ||
1545 | struct store_statfs_t statfs; | |
1546 | int r = store->statfs(&statfs); | |
1547 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1548 | ASSERT_EQ(0x30005, statfs.data_stored); |
7c673cae | 1549 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1550 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1551 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1552 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1553 | ||
1554 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1555 | bool per_pool_omap; |
1556 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1557 | ASSERT_EQ(r, 0); |
1558 | ASSERT_EQ(0x30005, statfs_pool.data_stored); | |
1559 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1560 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1561 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1562 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1563 | //force fsck |
11fdf7f2 | 1564 | ch.reset(); |
7c673cae | 1565 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1566 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1567 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1568 | ch = store->open_collection(cid); |
7c673cae FG |
1569 | } |
1570 | { | |
1571 | ObjectStore::Transaction t; | |
1572 | t.zero(cid, hoid, 1, 3); | |
1573 | t.zero(cid, hoid, 0x20000, 9); | |
1574 | cerr << "Punch hole at 1~3, 0x20000~9" << std::endl; | |
11fdf7f2 | 1575 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1576 | ASSERT_EQ(r, 0); |
1577 | ||
1578 | struct store_statfs_t statfs; | |
1579 | int r = store->statfs(&statfs); | |
1580 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1581 | ASSERT_EQ(0x30005 - 3 - 9, statfs.data_stored); |
7c673cae | 1582 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1583 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1584 | ASSERT_EQ(0x20000 - 9, statfs.data_compressed_original); | |
1585 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1586 | ||
1587 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1588 | bool per_pool_omap; |
1589 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1590 | ASSERT_EQ(r, 0); |
1591 | ASSERT_EQ(0x30005 - 3 - 9, statfs_pool.data_stored); | |
1592 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1593 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1594 | ASSERT_EQ(0x20000 - 9, statfs_pool.data_compressed_original); | |
1595 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1596 | //force fsck |
11fdf7f2 | 1597 | ch.reset(); |
7c673cae | 1598 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1599 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1600 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1601 | ch = store->open_collection(cid); |
7c673cae FG |
1602 | } |
1603 | { | |
1604 | ObjectStore::Transaction t; | |
1605 | std::string s(0x1000, 'b'); | |
1606 | bufferlist bl; | |
1607 | bl.append(s); | |
1608 | t.write(cid, hoid, 1, bl.length(), bl); | |
1609 | t.write(cid, hoid, 0x10001, bl.length(), bl); | |
1610 | cerr << "Overwrite first and second(compressible) extents" << std::endl; | |
11fdf7f2 | 1611 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1612 | ASSERT_EQ(r, 0); |
1613 | ||
1614 | struct store_statfs_t statfs; | |
1615 | int r = store->statfs(&statfs); | |
1616 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1617 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs.data_stored); |
7c673cae | 1618 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1619 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1620 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs.data_compressed_original); | |
1621 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1622 | ||
1623 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1624 | bool per_pool_omap; |
1625 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1626 | ASSERT_EQ(r, 0); |
1627 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs_pool.data_stored); | |
1628 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1629 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1630 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs_pool.data_compressed_original); | |
1631 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1632 | //force fsck |
11fdf7f2 | 1633 | ch.reset(); |
7c673cae | 1634 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1635 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1636 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1637 | ch = store->open_collection(cid); |
7c673cae FG |
1638 | } |
1639 | { | |
1640 | ObjectStore::Transaction t; | |
1641 | std::string s(0x10000, 'c'); | |
1642 | bufferlist bl; | |
1643 | bl.append(s); | |
1644 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1645 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
1646 | t.write(cid, hoid, 0x30000, bl.length(), bl); | |
1647 | cerr << "Overwrite compressed extent with 3 uncompressible ones" << std::endl; | |
11fdf7f2 | 1648 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1649 | ASSERT_EQ(r, 0); |
1650 | ||
1651 | struct store_statfs_t statfs; | |
1652 | int r = store->statfs(&statfs); | |
1653 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1654 | ASSERT_EQ(0x30000 + 0x1001, statfs.data_stored); |
7c673cae | 1655 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1656 | ASSERT_LE(statfs.data_compressed, 0); |
1657 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1658 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1659 | ||
1660 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1661 | bool per_pool_omap; |
1662 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1663 | ASSERT_EQ(r, 0); |
1664 | ASSERT_EQ(0x30000 + 0x1001, statfs_pool.data_stored); | |
1665 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1666 | ASSERT_LE(statfs_pool.data_compressed, 0); | |
1667 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1668 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
7c673cae | 1669 | //force fsck |
11fdf7f2 | 1670 | ch.reset(); |
7c673cae | 1671 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1672 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1673 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1674 | ch = store->open_collection(cid); |
7c673cae FG |
1675 | } |
1676 | { | |
1677 | ObjectStore::Transaction t; | |
1678 | t.zero(cid, hoid, 0, 0x40000); | |
1679 | cerr << "Zero object" << std::endl; | |
11fdf7f2 | 1680 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1681 | ASSERT_EQ(r, 0); |
1682 | struct store_statfs_t statfs; | |
1683 | int r = store->statfs(&statfs); | |
1684 | ASSERT_EQ(r, 0); | |
1685 | ASSERT_EQ(0u, statfs.allocated); | |
11fdf7f2 TL |
1686 | ASSERT_EQ(0u, statfs.data_stored); |
1687 | ASSERT_EQ(0u, statfs.data_compressed_original); | |
1688 | ASSERT_EQ(0u, statfs.data_compressed); | |
1689 | ASSERT_EQ(0u, statfs.data_compressed_allocated); | |
1690 | ||
1691 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1692 | bool per_pool_omap; |
1693 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1694 | ASSERT_EQ(r, 0); |
1695 | ASSERT_EQ(0u, statfs_pool.allocated); | |
1696 | ASSERT_EQ(0u, statfs_pool.data_stored); | |
1697 | ASSERT_EQ(0u, statfs_pool.data_compressed_original); | |
1698 | ASSERT_EQ(0u, statfs_pool.data_compressed); | |
1699 | ASSERT_EQ(0u, statfs_pool.data_compressed_allocated); | |
7c673cae | 1700 | //force fsck |
11fdf7f2 | 1701 | ch.reset(); |
7c673cae | 1702 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1703 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1704 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1705 | ch = store->open_collection(cid); |
7c673cae FG |
1706 | } |
1707 | { | |
1708 | ObjectStore::Transaction t; | |
1709 | std::string s(0x10000, 'c'); | |
1710 | bufferlist bl; | |
1711 | bl.append(s); | |
1712 | bl.append(s); | |
1713 | bl.append(s); | |
1714 | bl.append(s.substr(0, 0x10000-2)); | |
1715 | t.write(cid, hoid, 0, bl.length(), bl); | |
1716 | cerr << "Yet another compressible write" << std::endl; | |
11fdf7f2 | 1717 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1718 | ASSERT_EQ(r, 0); |
1719 | struct store_statfs_t statfs; | |
1720 | r = store->statfs(&statfs); | |
1721 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1722 | ASSERT_EQ(0x40000 - 2, statfs.data_stored); |
7c673cae | 1723 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1724 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1725 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1726 | ASSERT_EQ(0x10000, statfs.data_compressed_allocated); | |
1727 | ||
1728 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1729 | bool per_pool_omap; |
1730 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1731 | ASSERT_EQ(r, 0); |
1732 | ASSERT_EQ(0x40000 - 2, statfs_pool.data_stored); | |
1733 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1734 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1735 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1736 | ASSERT_EQ(0x10000, statfs_pool.data_compressed_allocated); | |
7c673cae | 1737 | //force fsck |
11fdf7f2 | 1738 | ch.reset(); |
7c673cae | 1739 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1740 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1741 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1742 | ch = store->open_collection(cid); |
7c673cae FG |
1743 | } |
1744 | { | |
1745 | struct store_statfs_t statfs; | |
1746 | r = store->statfs(&statfs); | |
1747 | ASSERT_EQ(r, 0); | |
1748 | ||
11fdf7f2 | 1749 | struct store_statfs_t statfs_pool; |
9f95a23c TL |
1750 | bool per_pool_omap; |
1751 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1752 | ASSERT_EQ(r, 0); |
1753 | ||
7c673cae FG |
1754 | ObjectStore::Transaction t; |
1755 | t.clone(cid, hoid, hoid2); | |
1756 | cerr << "Clone compressed objecte" << std::endl; | |
11fdf7f2 | 1757 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1758 | ASSERT_EQ(r, 0); |
1759 | struct store_statfs_t statfs2; | |
1760 | r = store->statfs(&statfs2); | |
1761 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1762 | ASSERT_GT(statfs2.data_stored, statfs.data_stored); |
7c673cae | 1763 | ASSERT_EQ(statfs2.allocated, statfs.allocated); |
11fdf7f2 TL |
1764 | ASSERT_GT(statfs2.data_compressed, statfs.data_compressed); |
1765 | ASSERT_GT(statfs2.data_compressed_original, statfs.data_compressed_original); | |
1766 | ASSERT_EQ(statfs2.data_compressed_allocated, statfs.data_compressed_allocated); | |
1767 | ||
1768 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1769 | r = store->pool_statfs(poolid, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1770 | ASSERT_EQ(r, 0); |
1771 | ASSERT_GT(statfs2_pool.data_stored, statfs_pool.data_stored); | |
1772 | ASSERT_EQ(statfs2_pool.allocated, statfs_pool.allocated); | |
1773 | ASSERT_GT(statfs2_pool.data_compressed, statfs_pool.data_compressed); | |
1774 | ASSERT_GT(statfs2_pool.data_compressed_original, | |
1775 | statfs_pool.data_compressed_original); | |
1776 | ASSERT_EQ(statfs2_pool.data_compressed_allocated, | |
1777 | statfs_pool.data_compressed_allocated); | |
1778 | } | |
1779 | ||
1780 | { | |
1781 | // verify no | |
1782 | auto poolid2 = poolid + 1; | |
1783 | coll_t cid2 = coll_t(spg_t(pg_t(20, poolid2), shard_id_t::NO_SHARD)); | |
1784 | ghobject_t hoid(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
1785 | string(), | |
1786 | 0, | |
1787 | poolid2, | |
1788 | string())); | |
1789 | auto ch = store->create_new_collection(cid2); | |
1790 | ||
1791 | { | |
1792 | ||
1793 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
1794 | bool per_pool_omap; |
1795 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
1796 | ASSERT_EQ(r, 0); |
1797 | ||
1798 | cerr << "Creating second collection " << cid2 << std::endl; | |
1799 | ObjectStore::Transaction t; | |
1800 | t.create_collection(cid2, 0); | |
1801 | r = queue_transaction(store, ch, std::move(t)); | |
1802 | ASSERT_EQ(r, 0); | |
1803 | ||
1804 | t = ObjectStore::Transaction(); | |
1805 | bufferlist bl; | |
1806 | bl.append("abcde"); | |
1807 | t.write(cid2, hoid, 0, 5, bl); | |
1808 | r = queue_transaction(store, ch, std::move(t)); | |
1809 | ASSERT_EQ(r, 0); | |
1810 | ||
1811 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1812 | r = store->pool_statfs(poolid2, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1813 | ASSERT_EQ(r, 0); |
1814 | ASSERT_EQ(5, statfs2_pool.data_stored); | |
1815 | ASSERT_EQ(0x10000, statfs2_pool.allocated); | |
1816 | ASSERT_EQ(0, statfs2_pool.data_compressed); | |
1817 | ASSERT_EQ(0, statfs2_pool.data_compressed_original); | |
1818 | ASSERT_EQ(0, statfs2_pool.data_compressed_allocated); | |
1819 | ||
1820 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 1821 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1822 | ASSERT_EQ(r, 0); |
1823 | // adjust 'available' since it has changed | |
1824 | statfs1_pool_again.available = statfs1_pool.available; | |
1825 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
1826 | ||
1827 | t = ObjectStore::Transaction(); | |
1828 | t.remove(cid2, hoid); | |
1829 | t.remove_collection(cid2); | |
1830 | cerr << "Cleaning" << std::endl; | |
1831 | r = queue_transaction(store, ch, std::move(t)); | |
1832 | ASSERT_EQ(r, 0); | |
1833 | } | |
1834 | } | |
1835 | ||
1836 | { | |
1837 | // verify ops on temporary object | |
1838 | ||
1839 | auto poolid3 = poolid + 2; | |
1840 | coll_t cid3 = coll_t(spg_t(pg_t(20, poolid3), shard_id_t::NO_SHARD)); | |
1841 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP), | |
1842 | string(), | |
1843 | 0, | |
1844 | poolid3, | |
1845 | string())); | |
1846 | ghobject_t hoid3_temp; | |
1847 | hoid3_temp.hobj = hoid3.hobj.make_temp_hobject("Object 3 temp"); | |
1848 | auto ch3 = store->create_new_collection(cid3); | |
1849 | { | |
1850 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
1851 | bool per_pool_omap; |
1852 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
1853 | ASSERT_EQ(r, 0); |
1854 | ||
1855 | cerr << "Creating third collection " << cid3 << std::endl; | |
1856 | ObjectStore::Transaction t; | |
1857 | t.create_collection(cid3, 0); | |
1858 | r = queue_transaction(store, ch3, std::move(t)); | |
1859 | ASSERT_EQ(r, 0); | |
1860 | ||
1861 | t = ObjectStore::Transaction(); | |
1862 | bufferlist bl; | |
1863 | bl.append("abcde"); | |
1864 | t.write(cid3, hoid3_temp, 0, 5, bl); | |
1865 | r = queue_transaction(store, ch3, std::move(t)); | |
1866 | ASSERT_EQ(r, 0); | |
1867 | ||
1868 | struct store_statfs_t statfs3_pool; | |
9f95a23c | 1869 | r = store->pool_statfs(poolid3, &statfs3_pool, &per_pool_omap); |
11fdf7f2 TL |
1870 | ASSERT_EQ(r, 0); |
1871 | ASSERT_EQ(5, statfs3_pool.data_stored); | |
1872 | ASSERT_EQ(0x10000, statfs3_pool.allocated); | |
1873 | ASSERT_EQ(0, statfs3_pool.data_compressed); | |
1874 | ASSERT_EQ(0, statfs3_pool.data_compressed_original); | |
1875 | ASSERT_EQ(0, statfs3_pool.data_compressed_allocated); | |
1876 | ||
1877 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 1878 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1879 | ASSERT_EQ(r, 0); |
1880 | // adjust 'available' since it has changed | |
1881 | statfs1_pool_again.available = statfs1_pool.available; | |
1882 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
1883 | ||
1884 | //force fsck | |
1885 | ch.reset(); | |
1886 | ch3.reset(); | |
1887 | EXPECT_EQ(store->umount(), 0); | |
1888 | EXPECT_EQ(store->mount(), 0); | |
1889 | ch = store->open_collection(cid); | |
1890 | ch3 = store->open_collection(cid3); | |
1891 | ||
1892 | t = ObjectStore::Transaction(); | |
1893 | t.collection_move_rename( | |
1894 | cid3, hoid3_temp, | |
1895 | cid3, hoid3); | |
1896 | r = queue_transaction(store, ch3, std::move(t)); | |
1897 | ASSERT_EQ(r, 0); | |
1898 | ||
1899 | struct store_statfs_t statfs3_pool_again; | |
9f95a23c | 1900 | r = store->pool_statfs(poolid3, &statfs3_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1901 | ASSERT_EQ(r, 0); |
1902 | ASSERT_EQ(statfs3_pool_again, statfs3_pool); | |
1903 | ||
1904 | //force fsck | |
1905 | ch.reset(); | |
1906 | ch3.reset(); | |
1907 | EXPECT_EQ(store->umount(), 0); | |
1908 | EXPECT_EQ(store->mount(), 0); | |
1909 | ch = store->open_collection(cid); | |
1910 | ch3 = store->open_collection(cid3); | |
1911 | ||
1912 | t = ObjectStore::Transaction(); | |
1913 | t.remove(cid3, hoid3); | |
1914 | t.remove_collection(cid3); | |
1915 | cerr << "Cleaning" << std::endl; | |
1916 | r = queue_transaction(store, ch3, std::move(t)); | |
1917 | ASSERT_EQ(r, 0); | |
1918 | } | |
7c673cae FG |
1919 | } |
1920 | ||
1921 | { | |
1922 | ObjectStore::Transaction t; | |
1923 | t.remove(cid, hoid); | |
1924 | t.remove(cid, hoid2); | |
1925 | t.remove_collection(cid); | |
1926 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1927 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1928 | ASSERT_EQ(r, 0); |
1929 | ||
1930 | struct store_statfs_t statfs; | |
1931 | r = store->statfs(&statfs); | |
1932 | ASSERT_EQ(r, 0); | |
1933 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
1934 | ASSERT_EQ( 0u, statfs.data_stored); |
1935 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
1936 | ASSERT_EQ( 0u, statfs.data_compressed); | |
1937 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
1938 | ||
1939 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1940 | bool per_pool_omap; |
1941 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1942 | ASSERT_EQ(r, 0); |
1943 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
1944 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
1945 | ASSERT_EQ( 0u, statfs_pool.data_compressed_original); | |
1946 | ASSERT_EQ( 0u, statfs_pool.data_compressed); | |
1947 | ASSERT_EQ( 0u, statfs_pool.data_compressed_allocated); | |
1948 | } | |
7c673cae FG |
1949 | } |
1950 | ||
1951 | TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) { | |
1952 | if(string(GetParam()) != "bluestore") | |
1953 | return; | |
1954 | StartDeferred(0x10000); | |
1955 | ||
7c673cae FG |
1956 | int r; |
1957 | coll_t cid; | |
1958 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 1959 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1960 | { |
1961 | ObjectStore::Transaction t; | |
1962 | t.create_collection(cid, 0); | |
1963 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1964 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1965 | ASSERT_EQ(r, 0); |
1966 | } | |
1967 | { | |
11fdf7f2 | 1968 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1969 | ASSERT_TRUE(!exists); |
1970 | ||
1971 | ObjectStore::Transaction t; | |
1972 | t.touch(cid, hoid); | |
1973 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1974 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1975 | ASSERT_EQ(r, 0); |
1976 | ||
11fdf7f2 | 1977 | exists = store->exists(ch, hoid); |
7c673cae FG |
1978 | ASSERT_EQ(true, exists); |
1979 | } | |
1980 | { | |
1981 | struct store_statfs_t statfs; | |
1982 | int r = store->statfs(&statfs); | |
1983 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1984 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); |
7c673cae | 1985 | ASSERT_EQ(0u, statfs.allocated); |
11fdf7f2 TL |
1986 | ASSERT_EQ(0u, statfs.data_stored); |
1987 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
7c673cae FG |
1988 | } |
1989 | std::string data; | |
1990 | data.resize(0x10000 * 3); | |
1991 | { | |
1992 | ObjectStore::Transaction t; | |
1993 | for(size_t i = 0;i < data.size(); i++) | |
1994 | data[i] = i / 256 + 1; | |
1995 | bufferlist bl, newdata; | |
1996 | bl.append(data); | |
1997 | t.write(cid, hoid, 0, bl.length(), bl); | |
1998 | t.zero(cid, hoid, 0x10000, 0x10000); | |
1999 | cerr << "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl; | |
11fdf7f2 | 2000 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2001 | ASSERT_EQ(r, 0); |
2002 | ||
2003 | struct store_statfs_t statfs; | |
2004 | int r = store->statfs(&statfs); | |
2005 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 2006 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae FG |
2007 | ASSERT_EQ(0x20000, statfs.allocated); |
2008 | ||
11fdf7f2 | 2009 | r = store->read(ch, hoid, 0, data.size(), newdata); |
7c673cae FG |
2010 | ASSERT_EQ(r, (int)data.size()); |
2011 | { | |
2012 | bufferlist expected; | |
2013 | expected.append(data.substr(0, 0x10000)); | |
2014 | expected.append(string(0x10000, 0)); | |
2015 | expected.append(data.substr(0x20000, 0x10000)); | |
2016 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2017 | } | |
2018 | newdata.clear(); | |
2019 | ||
11fdf7f2 | 2020 | r = store->read(ch, hoid, 1, data.size()-2, newdata); |
7c673cae FG |
2021 | ASSERT_EQ(r, (int)data.size()-2); |
2022 | { | |
2023 | bufferlist expected; | |
2024 | expected.append(data.substr(1, 0x10000-1)); | |
2025 | expected.append(string(0x10000, 0)); | |
2026 | expected.append(data.substr(0x20000, 0x10000 - 1)); | |
2027 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2028 | } | |
2029 | newdata.clear(); | |
2030 | } | |
2031 | //force fsck | |
11fdf7f2 | 2032 | ch.reset(); |
7c673cae | 2033 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2034 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2035 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2036 | ch = store->open_collection(cid); |
7c673cae FG |
2037 | |
2038 | { | |
2039 | ObjectStore::Transaction t; | |
2040 | std::string data2(3, 'b'); | |
2041 | bufferlist bl, newdata; | |
2042 | bl.append(data2); | |
2043 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
2044 | cerr << "Write 3 bytes after the hole" << std::endl; | |
11fdf7f2 | 2045 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2046 | ASSERT_EQ(r, 0); |
2047 | ||
2048 | struct store_statfs_t statfs; | |
2049 | int r = store->statfs(&statfs); | |
2050 | ASSERT_EQ(r, 0); | |
2051 | ASSERT_EQ(0x20000, statfs.allocated); | |
11fdf7f2 | 2052 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae | 2053 | |
11fdf7f2 | 2054 | r = store->read(ch, hoid, 0x20000-1, 21, newdata); |
7c673cae FG |
2055 | ASSERT_EQ(r, (int)21); |
2056 | { | |
2057 | bufferlist expected; | |
2058 | expected.append(string(0x1, 0)); | |
2059 | expected.append(string(data2)); | |
2060 | expected.append(data.substr(0x20003, 21-4)); | |
2061 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2062 | } | |
2063 | newdata.clear(); | |
2064 | } | |
2065 | //force fsck | |
11fdf7f2 | 2066 | ch.reset(); |
7c673cae | 2067 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2068 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2069 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2070 | ch = store->open_collection(cid); |
7c673cae FG |
2071 | |
2072 | { | |
2073 | ObjectStore::Transaction t; | |
2074 | std::string data2(3, 'a'); | |
2075 | bufferlist bl, newdata; | |
2076 | bl.append(data2); | |
2077 | t.write(cid, hoid, 0x10000+1, bl.length(), bl); | |
2078 | cerr << "Write 3 bytes to the hole" << std::endl; | |
11fdf7f2 | 2079 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2080 | ASSERT_EQ(r, 0); |
2081 | ||
2082 | struct store_statfs_t statfs; | |
2083 | int r = store->statfs(&statfs); | |
2084 | ASSERT_EQ(r, 0); | |
2085 | ASSERT_EQ(0x30000, statfs.allocated); | |
11fdf7f2 | 2086 | ASSERT_EQ(0x20003, statfs.data_stored); |
7c673cae | 2087 | |
11fdf7f2 | 2088 | r = store->read(ch, hoid, 0x10000-1, 0x10000+22, newdata); |
7c673cae FG |
2089 | ASSERT_EQ(r, (int)0x10000+22); |
2090 | { | |
2091 | bufferlist expected; | |
2092 | expected.append(data.substr(0x10000-1, 1)); | |
2093 | expected.append(string(0x1, 0)); | |
2094 | expected.append(data2); | |
2095 | expected.append(string(0x10000-4, 0)); | |
2096 | expected.append(string(0x3, 'b')); | |
2097 | expected.append(data.substr(0x20004, 21-3)); | |
2098 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2099 | } | |
2100 | newdata.clear(); | |
2101 | } | |
2102 | { | |
2103 | ObjectStore::Transaction t; | |
2104 | bufferlist bl, newdata; | |
2105 | bl.append(string(0x30000, 'c')); | |
2106 | t.write(cid, hoid, 0, 0x30000, bl); | |
2107 | t.zero(cid, hoid, 0, 0x10000); | |
2108 | t.zero(cid, hoid, 0x20000, 0x10000); | |
11fdf7f2 TL |
2109 | cerr << "Rewrite an object and create two holes at the beginning and the end" << std::endl; |
2110 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
2111 | ASSERT_EQ(r, 0); |
2112 | ||
2113 | struct store_statfs_t statfs; | |
2114 | int r = store->statfs(&statfs); | |
2115 | ASSERT_EQ(r, 0); | |
2116 | ASSERT_EQ(0x10000, statfs.allocated); | |
11fdf7f2 | 2117 | ASSERT_EQ(0x10000, statfs.data_stored); |
7c673cae | 2118 | |
11fdf7f2 | 2119 | r = store->read(ch, hoid, 0, 0x30000, newdata); |
7c673cae FG |
2120 | ASSERT_EQ(r, (int)0x30000); |
2121 | { | |
2122 | bufferlist expected; | |
2123 | expected.append(string(0x10000, 0)); | |
2124 | expected.append(string(0x10000, 'c')); | |
2125 | expected.append(string(0x10000, 0)); | |
2126 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2127 | } | |
2128 | newdata.clear(); | |
2129 | } | |
2130 | ||
2131 | //force fsck | |
11fdf7f2 | 2132 | ch.reset(); |
7c673cae | 2133 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2134 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2135 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2136 | ch = store->open_collection(cid); |
7c673cae FG |
2137 | |
2138 | { | |
2139 | ObjectStore::Transaction t; | |
2140 | t.remove(cid, hoid); | |
2141 | t.remove_collection(cid); | |
2142 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2143 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2144 | ASSERT_EQ(r, 0); |
2145 | ||
2146 | struct store_statfs_t statfs; | |
2147 | r = store->statfs(&statfs); | |
2148 | ASSERT_EQ(r, 0); | |
2149 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
2150 | ASSERT_EQ( 0u, statfs.data_stored); |
2151 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
2152 | ASSERT_EQ( 0u, statfs.data_compressed); | |
2153 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
7c673cae FG |
2154 | } |
2155 | } | |
2156 | #endif | |
2157 | ||
2158 | TEST_P(StoreTest, ManySmallWrite) { | |
7c673cae FG |
2159 | int r; |
2160 | coll_t cid; | |
2161 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2162 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2163 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2164 | { |
2165 | ObjectStore::Transaction t; | |
2166 | t.create_collection(cid, 0); | |
2167 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2168 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2169 | ASSERT_EQ(r, 0); |
2170 | } | |
2171 | bufferlist bl; | |
2172 | bufferptr bp(4096); | |
2173 | bp.zero(); | |
2174 | bl.append(bp); | |
2175 | for (int i=0; i<100; ++i) { | |
2176 | ObjectStore::Transaction t; | |
2177 | t.write(cid, a, i*4096, 4096, bl, 0); | |
11fdf7f2 | 2178 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2179 | ASSERT_EQ(r, 0); |
2180 | } | |
2181 | for (int i=0; i<100; ++i) { | |
2182 | ObjectStore::Transaction t; | |
2183 | t.write(cid, b, (rand() % 1024)*4096, 4096, bl, 0); | |
11fdf7f2 | 2184 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2185 | ASSERT_EQ(r, 0); |
2186 | } | |
2187 | { | |
2188 | ObjectStore::Transaction t; | |
2189 | t.remove(cid, a); | |
2190 | t.remove(cid, b); | |
2191 | t.remove_collection(cid); | |
2192 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2193 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2194 | ASSERT_EQ(r, 0); |
2195 | } | |
2196 | } | |
2197 | ||
2198 | TEST_P(StoreTest, MultiSmallWriteSameBlock) { | |
7c673cae FG |
2199 | int r; |
2200 | coll_t cid; | |
2201 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2202 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2203 | { |
2204 | ObjectStore::Transaction t; | |
2205 | t.create_collection(cid, 0); | |
2206 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2207 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2208 | ASSERT_EQ(r, 0); |
2209 | } | |
2210 | bufferlist bl; | |
2211 | bl.append("short"); | |
2212 | C_SaferCond c, d; | |
2213 | // touch same block in both same transaction, tls, and pipelined txns | |
2214 | { | |
2215 | ObjectStore::Transaction t, u; | |
2216 | t.write(cid, a, 0, 5, bl, 0); | |
2217 | t.write(cid, a, 5, 5, bl, 0); | |
2218 | t.write(cid, a, 4094, 5, bl, 0); | |
2219 | t.write(cid, a, 9000, 5, bl, 0); | |
2220 | u.write(cid, a, 10, 5, bl, 0); | |
2221 | u.write(cid, a, 7000, 5, bl, 0); | |
11fdf7f2 | 2222 | t.register_on_commit(&c); |
7c673cae | 2223 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2224 | store->queue_transactions(ch, v); |
7c673cae FG |
2225 | } |
2226 | { | |
2227 | ObjectStore::Transaction t, u; | |
2228 | t.write(cid, a, 40, 5, bl, 0); | |
2229 | t.write(cid, a, 45, 5, bl, 0); | |
2230 | t.write(cid, a, 4094, 5, bl, 0); | |
2231 | t.write(cid, a, 6000, 5, bl, 0); | |
2232 | u.write(cid, a, 610, 5, bl, 0); | |
2233 | u.write(cid, a, 11000, 5, bl, 0); | |
11fdf7f2 | 2234 | t.register_on_commit(&d); |
7c673cae | 2235 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2236 | store->queue_transactions(ch, v); |
7c673cae FG |
2237 | } |
2238 | c.wait(); | |
2239 | d.wait(); | |
2240 | { | |
2241 | bufferlist bl2; | |
11fdf7f2 | 2242 | r = store->read(ch, a, 0, 16000, bl2); |
7c673cae FG |
2243 | ASSERT_GE(r, 0); |
2244 | } | |
2245 | { | |
2246 | ObjectStore::Transaction t; | |
2247 | t.remove(cid, a); | |
2248 | t.remove_collection(cid); | |
2249 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2250 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2251 | ASSERT_EQ(r, 0); |
2252 | } | |
2253 | } | |
2254 | ||
2255 | TEST_P(StoreTest, SmallSkipFront) { | |
7c673cae FG |
2256 | int r; |
2257 | coll_t cid; | |
2258 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2259 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2260 | { |
2261 | ObjectStore::Transaction t; | |
2262 | t.create_collection(cid, 0); | |
2263 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2264 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2265 | ASSERT_EQ(r, 0); |
2266 | } | |
2267 | { | |
2268 | ObjectStore::Transaction t; | |
2269 | t.touch(cid, a); | |
2270 | t.truncate(cid, a, 3000); | |
11fdf7f2 | 2271 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2272 | ASSERT_EQ(r, 0); |
2273 | } | |
2274 | { | |
2275 | bufferlist bl; | |
2276 | bufferptr bp(4096); | |
2277 | memset(bp.c_str(), 1, 4096); | |
2278 | bl.append(bp); | |
2279 | ObjectStore::Transaction t; | |
2280 | t.write(cid, a, 4096, 4096, bl); | |
11fdf7f2 | 2281 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2282 | ASSERT_EQ(r, 0); |
2283 | } | |
2284 | { | |
2285 | bufferlist bl; | |
11fdf7f2 | 2286 | ASSERT_EQ(8192, store->read(ch, a, 0, 8192, bl)); |
7c673cae FG |
2287 | for (unsigned i=0; i<4096; ++i) |
2288 | ASSERT_EQ(0, bl[i]); | |
2289 | for (unsigned i=4096; i<8192; ++i) | |
2290 | ASSERT_EQ(1, bl[i]); | |
2291 | } | |
2292 | { | |
2293 | ObjectStore::Transaction t; | |
2294 | t.remove(cid, a); | |
2295 | t.remove_collection(cid); | |
2296 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2297 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2298 | ASSERT_EQ(r, 0); |
2299 | } | |
2300 | } | |
2301 | ||
2302 | TEST_P(StoreTest, AppendDeferredVsTailCache) { | |
7c673cae FG |
2303 | int r; |
2304 | coll_t cid; | |
2305 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
11fdf7f2 | 2306 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2307 | { |
2308 | ObjectStore::Transaction t; | |
2309 | t.create_collection(cid, 0); | |
2310 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2311 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2312 | ASSERT_EQ(r, 0); |
2313 | } | |
11fdf7f2 | 2314 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2315 | unsigned size = min_alloc / 3; |
2316 | bufferptr bpa(size); | |
2317 | memset(bpa.c_str(), 1, bpa.length()); | |
2318 | bufferlist bla; | |
2319 | bla.append(bpa); | |
2320 | { | |
2321 | ObjectStore::Transaction t; | |
2322 | t.write(cid, a, 0, bla.length(), bla, 0); | |
11fdf7f2 | 2323 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2324 | ASSERT_EQ(r, 0); |
2325 | } | |
2326 | ||
2327 | // force cached tail to clear ... | |
2328 | { | |
11fdf7f2 | 2329 | ch.reset(); |
7c673cae FG |
2330 | int r = store->umount(); |
2331 | ASSERT_EQ(0, r); | |
2332 | r = store->mount(); | |
2333 | ASSERT_EQ(0, r); | |
11fdf7f2 | 2334 | ch = store->open_collection(cid); |
7c673cae FG |
2335 | } |
2336 | ||
2337 | bufferptr bpb(size); | |
2338 | memset(bpb.c_str(), 2, bpb.length()); | |
2339 | bufferlist blb; | |
2340 | blb.append(bpb); | |
2341 | { | |
2342 | ObjectStore::Transaction t; | |
2343 | t.write(cid, a, bla.length(), blb.length(), blb, 0); | |
11fdf7f2 | 2344 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2345 | ASSERT_EQ(r, 0); |
2346 | } | |
2347 | bufferptr bpc(size); | |
2348 | memset(bpc.c_str(), 3, bpc.length()); | |
2349 | bufferlist blc; | |
2350 | blc.append(bpc); | |
2351 | { | |
2352 | ObjectStore::Transaction t; | |
2353 | t.write(cid, a, bla.length() + blb.length(), blc.length(), blc, 0); | |
11fdf7f2 | 2354 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2355 | ASSERT_EQ(r, 0); |
2356 | } | |
2357 | bufferlist final; | |
2358 | final.append(bla); | |
2359 | final.append(blb); | |
2360 | final.append(blc); | |
2361 | bufferlist actual; | |
2362 | { | |
2363 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2364 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2365 | ASSERT_TRUE(bl_eq(final, actual)); |
2366 | } | |
2367 | { | |
2368 | ObjectStore::Transaction t; | |
2369 | t.remove(cid, a); | |
2370 | t.remove_collection(cid); | |
2371 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2372 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2373 | ASSERT_EQ(r, 0); |
2374 | } | |
7c673cae FG |
2375 | } |
2376 | ||
2377 | TEST_P(StoreTest, AppendZeroTrailingSharedBlock) { | |
7c673cae FG |
2378 | int r; |
2379 | coll_t cid; | |
2380 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
2381 | ghobject_t b = a; | |
2382 | b.hobj.snap = 1; | |
11fdf7f2 | 2383 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2384 | { |
2385 | ObjectStore::Transaction t; | |
2386 | t.create_collection(cid, 0); | |
2387 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2388 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2389 | ASSERT_EQ(r, 0); |
2390 | } | |
11fdf7f2 | 2391 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2392 | unsigned size = min_alloc / 3; |
2393 | bufferptr bpa(size); | |
2394 | memset(bpa.c_str(), 1, bpa.length()); | |
2395 | bufferlist bla; | |
2396 | bla.append(bpa); | |
2397 | // make sure there is some trailing gunk in the last block | |
2398 | { | |
2399 | bufferlist bt; | |
2400 | bt.append(bla); | |
2401 | bt.append("BADBADBADBAD"); | |
2402 | ObjectStore::Transaction t; | |
2403 | t.write(cid, a, 0, bt.length(), bt, 0); | |
11fdf7f2 | 2404 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2405 | ASSERT_EQ(r, 0); |
2406 | } | |
2407 | { | |
2408 | ObjectStore::Transaction t; | |
2409 | t.truncate(cid, a, size); | |
11fdf7f2 | 2410 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2411 | ASSERT_EQ(r, 0); |
2412 | } | |
2413 | ||
2414 | // clone | |
2415 | { | |
2416 | ObjectStore::Transaction t; | |
2417 | t.clone(cid, a, b); | |
11fdf7f2 | 2418 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2419 | ASSERT_EQ(r, 0); |
2420 | } | |
2421 | ||
2422 | // append with implicit zeroing | |
2423 | bufferptr bpb(size); | |
2424 | memset(bpb.c_str(), 2, bpb.length()); | |
2425 | bufferlist blb; | |
2426 | blb.append(bpb); | |
2427 | { | |
2428 | ObjectStore::Transaction t; | |
2429 | t.write(cid, a, min_alloc * 3, blb.length(), blb, 0); | |
11fdf7f2 | 2430 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2431 | ASSERT_EQ(r, 0); |
2432 | } | |
2433 | bufferlist final; | |
2434 | final.append(bla); | |
2435 | bufferlist zeros; | |
2436 | zeros.append_zero(min_alloc * 3 - size); | |
2437 | final.append(zeros); | |
2438 | final.append(blb); | |
2439 | bufferlist actual; | |
2440 | { | |
2441 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2442 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2443 | final.hexdump(cout); |
2444 | actual.hexdump(cout); | |
2445 | ASSERT_TRUE(bl_eq(final, actual)); | |
2446 | } | |
2447 | { | |
2448 | ObjectStore::Transaction t; | |
2449 | t.remove(cid, a); | |
2450 | t.remove(cid, b); | |
2451 | t.remove_collection(cid); | |
2452 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2453 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2454 | ASSERT_EQ(r, 0); |
2455 | } | |
2456 | } | |
2457 | ||
2458 | TEST_P(StoreTest, SmallSequentialUnaligned) { | |
7c673cae FG |
2459 | int r; |
2460 | coll_t cid; | |
2461 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2462 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2463 | { |
2464 | ObjectStore::Transaction t; | |
2465 | t.create_collection(cid, 0); | |
2466 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2467 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2468 | ASSERT_EQ(r, 0); |
2469 | } | |
2470 | bufferlist bl; | |
2471 | int len = 1000; | |
2472 | bufferptr bp(len); | |
2473 | bp.zero(); | |
2474 | bl.append(bp); | |
2475 | for (int i=0; i<1000; ++i) { | |
2476 | ObjectStore::Transaction t; | |
2477 | t.write(cid, a, i*len, len, bl, 0); | |
11fdf7f2 | 2478 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2479 | ASSERT_EQ(r, 0); |
2480 | } | |
2481 | { | |
2482 | ObjectStore::Transaction t; | |
2483 | t.remove(cid, a); | |
2484 | t.remove_collection(cid); | |
2485 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2486 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2487 | ASSERT_EQ(r, 0); |
2488 | } | |
2489 | } | |
2490 | ||
2491 | TEST_P(StoreTest, ManyBigWrite) { | |
7c673cae FG |
2492 | int r; |
2493 | coll_t cid; | |
2494 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2495 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2496 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2497 | { |
2498 | ObjectStore::Transaction t; | |
2499 | t.create_collection(cid, 0); | |
2500 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2501 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2502 | ASSERT_EQ(r, 0); |
2503 | } | |
2504 | bufferlist bl; | |
2505 | bufferptr bp(4 * 1048576); | |
2506 | bp.zero(); | |
2507 | bl.append(bp); | |
2508 | for (int i=0; i<10; ++i) { | |
2509 | ObjectStore::Transaction t; | |
2510 | t.write(cid, a, i*4*1048586, 4*1048576, bl, 0); | |
11fdf7f2 | 2511 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2512 | ASSERT_EQ(r, 0); |
2513 | } | |
2514 | // aligned | |
2515 | for (int i=0; i<10; ++i) { | |
2516 | ObjectStore::Transaction t; | |
2517 | t.write(cid, b, (rand() % 256)*4*1048576, 4*1048576, bl, 0); | |
11fdf7f2 | 2518 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2519 | ASSERT_EQ(r, 0); |
2520 | } | |
2521 | // unaligned | |
2522 | for (int i=0; i<10; ++i) { | |
2523 | ObjectStore::Transaction t; | |
2524 | t.write(cid, b, (rand() % (256*4096))*1024, 4*1048576, bl, 0); | |
11fdf7f2 | 2525 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2526 | ASSERT_EQ(r, 0); |
2527 | } | |
2528 | // do some zeros | |
2529 | for (int i=0; i<10; ++i) { | |
2530 | ObjectStore::Transaction t; | |
2531 | t.zero(cid, b, (rand() % (256*4096))*1024, 16*1048576); | |
11fdf7f2 | 2532 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2533 | ASSERT_EQ(r, 0); |
2534 | } | |
2535 | { | |
2536 | ObjectStore::Transaction t; | |
2537 | t.remove(cid, a); | |
2538 | t.remove(cid, b); | |
2539 | t.remove_collection(cid); | |
2540 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2541 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2542 | ASSERT_EQ(r, 0); |
2543 | } | |
2544 | } | |
2545 | ||
2546 | TEST_P(StoreTest, BigWriteBigZero) { | |
7c673cae FG |
2547 | int r; |
2548 | coll_t cid; | |
2549 | ghobject_t a(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2550 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2551 | { |
2552 | ObjectStore::Transaction t; | |
2553 | t.create_collection(cid, 0); | |
11fdf7f2 | 2554 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2555 | ASSERT_EQ(r, 0); |
2556 | } | |
2557 | bufferlist bl; | |
2558 | bufferptr bp(1048576); | |
2559 | memset(bp.c_str(), 'b', bp.length()); | |
2560 | bl.append(bp); | |
2561 | bufferlist s; | |
2562 | bufferptr sp(4096); | |
2563 | memset(sp.c_str(), 's', sp.length()); | |
2564 | s.append(sp); | |
2565 | { | |
2566 | ObjectStore::Transaction t; | |
2567 | t.write(cid, a, 0, bl.length(), bl); | |
11fdf7f2 | 2568 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2569 | ASSERT_EQ(r, 0); |
2570 | } | |
2571 | { | |
2572 | ObjectStore::Transaction t; | |
2573 | t.zero(cid, a, bl.length() / 4, bl.length() / 2); | |
11fdf7f2 | 2574 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2575 | ASSERT_EQ(r, 0); |
2576 | } | |
2577 | { | |
2578 | ObjectStore::Transaction t; | |
2579 | t.write(cid, a, bl.length() / 2, s.length(), s); | |
11fdf7f2 | 2580 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2581 | ASSERT_EQ(r, 0); |
2582 | } | |
2583 | { | |
2584 | ObjectStore::Transaction t; | |
2585 | t.remove(cid, a); | |
2586 | t.remove_collection(cid); | |
11fdf7f2 | 2587 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2588 | ASSERT_EQ(r, 0); |
2589 | } | |
2590 | } | |
2591 | ||
2592 | TEST_P(StoreTest, MiscFragmentTests) { | |
7c673cae FG |
2593 | int r; |
2594 | coll_t cid; | |
2595 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2596 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2597 | { |
2598 | ObjectStore::Transaction t; | |
2599 | t.create_collection(cid, 0); | |
2600 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2601 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2602 | ASSERT_EQ(r, 0); |
2603 | } | |
2604 | bufferlist bl; | |
2605 | bufferptr bp(524288); | |
2606 | bp.zero(); | |
2607 | bl.append(bp); | |
2608 | { | |
2609 | ObjectStore::Transaction t; | |
2610 | t.write(cid, a, 0, 524288, bl, 0); | |
11fdf7f2 | 2611 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2612 | ASSERT_EQ(r, 0); |
2613 | } | |
2614 | { | |
2615 | ObjectStore::Transaction t; | |
2616 | t.write(cid, a, 1048576, 524288, bl, 0); | |
11fdf7f2 | 2617 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2618 | ASSERT_EQ(r, 0); |
2619 | } | |
2620 | { | |
2621 | bufferlist inbl; | |
11fdf7f2 | 2622 | int r = store->read(ch, a, 524288 + 131072, 1024, inbl); |
7c673cae FG |
2623 | ASSERT_EQ(r, 1024); |
2624 | ASSERT_EQ(inbl.length(), 1024u); | |
2625 | ASSERT_TRUE(inbl.is_zero()); | |
2626 | } | |
2627 | { | |
2628 | ObjectStore::Transaction t; | |
2629 | t.write(cid, a, 1048576 - 4096, 524288, bl, 0); | |
11fdf7f2 | 2630 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2631 | ASSERT_EQ(r, 0); |
2632 | } | |
2633 | { | |
2634 | ObjectStore::Transaction t; | |
2635 | t.remove(cid, a); | |
2636 | t.remove_collection(cid); | |
2637 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2638 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2639 | ASSERT_EQ(r, 0); |
2640 | } | |
2641 | ||
2642 | } | |
2643 | ||
b32b8144 | 2644 | TEST_P(StoreTest, ZeroVsObjectSize) { |
b32b8144 FG |
2645 | int r; |
2646 | coll_t cid; | |
2647 | struct stat stat; | |
2648 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2649 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2650 | { |
2651 | ObjectStore::Transaction t; | |
2652 | t.create_collection(cid, 0); | |
2653 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2654 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2655 | ASSERT_EQ(r, 0); |
2656 | } | |
2657 | bufferlist a; | |
2658 | a.append("stuff"); | |
2659 | { | |
2660 | ObjectStore::Transaction t; | |
2661 | t.write(cid, hoid, 0, 5, a); | |
11fdf7f2 | 2662 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2663 | ASSERT_EQ(r, 0); |
2664 | } | |
11fdf7f2 | 2665 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2666 | ASSERT_EQ(5, stat.st_size); |
2667 | { | |
2668 | ObjectStore::Transaction t; | |
2669 | t.zero(cid, hoid, 1, 2); | |
11fdf7f2 | 2670 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2671 | ASSERT_EQ(r, 0); |
2672 | } | |
11fdf7f2 | 2673 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2674 | ASSERT_EQ(5, stat.st_size); |
2675 | { | |
2676 | ObjectStore::Transaction t; | |
2677 | t.zero(cid, hoid, 3, 200); | |
11fdf7f2 | 2678 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2679 | ASSERT_EQ(r, 0); |
2680 | } | |
11fdf7f2 | 2681 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2682 | ASSERT_EQ(203, stat.st_size); |
2683 | { | |
2684 | ObjectStore::Transaction t; | |
2685 | t.zero(cid, hoid, 100000, 200); | |
11fdf7f2 | 2686 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2687 | ASSERT_EQ(r, 0); |
2688 | } | |
11fdf7f2 | 2689 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2690 | ASSERT_EQ(100200, stat.st_size); |
2691 | } | |
2692 | ||
7c673cae | 2693 | TEST_P(StoreTest, ZeroLengthWrite) { |
7c673cae FG |
2694 | int r; |
2695 | coll_t cid; | |
2696 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2697 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2698 | { |
2699 | ObjectStore::Transaction t; | |
2700 | t.create_collection(cid, 0); | |
2701 | t.touch(cid, hoid); | |
11fdf7f2 | 2702 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2703 | ASSERT_EQ(r, 0); |
2704 | } | |
2705 | { | |
2706 | ObjectStore::Transaction t; | |
2707 | bufferlist empty; | |
2708 | t.write(cid, hoid, 1048576, 0, empty); | |
11fdf7f2 | 2709 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2710 | ASSERT_EQ(r, 0); |
2711 | } | |
2712 | struct stat stat; | |
11fdf7f2 | 2713 | r = store->stat(ch, hoid, &stat); |
7c673cae FG |
2714 | ASSERT_EQ(0, r); |
2715 | ASSERT_EQ(0, stat.st_size); | |
2716 | ||
2717 | bufferlist newdata; | |
11fdf7f2 | 2718 | r = store->read(ch, hoid, 0, 1048576, newdata); |
7c673cae FG |
2719 | ASSERT_EQ(0, r); |
2720 | } | |
2721 | ||
b32b8144 | 2722 | TEST_P(StoreTest, ZeroLengthZero) { |
b32b8144 FG |
2723 | int r; |
2724 | coll_t cid; | |
2725 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2726 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2727 | { |
2728 | ObjectStore::Transaction t; | |
2729 | t.create_collection(cid, 0); | |
2730 | t.touch(cid, hoid); | |
11fdf7f2 | 2731 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2732 | ASSERT_EQ(0, r); |
2733 | } | |
2734 | { | |
2735 | ObjectStore::Transaction t; | |
2736 | t.zero(cid, hoid, 1048576, 0); | |
11fdf7f2 | 2737 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2738 | ASSERT_EQ(0, r); |
2739 | } | |
2740 | struct stat stat; | |
11fdf7f2 | 2741 | r = store->stat(ch, hoid, &stat); |
b32b8144 FG |
2742 | ASSERT_EQ(0, r); |
2743 | ASSERT_EQ(0, stat.st_size); | |
2744 | ||
2745 | bufferlist newdata; | |
11fdf7f2 | 2746 | r = store->read(ch, hoid, 0, 1048576, newdata); |
b32b8144 FG |
2747 | ASSERT_EQ(0, r); |
2748 | } | |
2749 | ||
7c673cae | 2750 | TEST_P(StoreTest, SimpleAttrTest) { |
7c673cae FG |
2751 | int r; |
2752 | coll_t cid; | |
2753 | ghobject_t hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP))); | |
2754 | bufferlist val, val2; | |
2755 | val.append("value"); | |
2756 | val.append("value2"); | |
2757 | { | |
11fdf7f2 TL |
2758 | auto ch = store->open_collection(cid); |
2759 | ASSERT_FALSE(ch); | |
7c673cae | 2760 | } |
11fdf7f2 | 2761 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2762 | { |
2763 | ObjectStore::Transaction t; | |
2764 | t.create_collection(cid, 0); | |
11fdf7f2 | 2765 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2766 | ASSERT_EQ(r, 0); |
2767 | } | |
2768 | { | |
2769 | bool empty; | |
11fdf7f2 | 2770 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2771 | ASSERT_EQ(0, r); |
2772 | ASSERT_TRUE(empty); | |
2773 | } | |
2774 | { | |
2775 | bufferptr bp; | |
11fdf7f2 | 2776 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2777 | ASSERT_EQ(-ENOENT, r); |
2778 | } | |
2779 | { | |
2780 | ObjectStore::Transaction t; | |
2781 | t.touch(cid, hoid); | |
2782 | t.setattr(cid, hoid, "foo", val); | |
2783 | t.setattr(cid, hoid, "bar", val2); | |
11fdf7f2 | 2784 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2785 | ASSERT_EQ(r, 0); |
2786 | } | |
2787 | { | |
2788 | bool empty; | |
11fdf7f2 | 2789 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2790 | ASSERT_EQ(0, r); |
2791 | ASSERT_TRUE(!empty); | |
2792 | } | |
2793 | { | |
2794 | bufferptr bp; | |
11fdf7f2 | 2795 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2796 | ASSERT_EQ(-ENODATA, r); |
2797 | ||
11fdf7f2 | 2798 | r = store->getattr(ch, hoid, "foo", bp); |
7c673cae FG |
2799 | ASSERT_EQ(0, r); |
2800 | bufferlist bl; | |
2801 | bl.append(bp); | |
2802 | ASSERT_TRUE(bl_eq(val, bl)); | |
2803 | ||
2804 | map<string,bufferptr> bm; | |
11fdf7f2 | 2805 | r = store->getattrs(ch, hoid, bm); |
7c673cae FG |
2806 | ASSERT_EQ(0, r); |
2807 | ||
2808 | } | |
2809 | { | |
2810 | ObjectStore::Transaction t; | |
2811 | t.remove(cid, hoid); | |
2812 | t.remove_collection(cid); | |
11fdf7f2 | 2813 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2814 | ASSERT_EQ(r, 0); |
2815 | } | |
2816 | } | |
2817 | ||
2818 | TEST_P(StoreTest, SimpleListTest) { | |
7c673cae FG |
2819 | int r; |
2820 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 2821 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2822 | { |
2823 | ObjectStore::Transaction t; | |
2824 | t.create_collection(cid, 0); | |
2825 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2826 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2827 | ASSERT_EQ(r, 0); |
2828 | } | |
2829 | set<ghobject_t> all; | |
2830 | { | |
2831 | ObjectStore::Transaction t; | |
2832 | for (int i=0; i<200; ++i) { | |
2833 | string name("object_"); | |
2834 | name += stringify(i); | |
2835 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2836 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2837 | hoid.hobj.pool = 1; | |
2838 | all.insert(hoid); | |
2839 | t.touch(cid, hoid); | |
2840 | cerr << "Creating object " << hoid << std::endl; | |
2841 | } | |
11fdf7f2 | 2842 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2843 | ASSERT_EQ(r, 0); |
2844 | } | |
2845 | { | |
2846 | set<ghobject_t> saw; | |
2847 | vector<ghobject_t> objects; | |
2848 | ghobject_t next, current; | |
2849 | while (!next.is_max()) { | |
f91f0fd5 TL |
2850 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 50, |
2851 | &objects, &next); | |
7c673cae FG |
2852 | ASSERT_EQ(r, 0); |
2853 | ASSERT_TRUE(sorted(objects)); | |
2854 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2855 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2856 | ++p) { | |
2857 | if (saw.count(*p)) { | |
2858 | cout << "got DUP " << *p << std::endl; | |
2859 | } else { | |
2860 | //cout << "got new " << *p << std::endl; | |
2861 | } | |
2862 | saw.insert(*p); | |
2863 | } | |
2864 | objects.clear(); | |
2865 | current = next; | |
2866 | } | |
2867 | ASSERT_EQ(saw.size(), all.size()); | |
2868 | ASSERT_EQ(saw, all); | |
2869 | } | |
2870 | { | |
2871 | ObjectStore::Transaction t; | |
2872 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2873 | t.remove(cid, *p); | |
2874 | t.remove_collection(cid); | |
2875 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2876 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2877 | ASSERT_EQ(r, 0); |
2878 | } | |
2879 | } | |
2880 | ||
2881 | TEST_P(StoreTest, ListEndTest) { | |
7c673cae FG |
2882 | int r; |
2883 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 2884 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2885 | { |
2886 | ObjectStore::Transaction t; | |
2887 | t.create_collection(cid, 0); | |
2888 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2889 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2890 | ASSERT_EQ(r, 0); |
2891 | } | |
2892 | set<ghobject_t> all; | |
2893 | { | |
2894 | ObjectStore::Transaction t; | |
2895 | for (int i=0; i<200; ++i) { | |
2896 | string name("object_"); | |
2897 | name += stringify(i); | |
2898 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2899 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2900 | hoid.hobj.pool = 1; | |
2901 | all.insert(hoid); | |
2902 | t.touch(cid, hoid); | |
2903 | cerr << "Creating object " << hoid << std::endl; | |
2904 | } | |
11fdf7f2 | 2905 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2906 | ASSERT_EQ(r, 0); |
2907 | } | |
2908 | { | |
2909 | ghobject_t end(hobject_t(sobject_t("object_100", CEPH_NOSNAP)), | |
2910 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2911 | end.hobj.pool = 1; | |
2912 | vector<ghobject_t> objects; | |
2913 | ghobject_t next; | |
f91f0fd5 | 2914 | int r = collection_list(store, ch, ghobject_t(), end, 500, &objects, &next); |
7c673cae FG |
2915 | ASSERT_EQ(r, 0); |
2916 | for (auto &p : objects) { | |
2917 | ASSERT_NE(p, end); | |
2918 | } | |
2919 | } | |
2920 | { | |
2921 | ObjectStore::Transaction t; | |
2922 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2923 | t.remove(cid, *p); | |
2924 | t.remove_collection(cid); | |
2925 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2926 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2927 | ASSERT_EQ(r, 0); |
2928 | } | |
2929 | } | |
2930 | ||
2931 | TEST_P(StoreTest, Sort) { | |
2932 | { | |
2933 | hobject_t a(sobject_t("a", CEPH_NOSNAP)); | |
2934 | hobject_t b = a; | |
2935 | ASSERT_EQ(a, b); | |
2936 | b.oid.name = "b"; | |
2937 | ASSERT_NE(a, b); | |
2938 | ASSERT_TRUE(a < b); | |
2939 | a.pool = 1; | |
2940 | b.pool = 2; | |
2941 | ASSERT_TRUE(a < b); | |
2942 | a.pool = 3; | |
2943 | ASSERT_TRUE(a > b); | |
2944 | } | |
2945 | { | |
2946 | ghobject_t a(hobject_t(sobject_t("a", CEPH_NOSNAP))); | |
2947 | ghobject_t b(hobject_t(sobject_t("b", CEPH_NOSNAP))); | |
2948 | a.hobj.pool = 1; | |
2949 | b.hobj.pool = 1; | |
2950 | ASSERT_TRUE(a < b); | |
2951 | a.hobj.pool = -3; | |
2952 | ASSERT_TRUE(a < b); | |
2953 | a.hobj.pool = 1; | |
2954 | b.hobj.pool = -3; | |
2955 | ASSERT_TRUE(a > b); | |
2956 | } | |
2957 | } | |
2958 | ||
2959 | TEST_P(StoreTest, MultipoolListTest) { | |
7c673cae FG |
2960 | int r; |
2961 | int poolid = 4373; | |
2962 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 2963 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2964 | { |
2965 | ObjectStore::Transaction t; | |
2966 | t.create_collection(cid, 0); | |
2967 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2968 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2969 | ASSERT_EQ(r, 0); |
2970 | } | |
2971 | set<ghobject_t> all, saw; | |
2972 | { | |
2973 | ObjectStore::Transaction t; | |
2974 | for (int i=0; i<200; ++i) { | |
2975 | string name("object_"); | |
2976 | name += stringify(i); | |
2977 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
2978 | if (rand() & 1) | |
2979 | hoid.hobj.pool = -2 - poolid; | |
2980 | else | |
2981 | hoid.hobj.pool = poolid; | |
2982 | all.insert(hoid); | |
2983 | t.touch(cid, hoid); | |
2984 | cerr << "Creating object " << hoid << std::endl; | |
2985 | } | |
11fdf7f2 | 2986 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2987 | ASSERT_EQ(r, 0); |
2988 | } | |
2989 | { | |
2990 | vector<ghobject_t> objects; | |
2991 | ghobject_t next, current; | |
2992 | while (!next.is_max()) { | |
f91f0fd5 TL |
2993 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 50, |
2994 | &objects, &next); | |
7c673cae FG |
2995 | ASSERT_EQ(r, 0); |
2996 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2997 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2998 | ++p) { | |
2999 | saw.insert(*p); | |
3000 | } | |
3001 | objects.clear(); | |
3002 | current = next; | |
3003 | } | |
3004 | ASSERT_EQ(saw, all); | |
3005 | } | |
3006 | { | |
3007 | ObjectStore::Transaction t; | |
3008 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
3009 | t.remove(cid, *p); | |
3010 | t.remove_collection(cid); | |
3011 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3012 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3013 | ASSERT_EQ(r, 0); |
3014 | } | |
3015 | } | |
3016 | ||
3017 | TEST_P(StoreTest, SimpleCloneTest) { | |
7c673cae FG |
3018 | int r; |
3019 | coll_t cid; | |
f91f0fd5 TL |
3020 | |
3021 | SetDeathTestStyle("threadsafe"); | |
3022 | ||
11fdf7f2 | 3023 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3024 | { |
3025 | ObjectStore::Transaction t; | |
3026 | t.create_collection(cid, 0); | |
3027 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3028 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3029 | ASSERT_EQ(r, 0); |
3030 | } | |
3031 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3032 | "key", 123, -1, "")); | |
3033 | bufferlist small, large, xlarge, newdata, attr; | |
3034 | small.append("small"); | |
3035 | large.append("large"); | |
3036 | xlarge.append("xlarge"); | |
3037 | { | |
3038 | ObjectStore::Transaction t; | |
3039 | t.touch(cid, hoid); | |
3040 | t.setattr(cid, hoid, "attr1", small); | |
3041 | t.setattr(cid, hoid, "attr2", large); | |
3042 | t.setattr(cid, hoid, "attr3", xlarge); | |
3043 | t.write(cid, hoid, 0, small.length(), small); | |
3044 | t.write(cid, hoid, 10, small.length(), small); | |
3045 | cerr << "Creating object and set attr " << hoid << std::endl; | |
11fdf7f2 | 3046 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3047 | ASSERT_EQ(r, 0); |
3048 | } | |
3049 | ||
3050 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3051 | "key", 123, -1, "")); | |
3052 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP))); | |
3053 | { | |
3054 | ObjectStore::Transaction t; | |
3055 | t.clone(cid, hoid, hoid2); | |
3056 | t.setattr(cid, hoid2, "attr2", small); | |
3057 | t.rmattr(cid, hoid2, "attr1"); | |
3058 | t.write(cid, hoid, 10, large.length(), large); | |
3059 | t.setattr(cid, hoid, "attr1", large); | |
3060 | t.setattr(cid, hoid, "attr2", small); | |
3061 | cerr << "Clone object and rm attr" << std::endl; | |
11fdf7f2 | 3062 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3063 | ASSERT_EQ(r, 0); |
3064 | ||
11fdf7f2 | 3065 | r = store->read(ch, hoid, 10, 5, newdata); |
7c673cae FG |
3066 | ASSERT_EQ(r, 5); |
3067 | ASSERT_TRUE(bl_eq(large, newdata)); | |
3068 | ||
3069 | newdata.clear(); | |
11fdf7f2 | 3070 | r = store->read(ch, hoid, 0, 5, newdata); |
7c673cae FG |
3071 | ASSERT_EQ(r, 5); |
3072 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3073 | ||
3074 | newdata.clear(); | |
11fdf7f2 | 3075 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3076 | ASSERT_EQ(r, 5); |
3077 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3078 | ||
11fdf7f2 | 3079 | r = store->getattr(ch, hoid2, "attr2", attr); |
7c673cae FG |
3080 | ASSERT_EQ(r, 0); |
3081 | ASSERT_TRUE(bl_eq(small, attr)); | |
3082 | ||
3083 | attr.clear(); | |
11fdf7f2 | 3084 | r = store->getattr(ch, hoid2, "attr3", attr); |
7c673cae FG |
3085 | ASSERT_EQ(r, 0); |
3086 | ASSERT_TRUE(bl_eq(xlarge, attr)); | |
3087 | ||
3088 | attr.clear(); | |
11fdf7f2 | 3089 | r = store->getattr(ch, hoid, "attr1", attr); |
7c673cae FG |
3090 | ASSERT_EQ(r, 0); |
3091 | ASSERT_TRUE(bl_eq(large, attr)); | |
3092 | } | |
3093 | { | |
3094 | ObjectStore::Transaction t; | |
3095 | t.remove(cid, hoid); | |
3096 | t.remove(cid, hoid2); | |
11fdf7f2 | 3097 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3098 | } |
3099 | { | |
3100 | bufferlist final; | |
3101 | bufferptr p(16384); | |
3102 | memset(p.c_str(), 1, p.length()); | |
3103 | bufferlist pl; | |
3104 | pl.append(p); | |
3105 | final.append(p); | |
3106 | ObjectStore::Transaction t; | |
3107 | t.write(cid, hoid, 0, pl.length(), pl); | |
3108 | t.clone(cid, hoid, hoid2); | |
3109 | bufferptr a(4096); | |
3110 | memset(a.c_str(), 2, a.length()); | |
3111 | bufferlist al; | |
3112 | al.append(a); | |
3113 | final.append(a); | |
3114 | t.write(cid, hoid, pl.length(), a.length(), al); | |
11fdf7f2 | 3115 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3116 | ASSERT_EQ(r, 0); |
3117 | bufferlist rl; | |
3118 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3119 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3120 | ASSERT_TRUE(bl_eq(rl, final)); |
3121 | } | |
3122 | { | |
3123 | ObjectStore::Transaction t; | |
3124 | t.remove(cid, hoid); | |
3125 | t.remove(cid, hoid2); | |
11fdf7f2 | 3126 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3127 | } |
3128 | { | |
3129 | bufferlist final; | |
3130 | bufferptr p(16384); | |
3131 | memset(p.c_str(), 111, p.length()); | |
3132 | bufferlist pl; | |
3133 | pl.append(p); | |
3134 | final.append(p); | |
3135 | ObjectStore::Transaction t; | |
3136 | t.write(cid, hoid, 0, pl.length(), pl); | |
3137 | t.clone(cid, hoid, hoid2); | |
3138 | bufferptr z(4096); | |
3139 | z.zero(); | |
3140 | final.append(z); | |
3141 | bufferptr a(4096); | |
3142 | memset(a.c_str(), 112, a.length()); | |
3143 | bufferlist al; | |
3144 | al.append(a); | |
3145 | final.append(a); | |
3146 | t.write(cid, hoid, pl.length() + z.length(), a.length(), al); | |
11fdf7f2 | 3147 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3148 | ASSERT_EQ(r, 0); |
3149 | bufferlist rl; | |
3150 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3151 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3152 | ASSERT_TRUE(bl_eq(rl, final)); |
3153 | } | |
3154 | { | |
3155 | ObjectStore::Transaction t; | |
3156 | t.remove(cid, hoid); | |
3157 | t.remove(cid, hoid2); | |
11fdf7f2 | 3158 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3159 | } |
3160 | { | |
3161 | bufferlist final; | |
3162 | bufferptr p(16000); | |
3163 | memset(p.c_str(), 5, p.length()); | |
3164 | bufferlist pl; | |
3165 | pl.append(p); | |
3166 | final.append(p); | |
3167 | ObjectStore::Transaction t; | |
3168 | t.write(cid, hoid, 0, pl.length(), pl); | |
3169 | t.clone(cid, hoid, hoid2); | |
3170 | bufferptr z(1000); | |
3171 | z.zero(); | |
3172 | final.append(z); | |
3173 | bufferptr a(8000); | |
3174 | memset(a.c_str(), 6, a.length()); | |
3175 | bufferlist al; | |
3176 | al.append(a); | |
3177 | final.append(a); | |
3178 | t.write(cid, hoid, 17000, a.length(), al); | |
11fdf7f2 | 3179 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3180 | bufferlist rl; |
3181 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3182 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3183 | /*cout << "expected:\n"; |
3184 | final.hexdump(cout); | |
3185 | cout << "got:\n"; | |
3186 | rl.hexdump(cout);*/ | |
3187 | ASSERT_TRUE(bl_eq(rl, final)); | |
3188 | } | |
3189 | { | |
3190 | ObjectStore::Transaction t; | |
3191 | t.remove(cid, hoid); | |
3192 | t.remove(cid, hoid2); | |
11fdf7f2 | 3193 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3194 | } |
3195 | { | |
3196 | bufferptr p(1048576); | |
3197 | memset(p.c_str(), 3, p.length()); | |
3198 | bufferlist pl; | |
3199 | pl.append(p); | |
3200 | ObjectStore::Transaction t; | |
3201 | t.write(cid, hoid, 0, pl.length(), pl); | |
3202 | t.clone(cid, hoid, hoid2); | |
3203 | bufferptr a(65536); | |
3204 | memset(a.c_str(), 4, a.length()); | |
3205 | bufferlist al; | |
3206 | al.append(a); | |
3207 | t.write(cid, hoid, a.length(), a.length(), al); | |
11fdf7f2 | 3208 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3209 | bufferlist rl; |
3210 | bufferlist final; | |
3211 | final.substr_of(pl, 0, al.length()); | |
3212 | final.append(al); | |
3213 | bufferlist end; | |
3214 | end.substr_of(pl, al.length()*2, pl.length() - al.length()*2); | |
3215 | final.append(end); | |
3216 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3217 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3218 | /*cout << "expected:\n"; |
3219 | final.hexdump(cout); | |
3220 | cout << "got:\n"; | |
3221 | rl.hexdump(cout);*/ | |
3222 | ASSERT_TRUE(bl_eq(rl, final)); | |
3223 | } | |
3224 | { | |
3225 | ObjectStore::Transaction t; | |
3226 | t.remove(cid, hoid); | |
3227 | t.remove(cid, hoid2); | |
11fdf7f2 | 3228 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3229 | } |
3230 | { | |
3231 | bufferptr p(65536); | |
3232 | memset(p.c_str(), 7, p.length()); | |
3233 | bufferlist pl; | |
3234 | pl.append(p); | |
3235 | ObjectStore::Transaction t; | |
3236 | t.write(cid, hoid, 0, pl.length(), pl); | |
3237 | t.clone(cid, hoid, hoid2); | |
3238 | bufferptr a(4096); | |
3239 | memset(a.c_str(), 8, a.length()); | |
3240 | bufferlist al; | |
3241 | al.append(a); | |
3242 | t.write(cid, hoid, 32768, a.length(), al); | |
11fdf7f2 | 3243 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3244 | bufferlist rl; |
3245 | bufferlist final; | |
3246 | final.substr_of(pl, 0, 32768); | |
3247 | final.append(al); | |
3248 | bufferlist end; | |
3249 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3250 | final.append(end); | |
3251 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3252 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3253 | /*cout << "expected:\n"; |
3254 | final.hexdump(cout); | |
3255 | cout << "got:\n"; | |
3256 | rl.hexdump(cout);*/ | |
3257 | ASSERT_TRUE(bl_eq(rl, final)); | |
3258 | } | |
3259 | { | |
3260 | ObjectStore::Transaction t; | |
3261 | t.remove(cid, hoid); | |
3262 | t.remove(cid, hoid2); | |
11fdf7f2 | 3263 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3264 | } |
3265 | { | |
3266 | bufferptr p(65536); | |
3267 | memset(p.c_str(), 9, p.length()); | |
3268 | bufferlist pl; | |
3269 | pl.append(p); | |
3270 | ObjectStore::Transaction t; | |
3271 | t.write(cid, hoid, 0, pl.length(), pl); | |
3272 | t.clone(cid, hoid, hoid2); | |
3273 | bufferptr a(4096); | |
3274 | memset(a.c_str(), 10, a.length()); | |
3275 | bufferlist al; | |
3276 | al.append(a); | |
3277 | t.write(cid, hoid, 33768, a.length(), al); | |
11fdf7f2 | 3278 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3279 | bufferlist rl; |
3280 | bufferlist final; | |
3281 | final.substr_of(pl, 0, 33768); | |
3282 | final.append(al); | |
3283 | bufferlist end; | |
3284 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3285 | final.append(end); | |
3286 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3287 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3288 | /*cout << "expected:\n"; |
3289 | final.hexdump(cout); | |
3290 | cout << "got:\n"; | |
3291 | rl.hexdump(cout);*/ | |
3292 | ASSERT_TRUE(bl_eq(rl, final)); | |
3293 | } | |
3294 | ||
f91f0fd5 | 3295 | { |
7c673cae | 3296 | //verify if non-empty collection is properly handled after store reload |
11fdf7f2 | 3297 | ch.reset(); |
7c673cae FG |
3298 | r = store->umount(); |
3299 | ASSERT_EQ(r, 0); | |
3300 | r = store->mount(); | |
3301 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3302 | ch = store->open_collection(cid); |
7c673cae FG |
3303 | |
3304 | ObjectStore::Transaction t; | |
3305 | t.remove_collection(cid); | |
3306 | cerr << "Invalid rm coll" << std::endl; | |
3307 | PrCtl unset_dumpable; | |
11fdf7f2 | 3308 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3309 | } |
3310 | { | |
3311 | ObjectStore::Transaction t; | |
3312 | t.touch(cid, hoid3); //new record in db | |
11fdf7f2 | 3313 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3314 | ASSERT_EQ(r, 0); |
3315 | } | |
f91f0fd5 | 3316 | { |
7c673cae FG |
3317 | ObjectStore::Transaction t; |
3318 | //verify if non-empty collection is properly handled when there are some pending removes and live records in db | |
3319 | cerr << "Invalid rm coll again" << std::endl; | |
11fdf7f2 | 3320 | ch.reset(); |
7c673cae FG |
3321 | r = store->umount(); |
3322 | ASSERT_EQ(r, 0); | |
3323 | r = store->mount(); | |
3324 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3325 | ch = store->open_collection(cid); |
7c673cae FG |
3326 | |
3327 | t.remove(cid, hoid); | |
3328 | t.remove(cid, hoid2); | |
3329 | t.remove_collection(cid); | |
3330 | PrCtl unset_dumpable; | |
11fdf7f2 | 3331 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3332 | } |
3333 | { | |
3334 | ObjectStore::Transaction t; | |
3335 | t.remove(cid, hoid); | |
3336 | t.remove(cid, hoid2); | |
3337 | t.remove(cid, hoid3); | |
3338 | t.remove_collection(cid); | |
3339 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3340 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3341 | ASSERT_EQ(r, 0); |
3342 | } | |
3343 | } | |
3344 | ||
3345 | TEST_P(StoreTest, OmapSimple) { | |
7c673cae FG |
3346 | int r; |
3347 | coll_t cid; | |
11fdf7f2 | 3348 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3349 | { |
3350 | ObjectStore::Transaction t; | |
3351 | t.create_collection(cid, 0); | |
3352 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3353 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3354 | ASSERT_EQ(r, 0); |
3355 | } | |
3356 | ghobject_t hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP), | |
3357 | "key", 123, -1, "")); | |
3358 | bufferlist small; | |
3359 | small.append("small"); | |
3360 | map<string,bufferlist> km; | |
3361 | km["foo"] = small; | |
3362 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3363 | bufferlist header; | |
3364 | header.append("this is a header"); | |
3365 | { | |
3366 | ObjectStore::Transaction t; | |
3367 | t.touch(cid, hoid); | |
3368 | t.omap_setkeys(cid, hoid, km); | |
3369 | t.omap_setheader(cid, hoid, header); | |
3370 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3371 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3372 | ASSERT_EQ(r, 0); |
3373 | } | |
3374 | // get header, keys | |
3375 | { | |
3376 | bufferlist h; | |
3377 | map<string,bufferlist> r; | |
11fdf7f2 | 3378 | store->omap_get(ch, hoid, &h, &r); |
7c673cae FG |
3379 | ASSERT_TRUE(bl_eq(header, h)); |
3380 | ASSERT_EQ(r.size(), km.size()); | |
3381 | cout << "r: " << r << std::endl; | |
3382 | } | |
3383 | // test iterator with seek_to_first | |
3384 | { | |
3385 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3386 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3387 | for (iter->seek_to_first(); iter->valid(); iter->next()) { | |
7c673cae FG |
3388 | r[iter->key()] = iter->value(); |
3389 | } | |
3390 | cout << "r: " << r << std::endl; | |
3391 | ASSERT_EQ(r.size(), km.size()); | |
3392 | } | |
3393 | // test iterator with initial lower_bound | |
3394 | { | |
3395 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3396 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3397 | for (iter->lower_bound(string()); iter->valid(); iter->next()) { | |
7c673cae FG |
3398 | r[iter->key()] = iter->value(); |
3399 | } | |
3400 | cout << "r: " << r << std::endl; | |
3401 | ASSERT_EQ(r.size(), km.size()); | |
3402 | } | |
3403 | { | |
3404 | ObjectStore::Transaction t; | |
3405 | t.remove(cid, hoid); | |
3406 | t.remove_collection(cid); | |
3407 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3408 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3409 | ASSERT_EQ(r, 0); |
3410 | } | |
3411 | } | |
3412 | ||
3413 | TEST_P(StoreTest, OmapCloneTest) { | |
7c673cae FG |
3414 | int r; |
3415 | coll_t cid; | |
11fdf7f2 | 3416 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3417 | { |
3418 | ObjectStore::Transaction t; | |
3419 | t.create_collection(cid, 0); | |
3420 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3421 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3422 | ASSERT_EQ(r, 0); |
3423 | } | |
3424 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3425 | "key", 123, -1, "")); | |
3426 | bufferlist small; | |
3427 | small.append("small"); | |
3428 | map<string,bufferlist> km; | |
3429 | km["foo"] = small; | |
3430 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3431 | bufferlist header; | |
3432 | header.append("this is a header"); | |
3433 | { | |
3434 | ObjectStore::Transaction t; | |
3435 | t.touch(cid, hoid); | |
3436 | t.omap_setkeys(cid, hoid, km); | |
3437 | t.omap_setheader(cid, hoid, header); | |
3438 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3439 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3440 | ASSERT_EQ(r, 0); |
3441 | } | |
3442 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3443 | "key", 123, -1, "")); | |
3444 | { | |
3445 | ObjectStore::Transaction t; | |
3446 | t.clone(cid, hoid, hoid2); | |
3447 | cerr << "Clone object" << std::endl; | |
11fdf7f2 | 3448 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3449 | ASSERT_EQ(r, 0); |
3450 | } | |
3451 | { | |
3452 | map<string,bufferlist> r; | |
3453 | bufferlist h; | |
11fdf7f2 | 3454 | store->omap_get(ch, hoid2, &h, &r); |
7c673cae FG |
3455 | ASSERT_TRUE(bl_eq(header, h)); |
3456 | ASSERT_EQ(r.size(), km.size()); | |
3457 | } | |
3458 | { | |
3459 | ObjectStore::Transaction t; | |
3460 | t.remove(cid, hoid); | |
3461 | t.remove(cid, hoid2); | |
3462 | t.remove_collection(cid); | |
3463 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3464 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3465 | ASSERT_EQ(r, 0); |
3466 | } | |
3467 | } | |
3468 | ||
3469 | TEST_P(StoreTest, SimpleCloneRangeTest) { | |
7c673cae FG |
3470 | int r; |
3471 | coll_t cid; | |
11fdf7f2 | 3472 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3473 | { |
3474 | ObjectStore::Transaction t; | |
3475 | t.create_collection(cid, 0); | |
3476 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3477 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3478 | ASSERT_EQ(r, 0); |
3479 | } | |
3480 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3481 | hoid.hobj.pool = -1; | |
3482 | bufferlist small, newdata; | |
3483 | small.append("small"); | |
3484 | { | |
3485 | ObjectStore::Transaction t; | |
3486 | t.write(cid, hoid, 10, 5, small); | |
3487 | cerr << "Creating object and write bl " << hoid << std::endl; | |
11fdf7f2 | 3488 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3489 | ASSERT_EQ(r, 0); |
3490 | } | |
3491 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
3492 | hoid2.hobj.pool = -1; | |
3493 | { | |
3494 | ObjectStore::Transaction t; | |
3495 | t.clone_range(cid, hoid, hoid2, 10, 5, 10); | |
3496 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3497 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 3498 | ASSERT_EQ(r, 0); |
11fdf7f2 | 3499 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3500 | ASSERT_EQ(r, 5); |
3501 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3502 | } | |
3503 | { | |
3504 | ObjectStore::Transaction t; | |
3505 | t.truncate(cid, hoid, 1024*1024); | |
3506 | t.clone_range(cid, hoid, hoid2, 0, 1024*1024, 0); | |
3507 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3508 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3509 | ASSERT_EQ(r, 0); |
3510 | struct stat stat, stat2; | |
11fdf7f2 TL |
3511 | r = store->stat(ch, hoid, &stat); |
3512 | r = store->stat(ch, hoid2, &stat2); | |
7c673cae FG |
3513 | ASSERT_EQ(stat.st_size, stat2.st_size); |
3514 | ASSERT_EQ(1024*1024, stat2.st_size); | |
3515 | } | |
3516 | { | |
3517 | ObjectStore::Transaction t; | |
3518 | t.remove(cid, hoid); | |
3519 | t.remove(cid, hoid2); | |
3520 | t.remove_collection(cid); | |
3521 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3522 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3523 | ASSERT_EQ(r, 0); |
3524 | } | |
3525 | } | |
3526 | ||
3527 | ||
3528 | TEST_P(StoreTest, SimpleObjectLongnameTest) { | |
7c673cae FG |
3529 | int r; |
3530 | coll_t cid; | |
11fdf7f2 | 3531 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3532 | { |
3533 | ObjectStore::Transaction t; | |
3534 | t.create_collection(cid, 0); | |
3535 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3536 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3537 | ASSERT_EQ(r, 0); |
3538 | } | |
3539 | ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP))); | |
3540 | { | |
3541 | ObjectStore::Transaction t; | |
3542 | t.touch(cid, hoid); | |
3543 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3544 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3545 | ASSERT_EQ(r, 0); |
3546 | } | |
3547 | { | |
3548 | ObjectStore::Transaction t; | |
3549 | t.remove(cid, hoid); | |
3550 | t.remove_collection(cid); | |
3551 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3552 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3553 | ASSERT_EQ(r, 0); |
3554 | } | |
3555 | } | |
3556 | ||
3557 | ghobject_t generate_long_name(unsigned i) | |
3558 | { | |
3559 | stringstream name; | |
3560 | name << "object id " << i << " "; | |
3561 | for (unsigned j = 0; j < 500; ++j) name << 'a'; | |
3562 | ghobject_t hoid(hobject_t(sobject_t(name.str(), CEPH_NOSNAP))); | |
3563 | hoid.hobj.set_hash(i % 2); | |
3564 | return hoid; | |
3565 | } | |
3566 | ||
3567 | TEST_P(StoreTest, LongnameSplitTest) { | |
7c673cae FG |
3568 | int r; |
3569 | coll_t cid; | |
11fdf7f2 | 3570 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3571 | { |
3572 | ObjectStore::Transaction t; | |
3573 | t.create_collection(cid, 0); | |
3574 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3575 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3576 | ASSERT_EQ(0, r); |
3577 | } | |
3578 | for (unsigned i = 0; i < 320; ++i) { | |
3579 | ObjectStore::Transaction t; | |
3580 | ghobject_t hoid = generate_long_name(i); | |
3581 | t.touch(cid, hoid); | |
3582 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3583 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3584 | ASSERT_EQ(0, r); |
3585 | } | |
3586 | ||
3587 | ghobject_t test_obj = generate_long_name(319); | |
3588 | ghobject_t test_obj_2 = test_obj; | |
3589 | test_obj_2.generation = 0; | |
3590 | { | |
3591 | ObjectStore::Transaction t; | |
3592 | // should cause a split | |
3593 | t.collection_move_rename( | |
3594 | cid, test_obj, | |
3595 | cid, test_obj_2); | |
11fdf7f2 | 3596 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3597 | ASSERT_EQ(0, r); |
3598 | } | |
3599 | ||
3600 | for (unsigned i = 0; i < 319; ++i) { | |
3601 | ObjectStore::Transaction t; | |
3602 | ghobject_t hoid = generate_long_name(i); | |
3603 | t.remove(cid, hoid); | |
3604 | cerr << "Removing object " << hoid << std::endl; | |
11fdf7f2 | 3605 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3606 | ASSERT_EQ(0, r); |
3607 | } | |
3608 | { | |
3609 | ObjectStore::Transaction t; | |
3610 | t.remove(cid, test_obj_2); | |
3611 | t.remove_collection(cid); | |
3612 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3613 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3614 | ASSERT_EQ(0, r); |
3615 | } | |
3616 | ||
3617 | } | |
3618 | ||
3619 | TEST_P(StoreTest, ManyObjectTest) { | |
7c673cae FG |
3620 | int NUM_OBJS = 2000; |
3621 | int r = 0; | |
3622 | coll_t cid; | |
3623 | string base = ""; | |
3624 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
3625 | set<ghobject_t> created; | |
11fdf7f2 | 3626 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3627 | { |
3628 | ObjectStore::Transaction t; | |
3629 | t.create_collection(cid, 0); | |
11fdf7f2 | 3630 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3631 | ASSERT_EQ(r, 0); |
3632 | } | |
3633 | for (int i = 0; i < NUM_OBJS; ++i) { | |
3634 | if (!(i % 5)) { | |
3635 | cerr << "Object " << i << std::endl; | |
3636 | } | |
3637 | ObjectStore::Transaction t; | |
3638 | char buf[100]; | |
3639 | snprintf(buf, sizeof(buf), "%d", i); | |
3640 | ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP))); | |
3641 | t.touch(cid, hoid); | |
3642 | created.insert(hoid); | |
11fdf7f2 | 3643 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3644 | ASSERT_EQ(r, 0); |
3645 | } | |
3646 | ||
3647 | for (set<ghobject_t>::iterator i = created.begin(); | |
3648 | i != created.end(); | |
3649 | ++i) { | |
3650 | struct stat buf; | |
11fdf7f2 | 3651 | ASSERT_TRUE(!store->stat(ch, *i, &buf)); |
7c673cae FG |
3652 | } |
3653 | ||
3654 | set<ghobject_t> listed, listed2; | |
3655 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
3656 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
3657 | &objects, 0); | |
7c673cae FG |
3658 | ASSERT_EQ(r, 0); |
3659 | ||
3660 | cerr << "objects.size() is " << objects.size() << std::endl; | |
3661 | for (vector<ghobject_t> ::iterator i = objects.begin(); | |
3662 | i != objects.end(); | |
3663 | ++i) { | |
3664 | listed.insert(*i); | |
3665 | ASSERT_TRUE(created.count(*i)); | |
3666 | } | |
3667 | ASSERT_TRUE(listed.size() == created.size()); | |
3668 | ||
3669 | ghobject_t start, next; | |
3670 | objects.clear(); | |
f91f0fd5 TL |
3671 | r = collection_list( |
3672 | store, | |
11fdf7f2 | 3673 | ch, |
7c673cae FG |
3674 | ghobject_t::get_max(), |
3675 | ghobject_t::get_max(), | |
3676 | 50, | |
3677 | &objects, | |
3678 | &next | |
3679 | ); | |
3680 | ASSERT_EQ(r, 0); | |
3681 | ASSERT_TRUE(objects.empty()); | |
3682 | ||
3683 | objects.clear(); | |
3684 | listed.clear(); | |
3685 | ghobject_t start2, next2; | |
3686 | while (1) { | |
f91f0fd5 TL |
3687 | r = collection_list(store, ch, start, ghobject_t::get_max(), 50, &objects, |
3688 | &next); | |
7c673cae FG |
3689 | ASSERT_TRUE(sorted(objects)); |
3690 | ASSERT_EQ(r, 0); | |
3691 | listed.insert(objects.begin(), objects.end()); | |
3692 | if (objects.size() < 50) { | |
3693 | ASSERT_TRUE(next.is_max()); | |
3694 | break; | |
3695 | } | |
3696 | objects.clear(); | |
3697 | ||
3698 | start = next; | |
3699 | } | |
3700 | cerr << "listed.size() is " << listed.size() << std::endl; | |
3701 | ASSERT_TRUE(listed.size() == created.size()); | |
3702 | if (listed2.size()) { | |
3703 | ASSERT_EQ(listed.size(), listed2.size()); | |
3704 | } | |
3705 | for (set<ghobject_t>::iterator i = listed.begin(); | |
3706 | i != listed.end(); | |
3707 | ++i) { | |
3708 | ASSERT_TRUE(created.count(*i)); | |
3709 | } | |
3710 | ||
3711 | for (set<ghobject_t>::iterator i = created.begin(); | |
3712 | i != created.end(); | |
3713 | ++i) { | |
3714 | ObjectStore::Transaction t; | |
3715 | t.remove(cid, *i); | |
11fdf7f2 | 3716 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3717 | ASSERT_EQ(r, 0); |
3718 | } | |
3719 | cerr << "cleaning up" << std::endl; | |
3720 | { | |
3721 | ObjectStore::Transaction t; | |
3722 | t.remove_collection(cid); | |
11fdf7f2 | 3723 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3724 | ASSERT_EQ(r, 0); |
3725 | } | |
3726 | } | |
3727 | ||
3728 | ||
3729 | class ObjectGenerator { | |
3730 | public: | |
3731 | virtual ghobject_t create_object(gen_type *gen) = 0; | |
3732 | virtual ~ObjectGenerator() {} | |
3733 | }; | |
3734 | ||
3735 | class MixedGenerator : public ObjectGenerator { | |
3736 | public: | |
3737 | unsigned seq; | |
3738 | int64_t poolid; | |
3739 | explicit MixedGenerator(int64_t p) : seq(0), poolid(p) {} | |
3740 | ghobject_t create_object(gen_type *gen) override { | |
3741 | char buf[100]; | |
3742 | snprintf(buf, sizeof(buf), "OBJ_%u", seq); | |
3743 | string name(buf); | |
3744 | if (seq % 2) { | |
3745 | for (unsigned i = 0; i < 300; ++i) { | |
3746 | name.push_back('a'); | |
3747 | } | |
3748 | } | |
3749 | ++seq; | |
3750 | return ghobject_t( | |
3751 | hobject_t( | |
3752 | name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), | |
3753 | (((seq / 1024) % 2) * 0xF00 ) + | |
3754 | (seq & 0xFF), | |
3755 | poolid, "")); | |
3756 | } | |
3757 | }; | |
3758 | ||
3759 | class SyntheticWorkloadState { | |
3760 | struct Object { | |
3761 | bufferlist data; | |
3762 | map<string, bufferlist> attrs; | |
3763 | }; | |
3764 | public: | |
3765 | static const unsigned max_in_flight = 16; | |
3766 | static const unsigned max_objects = 3000; | |
3767 | static const unsigned max_attr_size = 5; | |
3768 | static const unsigned max_attr_name_len = 100; | |
3769 | static const unsigned max_attr_value_len = 1024 * 64; | |
3770 | coll_t cid; | |
3771 | unsigned write_alignment; | |
3772 | unsigned max_object_len, max_write_len; | |
3773 | unsigned in_flight; | |
3774 | map<ghobject_t, Object> contents; | |
3775 | set<ghobject_t> available_objects; | |
f6b5b4d7 | 3776 | set<ghobject_t>::iterator next_available_object; |
7c673cae FG |
3777 | set<ghobject_t> in_flight_objects; |
3778 | ObjectGenerator *object_gen; | |
3779 | gen_type *rng; | |
3780 | ObjectStore *store; | |
11fdf7f2 | 3781 | ObjectStore::CollectionHandle ch; |
7c673cae | 3782 | |
9f95a23c TL |
3783 | ceph::mutex lock = ceph::make_mutex("State lock"); |
3784 | ceph::condition_variable cond; | |
7c673cae FG |
3785 | |
3786 | struct EnterExit { | |
3787 | const char *msg; | |
3788 | explicit EnterExit(const char *m) : msg(m) { | |
3789 | //cout << pthread_self() << " enter " << msg << std::endl; | |
3790 | } | |
3791 | ~EnterExit() { | |
3792 | //cout << pthread_self() << " exit " << msg << std::endl; | |
3793 | } | |
3794 | }; | |
3795 | ||
3796 | class C_SyntheticOnReadable : public Context { | |
3797 | public: | |
3798 | SyntheticWorkloadState *state; | |
3799 | ghobject_t hoid; | |
3800 | C_SyntheticOnReadable(SyntheticWorkloadState *state, ghobject_t hoid) | |
3801 | : state(state), hoid(hoid) {} | |
3802 | ||
3803 | void finish(int r) override { | |
9f95a23c | 3804 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3805 | EnterExit ee("onreadable finish"); |
3806 | ASSERT_TRUE(state->in_flight_objects.count(hoid)); | |
3807 | ASSERT_EQ(r, 0); | |
3808 | state->in_flight_objects.erase(hoid); | |
3809 | if (state->contents.count(hoid)) | |
3810 | state->available_objects.insert(hoid); | |
3811 | --(state->in_flight); | |
9f95a23c | 3812 | state->cond.notify_all(); |
7c673cae FG |
3813 | |
3814 | bufferlist r2; | |
11fdf7f2 TL |
3815 | r = state->store->read(state->ch, hoid, 0, state->contents[hoid].data.length(), r2); |
3816 | ceph_assert(bl_eq(state->contents[hoid].data, r2)); | |
9f95a23c | 3817 | state->cond.notify_all(); |
7c673cae FG |
3818 | } |
3819 | }; | |
3820 | ||
3821 | class C_SyntheticOnStash : public Context { | |
3822 | public: | |
3823 | SyntheticWorkloadState *state; | |
3824 | ghobject_t oid, noid; | |
3825 | ||
3826 | C_SyntheticOnStash(SyntheticWorkloadState *state, | |
3827 | ghobject_t oid, ghobject_t noid) | |
3828 | : state(state), oid(oid), noid(noid) {} | |
3829 | ||
3830 | void finish(int r) override { | |
9f95a23c | 3831 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3832 | EnterExit ee("stash finish"); |
3833 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3834 | ASSERT_EQ(r, 0); | |
3835 | state->in_flight_objects.erase(oid); | |
3836 | if (state->contents.count(noid)) | |
3837 | state->available_objects.insert(noid); | |
3838 | --(state->in_flight); | |
3839 | bufferlist r2; | |
3840 | r = state->store->read( | |
11fdf7f2 | 3841 | state->ch, noid, 0, |
7c673cae | 3842 | state->contents[noid].data.length(), r2); |
11fdf7f2 | 3843 | ceph_assert(bl_eq(state->contents[noid].data, r2)); |
9f95a23c | 3844 | state->cond.notify_all(); |
7c673cae FG |
3845 | } |
3846 | }; | |
3847 | ||
3848 | class C_SyntheticOnClone : public Context { | |
3849 | public: | |
3850 | SyntheticWorkloadState *state; | |
3851 | ghobject_t oid, noid; | |
3852 | ||
3853 | C_SyntheticOnClone(SyntheticWorkloadState *state, | |
3854 | ghobject_t oid, ghobject_t noid) | |
3855 | : state(state), oid(oid), noid(noid) {} | |
3856 | ||
3857 | void finish(int r) override { | |
9f95a23c | 3858 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3859 | EnterExit ee("clone finish"); |
3860 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3861 | ASSERT_EQ(r, 0); | |
3862 | state->in_flight_objects.erase(oid); | |
3863 | if (state->contents.count(oid)) | |
3864 | state->available_objects.insert(oid); | |
3865 | if (state->contents.count(noid)) | |
3866 | state->available_objects.insert(noid); | |
3867 | --(state->in_flight); | |
3868 | bufferlist r2; | |
11fdf7f2 TL |
3869 | r = state->store->read(state->ch, noid, 0, state->contents[noid].data.length(), r2); |
3870 | ceph_assert(bl_eq(state->contents[noid].data, r2)); | |
9f95a23c | 3871 | state->cond.notify_all(); |
7c673cae FG |
3872 | } |
3873 | }; | |
3874 | ||
3875 | static void filled_byte_array(bufferlist& bl, size_t size) | |
3876 | { | |
3877 | static const char alphanum[] = "0123456789" | |
3878 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
3879 | "abcdefghijklmnopqrstuvwxyz"; | |
3880 | if (!size) { | |
3881 | return; | |
3882 | } | |
3883 | bufferptr bp(size); | |
3884 | for (unsigned int i = 0; i < size - 1; i++) { | |
3885 | // severely limit entropy so we can compress... | |
3886 | bp[i] = alphanum[rand() % 10]; //(sizeof(alphanum) - 1)]; | |
3887 | } | |
3888 | bp[size - 1] = '\0'; | |
3889 | ||
3890 | bl.append(bp); | |
3891 | } | |
3892 | ||
3893 | SyntheticWorkloadState(ObjectStore *store, | |
3894 | ObjectGenerator *gen, | |
3895 | gen_type *rng, | |
7c673cae FG |
3896 | coll_t cid, |
3897 | unsigned max_size, | |
3898 | unsigned max_write, | |
3899 | unsigned alignment) | |
3900 | : cid(cid), write_alignment(alignment), max_object_len(max_size), | |
f6b5b4d7 TL |
3901 | max_write_len(max_write), in_flight(0), |
3902 | next_available_object(available_objects.end()), | |
3903 | object_gen(gen), rng(rng), store(store) {} | |
7c673cae FG |
3904 | |
3905 | int init() { | |
3906 | ObjectStore::Transaction t; | |
11fdf7f2 | 3907 | ch = store->create_new_collection(cid); |
7c673cae | 3908 | t.create_collection(cid, 0); |
11fdf7f2 | 3909 | return queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3910 | } |
3911 | void shutdown() { | |
f6b5b4d7 | 3912 | ghobject_t next; |
7c673cae FG |
3913 | while (1) { |
3914 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
3915 | int r = collection_list(store, ch, next, ghobject_t::get_max(), 10, |
3916 | &objects, &next); | |
11fdf7f2 | 3917 | ceph_assert(r >= 0); |
f6b5b4d7 TL |
3918 | if (objects.size() == 0) |
3919 | break; | |
7c673cae | 3920 | ObjectStore::Transaction t; |
f6b5b4d7 | 3921 | std::map<std::string, ceph::buffer::list> attrset; |
7c673cae | 3922 | for (vector<ghobject_t>::iterator p = objects.begin(); |
f6b5b4d7 TL |
3923 | p != objects.end(); ++p) { |
3924 | t.remove(cid, *p); | |
7c673cae | 3925 | } |
11fdf7f2 | 3926 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3927 | } |
3928 | ObjectStore::Transaction t; | |
3929 | t.remove_collection(cid); | |
11fdf7f2 | 3930 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3931 | } |
3932 | void statfs(store_statfs_t& stat) { | |
3933 | store->statfs(&stat); | |
3934 | } | |
3935 | ||
9f95a23c TL |
3936 | ghobject_t get_uniform_random_object(std::unique_lock<ceph::mutex>& locker) { |
3937 | cond.wait(locker, [this] { | |
3938 | return in_flight < max_in_flight && !available_objects.empty(); | |
3939 | }); | |
7c673cae FG |
3940 | boost::uniform_int<> choose(0, available_objects.size() - 1); |
3941 | int index = choose(*rng); | |
3942 | set<ghobject_t>::iterator i = available_objects.begin(); | |
3943 | for ( ; index > 0; --index, ++i) ; | |
3944 | ghobject_t ret = *i; | |
3945 | return ret; | |
3946 | } | |
3947 | ||
f6b5b4d7 TL |
3948 | ghobject_t get_next_object(std::unique_lock<ceph::mutex>& locker) { |
3949 | cond.wait(locker, [this] { | |
3950 | return in_flight < max_in_flight && !available_objects.empty(); | |
3951 | }); | |
3952 | ||
3953 | if (next_available_object == available_objects.end()) { | |
3954 | next_available_object = available_objects.begin(); | |
3955 | } | |
3956 | ||
3957 | ghobject_t ret = *next_available_object; | |
3958 | ++next_available_object; | |
3959 | return ret; | |
3960 | } | |
3961 | ||
9f95a23c TL |
3962 | void wait_for_ready(std::unique_lock<ceph::mutex>& locker) { |
3963 | cond.wait(locker, [this] { return in_flight < max_in_flight; }); | |
7c673cae FG |
3964 | } |
3965 | ||
3966 | void wait_for_done() { | |
9f95a23c TL |
3967 | std::unique_lock locker{lock}; |
3968 | cond.wait(locker, [this] { return in_flight == 0; }); | |
7c673cae FG |
3969 | } |
3970 | ||
3971 | bool can_create() { | |
3972 | return (available_objects.size() + in_flight_objects.size()) < max_objects; | |
3973 | } | |
3974 | ||
3975 | bool can_unlink() { | |
3976 | return (available_objects.size() + in_flight_objects.size()) > 0; | |
3977 | } | |
3978 | ||
3979 | unsigned get_random_alloc_hints() { | |
3980 | unsigned f = 0; | |
3981 | { | |
3982 | boost::uniform_int<> u(0, 3); | |
3983 | switch (u(*rng)) { | |
3984 | case 1: | |
3985 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE; | |
3986 | break; | |
3987 | case 2: | |
3988 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE; | |
3989 | break; | |
3990 | } | |
3991 | } | |
3992 | { | |
3993 | boost::uniform_int<> u(0, 3); | |
3994 | switch (u(*rng)) { | |
3995 | case 1: | |
3996 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ; | |
3997 | break; | |
3998 | case 2: | |
3999 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ; | |
4000 | break; | |
4001 | } | |
4002 | } | |
4003 | { | |
4004 | // append_only, immutable | |
4005 | boost::uniform_int<> u(0, 4); | |
4006 | f |= u(*rng) << 4; | |
4007 | } | |
4008 | { | |
4009 | boost::uniform_int<> u(0, 3); | |
4010 | switch (u(*rng)) { | |
4011 | case 1: | |
4012 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED; | |
4013 | break; | |
4014 | case 2: | |
4015 | f |= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED; | |
4016 | break; | |
4017 | } | |
4018 | } | |
4019 | { | |
4020 | boost::uniform_int<> u(0, 3); | |
4021 | switch (u(*rng)) { | |
4022 | case 1: | |
4023 | f |= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE; | |
4024 | break; | |
4025 | case 2: | |
4026 | f |= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE; | |
4027 | break; | |
4028 | } | |
4029 | } | |
4030 | return f; | |
4031 | } | |
4032 | ||
4033 | int touch() { | |
9f95a23c | 4034 | std::unique_lock locker{lock}; |
7c673cae FG |
4035 | EnterExit ee("touch"); |
4036 | if (!can_create()) | |
4037 | return -ENOSPC; | |
9f95a23c | 4038 | wait_for_ready(locker); |
7c673cae FG |
4039 | ghobject_t new_obj = object_gen->create_object(rng); |
4040 | available_objects.erase(new_obj); | |
4041 | ObjectStore::Transaction t; | |
4042 | t.touch(cid, new_obj); | |
4043 | boost::uniform_int<> u(17, 22); | |
4044 | boost::uniform_int<> v(12, 17); | |
4045 | t.set_alloc_hint(cid, new_obj, | |
4046 | 1ull << u(*rng), | |
4047 | 1ull << v(*rng), | |
4048 | get_random_alloc_hints()); | |
4049 | ++in_flight; | |
4050 | in_flight_objects.insert(new_obj); | |
4051 | if (!contents.count(new_obj)) | |
4052 | contents[new_obj] = Object(); | |
11fdf7f2 TL |
4053 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4054 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4055 | return status; |
4056 | } | |
4057 | ||
4058 | int stash() { | |
9f95a23c | 4059 | std::unique_lock locker{lock}; |
7c673cae FG |
4060 | EnterExit ee("stash"); |
4061 | if (!can_unlink()) | |
4062 | return -ENOENT; | |
4063 | if (!can_create()) | |
4064 | return -ENOSPC; | |
9f95a23c | 4065 | wait_for_ready(locker); |
7c673cae FG |
4066 | |
4067 | ghobject_t old_obj; | |
4068 | int max = 20; | |
4069 | do { | |
9f95a23c | 4070 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4071 | } while (--max && !contents[old_obj].data.length()); |
4072 | available_objects.erase(old_obj); | |
4073 | ghobject_t new_obj = old_obj; | |
4074 | new_obj.generation++; | |
4075 | available_objects.erase(new_obj); | |
4076 | ||
4077 | ObjectStore::Transaction t; | |
4078 | t.collection_move_rename(cid, old_obj, cid, new_obj); | |
4079 | ++in_flight; | |
4080 | in_flight_objects.insert(old_obj); | |
4081 | ||
4082 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4083 | contents[new_obj].data = contents[old_obj].data; | |
4084 | contents.erase(old_obj); | |
11fdf7f2 TL |
4085 | t.register_on_applied(new C_SyntheticOnStash(this, old_obj, new_obj)); |
4086 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4087 | return status; |
4088 | } | |
4089 | ||
4090 | int clone() { | |
9f95a23c | 4091 | std::unique_lock locker{lock}; |
7c673cae FG |
4092 | EnterExit ee("clone"); |
4093 | if (!can_unlink()) | |
4094 | return -ENOENT; | |
4095 | if (!can_create()) | |
4096 | return -ENOSPC; | |
9f95a23c | 4097 | wait_for_ready(locker); |
7c673cae FG |
4098 | |
4099 | ghobject_t old_obj; | |
4100 | int max = 20; | |
4101 | do { | |
9f95a23c | 4102 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4103 | } while (--max && !contents[old_obj].data.length()); |
4104 | available_objects.erase(old_obj); | |
4105 | ghobject_t new_obj = object_gen->create_object(rng); | |
4106 | // make the hash match | |
4107 | new_obj.hobj.set_hash(old_obj.hobj.get_hash()); | |
4108 | available_objects.erase(new_obj); | |
4109 | ||
4110 | ObjectStore::Transaction t; | |
4111 | t.clone(cid, old_obj, new_obj); | |
4112 | ++in_flight; | |
4113 | in_flight_objects.insert(old_obj); | |
4114 | ||
4115 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4116 | contents[new_obj].data = contents[old_obj].data; | |
4117 | ||
11fdf7f2 TL |
4118 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4119 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4120 | return status; |
4121 | } | |
4122 | ||
4123 | int clone_range() { | |
9f95a23c | 4124 | std::unique_lock locker{lock}; |
7c673cae FG |
4125 | EnterExit ee("clone_range"); |
4126 | if (!can_unlink()) | |
4127 | return -ENOENT; | |
4128 | if (!can_create()) | |
4129 | return -ENOSPC; | |
9f95a23c | 4130 | wait_for_ready(locker); |
7c673cae FG |
4131 | |
4132 | ghobject_t old_obj; | |
4133 | int max = 20; | |
4134 | do { | |
9f95a23c | 4135 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4136 | } while (--max && !contents[old_obj].data.length()); |
4137 | bufferlist &srcdata = contents[old_obj].data; | |
4138 | if (srcdata.length() == 0) { | |
4139 | return 0; | |
4140 | } | |
4141 | available_objects.erase(old_obj); | |
9f95a23c | 4142 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4143 | available_objects.erase(new_obj); |
4144 | ||
4145 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4146 | boost::uniform_int<> u2(0, max_write_len); | |
4147 | uint64_t srcoff = u1(*rng); | |
4148 | // make src and dst offsets match, since that's what the osd does | |
4149 | uint64_t dstoff = srcoff; //u1(*rng); | |
4150 | uint64_t len = u2(*rng); | |
4151 | if (write_alignment) { | |
11fdf7f2 TL |
4152 | srcoff = round_up_to(srcoff, write_alignment); |
4153 | dstoff = round_up_to(dstoff, write_alignment); | |
4154 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4155 | } |
4156 | ||
4157 | if (srcoff > srcdata.length() - 1) { | |
4158 | srcoff = srcdata.length() - 1; | |
4159 | } | |
4160 | if (srcoff + len > srcdata.length()) { | |
4161 | len = srcdata.length() - srcoff; | |
4162 | } | |
4163 | if (0) | |
4164 | cout << __func__ << " from " << srcoff << "~" << len | |
4165 | << " (size " << srcdata.length() << ") to " | |
4166 | << dstoff << "~" << len << std::endl; | |
4167 | ||
4168 | ObjectStore::Transaction t; | |
4169 | t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff); | |
4170 | ++in_flight; | |
4171 | in_flight_objects.insert(old_obj); | |
4172 | ||
4173 | bufferlist bl; | |
4174 | if (srcoff < srcdata.length()) { | |
4175 | if (srcoff + len > srcdata.length()) { | |
4176 | bl.substr_of(srcdata, srcoff, srcdata.length() - srcoff); | |
4177 | } else { | |
4178 | bl.substr_of(srcdata, srcoff, len); | |
4179 | } | |
4180 | } | |
4181 | ||
4182 | bufferlist& dstdata = contents[new_obj].data; | |
4183 | if (dstdata.length() <= dstoff) { | |
4184 | if (bl.length() > 0) { | |
4185 | dstdata.append_zero(dstoff - dstdata.length()); | |
4186 | dstdata.append(bl); | |
4187 | } | |
4188 | } else { | |
4189 | bufferlist value; | |
11fdf7f2 | 4190 | ceph_assert(dstdata.length() > dstoff); |
9f95a23c | 4191 | dstdata.cbegin().copy(dstoff, value); |
7c673cae FG |
4192 | value.append(bl); |
4193 | if (value.length() < dstdata.length()) | |
9f95a23c TL |
4194 | dstdata.cbegin(value.length()).copy( |
4195 | dstdata.length() - value.length(), value); | |
7c673cae FG |
4196 | value.swap(dstdata); |
4197 | } | |
4198 | ||
11fdf7f2 TL |
4199 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4200 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4201 | return status; |
4202 | } | |
4203 | ||
4204 | ||
4205 | int write() { | |
9f95a23c | 4206 | std::unique_lock locker{lock}; |
7c673cae FG |
4207 | EnterExit ee("write"); |
4208 | if (!can_unlink()) | |
4209 | return -ENOENT; | |
9f95a23c | 4210 | wait_for_ready(locker); |
7c673cae | 4211 | |
9f95a23c | 4212 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4213 | available_objects.erase(new_obj); |
4214 | ObjectStore::Transaction t; | |
4215 | ||
4216 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4217 | boost::uniform_int<> u2(0, max_write_len); | |
4218 | uint64_t offset = u1(*rng); | |
4219 | uint64_t len = u2(*rng); | |
4220 | bufferlist bl; | |
4221 | if (write_alignment) { | |
11fdf7f2 TL |
4222 | offset = round_up_to(offset, write_alignment); |
4223 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4224 | } |
4225 | ||
4226 | filled_byte_array(bl, len); | |
4227 | ||
4228 | bufferlist& data = contents[new_obj].data; | |
4229 | if (data.length() <= offset) { | |
4230 | if (len > 0) { | |
4231 | data.append_zero(offset-data.length()); | |
4232 | data.append(bl); | |
4233 | } | |
4234 | } else { | |
4235 | bufferlist value; | |
11fdf7f2 | 4236 | ceph_assert(data.length() > offset); |
9f95a23c | 4237 | data.cbegin().copy(offset, value); |
7c673cae FG |
4238 | value.append(bl); |
4239 | if (value.length() < data.length()) | |
9f95a23c TL |
4240 | data.cbegin(value.length()).copy( |
4241 | data.length()-value.length(), value); | |
7c673cae FG |
4242 | value.swap(data); |
4243 | } | |
4244 | ||
4245 | t.write(cid, new_obj, offset, len, bl); | |
4246 | ++in_flight; | |
4247 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4248 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4249 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4250 | return status; |
4251 | } | |
4252 | ||
4253 | int truncate() { | |
9f95a23c | 4254 | std::unique_lock locker{lock}; |
7c673cae FG |
4255 | EnterExit ee("truncate"); |
4256 | if (!can_unlink()) | |
4257 | return -ENOENT; | |
9f95a23c | 4258 | wait_for_ready(locker); |
7c673cae | 4259 | |
9f95a23c | 4260 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4261 | available_objects.erase(obj); |
4262 | ObjectStore::Transaction t; | |
4263 | ||
4264 | boost::uniform_int<> choose(0, max_object_len); | |
4265 | size_t len = choose(*rng); | |
4266 | if (write_alignment) { | |
11fdf7f2 | 4267 | len = round_up_to(len, write_alignment); |
7c673cae FG |
4268 | } |
4269 | ||
4270 | t.truncate(cid, obj, len); | |
4271 | ++in_flight; | |
4272 | in_flight_objects.insert(obj); | |
4273 | bufferlist& data = contents[obj].data; | |
4274 | if (data.length() <= len) { | |
4275 | data.append_zero(len - data.length()); | |
4276 | } else { | |
4277 | bufferlist bl; | |
9f95a23c | 4278 | data.cbegin().copy(len, bl); |
7c673cae FG |
4279 | bl.swap(data); |
4280 | } | |
4281 | ||
11fdf7f2 TL |
4282 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4283 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4284 | return status; |
4285 | } | |
4286 | ||
4287 | int zero() { | |
9f95a23c | 4288 | std::unique_lock locker{lock}; |
7c673cae FG |
4289 | EnterExit ee("zero"); |
4290 | if (!can_unlink()) | |
4291 | return -ENOENT; | |
9f95a23c | 4292 | wait_for_ready(locker); |
7c673cae | 4293 | |
9f95a23c | 4294 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4295 | available_objects.erase(new_obj); |
4296 | ObjectStore::Transaction t; | |
4297 | ||
4298 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4299 | boost::uniform_int<> u2(0, max_write_len); | |
4300 | uint64_t offset = u1(*rng); | |
4301 | uint64_t len = u2(*rng); | |
4302 | if (write_alignment) { | |
11fdf7f2 TL |
4303 | offset = round_up_to(offset, write_alignment); |
4304 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4305 | } |
4306 | ||
b32b8144 FG |
4307 | if (len > 0) { |
4308 | auto& data = contents[new_obj].data; | |
4309 | if (data.length() < offset + len) { | |
4310 | data.append_zero(offset+len-data.length()); | |
4311 | } | |
4312 | bufferlist n; | |
4313 | n.substr_of(data, 0, offset); | |
4314 | n.append_zero(len); | |
4315 | if (data.length() > offset + len) | |
9f95a23c | 4316 | data.cbegin(offset + len).copy(data.length() - offset - len, n); |
b32b8144 | 4317 | data.swap(n); |
7c673cae | 4318 | } |
7c673cae FG |
4319 | |
4320 | t.zero(cid, new_obj, offset, len); | |
4321 | ++in_flight; | |
4322 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4323 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4324 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4325 | return status; |
4326 | } | |
4327 | ||
4328 | void read() { | |
4329 | EnterExit ee("read"); | |
4330 | boost::uniform_int<> u1(0, max_object_len/2); | |
4331 | boost::uniform_int<> u2(0, max_object_len); | |
4332 | uint64_t offset = u1(*rng); | |
4333 | uint64_t len = u2(*rng); | |
4334 | if (offset > len) | |
4335 | swap(offset, len); | |
4336 | ||
4337 | ghobject_t obj; | |
4338 | bufferlist expected; | |
4339 | int r; | |
4340 | { | |
9f95a23c | 4341 | std::unique_lock locker{lock}; |
7c673cae FG |
4342 | EnterExit ee("read locked"); |
4343 | if (!can_unlink()) | |
4344 | return ; | |
9f95a23c | 4345 | wait_for_ready(locker); |
7c673cae | 4346 | |
9f95a23c | 4347 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4348 | expected = contents[obj].data; |
4349 | } | |
4350 | bufferlist bl, result; | |
4351 | if (0) cout << " obj " << obj | |
4352 | << " size " << expected.length() | |
4353 | << " offset " << offset | |
4354 | << " len " << len << std::endl; | |
11fdf7f2 | 4355 | r = store->read(ch, obj, offset, len, result); |
7c673cae FG |
4356 | if (offset >= expected.length()) { |
4357 | ASSERT_EQ(r, 0); | |
4358 | } else { | |
4359 | size_t max_len = expected.length() - offset; | |
4360 | if (len > max_len) | |
4361 | len = max_len; | |
11fdf7f2 | 4362 | ceph_assert(len == result.length()); |
7c673cae | 4363 | ASSERT_EQ(len, result.length()); |
9f95a23c | 4364 | expected.cbegin(offset).copy(len, bl); |
7c673cae FG |
4365 | ASSERT_EQ(r, (int)len); |
4366 | ASSERT_TRUE(bl_eq(bl, result)); | |
4367 | } | |
4368 | } | |
4369 | ||
4370 | int setattrs() { | |
9f95a23c | 4371 | std::unique_lock locker{lock}; |
7c673cae FG |
4372 | EnterExit ee("setattrs"); |
4373 | if (!can_unlink()) | |
4374 | return -ENOENT; | |
9f95a23c | 4375 | wait_for_ready(locker); |
7c673cae | 4376 | |
9f95a23c | 4377 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4378 | available_objects.erase(obj); |
4379 | ObjectStore::Transaction t; | |
4380 | ||
4381 | boost::uniform_int<> u0(1, max_attr_size); | |
4382 | boost::uniform_int<> u1(4, max_attr_name_len); | |
4383 | boost::uniform_int<> u2(4, max_attr_value_len); | |
4384 | boost::uniform_int<> u3(0, 100); | |
4385 | uint64_t size = u0(*rng); | |
4386 | uint64_t name_len; | |
4387 | map<string, bufferlist> attrs; | |
4388 | set<string> keys; | |
4389 | for (map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4390 | it != contents[obj].attrs.end(); ++it) | |
4391 | keys.insert(it->first); | |
4392 | ||
4393 | while (size--) { | |
4394 | bufferlist name, value; | |
4395 | uint64_t get_exist = u3(*rng); | |
4396 | uint64_t value_len = u2(*rng); | |
4397 | filled_byte_array(value, value_len); | |
4398 | if (get_exist < 50 && keys.size()) { | |
4399 | set<string>::iterator k = keys.begin(); | |
4400 | attrs[*k] = value; | |
4401 | contents[obj].attrs[*k] = value; | |
4402 | keys.erase(k); | |
4403 | } else { | |
4404 | name_len = u1(*rng); | |
4405 | filled_byte_array(name, name_len); | |
4406 | attrs[name.c_str()] = value; | |
4407 | contents[obj].attrs[name.c_str()] = value; | |
4408 | } | |
4409 | } | |
4410 | t.setattrs(cid, obj, attrs); | |
4411 | ++in_flight; | |
4412 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4413 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4414 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4415 | return status; |
4416 | } | |
4417 | ||
f6b5b4d7 TL |
4418 | int set_fixed_attrs(size_t entries, size_t key_size, size_t val_size) { |
4419 | std::unique_lock locker{ lock }; | |
4420 | EnterExit ee("setattrs"); | |
4421 | if (!can_unlink()) | |
4422 | return -ENOENT; | |
4423 | wait_for_ready(locker); | |
4424 | ||
4425 | ghobject_t obj = get_next_object(locker); | |
4426 | available_objects.erase(obj); | |
4427 | ObjectStore::Transaction t; | |
4428 | ||
4429 | map<string, bufferlist> attrs; | |
4430 | set<string> keys; | |
4431 | ||
4432 | while (entries--) { | |
4433 | bufferlist name, value; | |
4434 | filled_byte_array(value, val_size); | |
4435 | filled_byte_array(name, key_size); | |
4436 | attrs[name.c_str()] = value; | |
4437 | contents[obj].attrs[name.c_str()] = value; | |
4438 | } | |
4439 | t.setattrs(cid, obj, attrs); | |
4440 | ++in_flight; | |
4441 | in_flight_objects.insert(obj); | |
4442 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); | |
4443 | int status = store->queue_transaction(ch, std::move(t)); | |
4444 | return status; | |
4445 | } | |
4446 | ||
7c673cae FG |
4447 | void getattrs() { |
4448 | EnterExit ee("getattrs"); | |
4449 | ghobject_t obj; | |
4450 | map<string, bufferlist> expected; | |
4451 | { | |
9f95a23c | 4452 | std::unique_lock locker{lock}; |
7c673cae FG |
4453 | EnterExit ee("getattrs locked"); |
4454 | if (!can_unlink()) | |
4455 | return ; | |
9f95a23c | 4456 | wait_for_ready(locker); |
7c673cae FG |
4457 | |
4458 | int retry = 10; | |
4459 | do { | |
9f95a23c | 4460 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4461 | if (!--retry) |
4462 | return ; | |
4463 | } while (contents[obj].attrs.empty()); | |
4464 | expected = contents[obj].attrs; | |
4465 | } | |
4466 | map<string, bufferlist> attrs; | |
11fdf7f2 | 4467 | int r = store->getattrs(ch, obj, attrs); |
7c673cae FG |
4468 | ASSERT_TRUE(r == 0); |
4469 | ASSERT_TRUE(attrs.size() == expected.size()); | |
4470 | for (map<string, bufferlist>::iterator it = expected.begin(); | |
4471 | it != expected.end(); ++it) { | |
4472 | ASSERT_TRUE(bl_eq(attrs[it->first], it->second)); | |
4473 | } | |
4474 | } | |
4475 | ||
4476 | void getattr() { | |
4477 | EnterExit ee("getattr"); | |
4478 | ghobject_t obj; | |
4479 | int r; | |
4480 | int retry; | |
4481 | map<string, bufferlist> expected; | |
4482 | { | |
9f95a23c | 4483 | std::unique_lock locker{lock}; |
7c673cae FG |
4484 | EnterExit ee("getattr locked"); |
4485 | if (!can_unlink()) | |
4486 | return ; | |
9f95a23c | 4487 | wait_for_ready(locker); |
7c673cae FG |
4488 | |
4489 | retry = 10; | |
4490 | do { | |
9f95a23c | 4491 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4492 | if (!--retry) |
4493 | return ; | |
4494 | } while (contents[obj].attrs.empty()); | |
4495 | expected = contents[obj].attrs; | |
4496 | } | |
4497 | boost::uniform_int<> u(0, expected.size()-1); | |
4498 | retry = u(*rng); | |
4499 | map<string, bufferlist>::iterator it = expected.begin(); | |
4500 | while (retry) { | |
4501 | retry--; | |
4502 | ++it; | |
4503 | } | |
4504 | ||
4505 | bufferlist bl; | |
11fdf7f2 | 4506 | r = store->getattr(ch, obj, it->first, bl); |
7c673cae FG |
4507 | ASSERT_EQ(r, 0); |
4508 | ASSERT_TRUE(bl_eq(it->second, bl)); | |
4509 | } | |
4510 | ||
4511 | int rmattr() { | |
9f95a23c | 4512 | std::unique_lock locker{lock}; |
7c673cae FG |
4513 | EnterExit ee("rmattr"); |
4514 | if (!can_unlink()) | |
4515 | return -ENOENT; | |
9f95a23c | 4516 | wait_for_ready(locker); |
7c673cae FG |
4517 | |
4518 | ghobject_t obj; | |
4519 | int retry = 10; | |
4520 | do { | |
9f95a23c | 4521 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4522 | if (!--retry) |
4523 | return 0; | |
4524 | } while (contents[obj].attrs.empty()); | |
4525 | ||
4526 | boost::uniform_int<> u(0, contents[obj].attrs.size()-1); | |
4527 | retry = u(*rng); | |
4528 | map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4529 | while (retry) { | |
4530 | retry--; | |
4531 | ++it; | |
4532 | } | |
4533 | ||
4534 | available_objects.erase(obj); | |
4535 | ObjectStore::Transaction t; | |
4536 | t.rmattr(cid, obj, it->first); | |
4537 | ||
4538 | contents[obj].attrs.erase(it->first); | |
4539 | ++in_flight; | |
4540 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4541 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4542 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4543 | return status; |
4544 | } | |
4545 | ||
4546 | void fsck(bool deep) { | |
9f95a23c | 4547 | std::unique_lock locker{lock}; |
7c673cae | 4548 | EnterExit ee("fsck"); |
9f95a23c | 4549 | cond.wait(locker, [this] { return in_flight == 0; }); |
11fdf7f2 | 4550 | ch.reset(); |
7c673cae FG |
4551 | store->umount(); |
4552 | int r = store->fsck(deep); | |
11fdf7f2 | 4553 | ceph_assert(r == 0 || r == -EOPNOTSUPP); |
7c673cae | 4554 | store->mount(); |
11fdf7f2 | 4555 | ch = store->open_collection(cid); |
7c673cae FG |
4556 | } |
4557 | ||
4558 | void scan() { | |
9f95a23c | 4559 | std::unique_lock locker{lock}; |
7c673cae | 4560 | EnterExit ee("scan"); |
9f95a23c | 4561 | cond.wait(locker, [this] { return in_flight == 0; }); |
7c673cae FG |
4562 | vector<ghobject_t> objects; |
4563 | set<ghobject_t> objects_set, objects_set2; | |
4564 | ghobject_t next, current; | |
4565 | while (1) { | |
4566 | //cerr << "scanning..." << std::endl; | |
f91f0fd5 TL |
4567 | int r = collection_list(store, ch, current, ghobject_t::get_max(), 100, |
4568 | &objects, &next); | |
7c673cae FG |
4569 | ASSERT_EQ(r, 0); |
4570 | ASSERT_TRUE(sorted(objects)); | |
4571 | objects_set.insert(objects.begin(), objects.end()); | |
4572 | objects.clear(); | |
4573 | if (next.is_max()) break; | |
4574 | current = next; | |
4575 | } | |
4576 | if (objects_set.size() != available_objects.size()) { | |
4577 | for (set<ghobject_t>::iterator p = objects_set.begin(); | |
4578 | p != objects_set.end(); | |
4579 | ++p) | |
4580 | if (available_objects.count(*p) == 0) { | |
4581 | cerr << "+ " << *p << std::endl; | |
4582 | ceph_abort(); | |
4583 | } | |
4584 | for (set<ghobject_t>::iterator p = available_objects.begin(); | |
4585 | p != available_objects.end(); | |
4586 | ++p) | |
4587 | if (objects_set.count(*p) == 0) | |
4588 | cerr << "- " << *p << std::endl; | |
4589 | //cerr << " objects_set: " << objects_set << std::endl; | |
4590 | //cerr << " available_set: " << available_objects << std::endl; | |
11fdf7f2 | 4591 | ceph_abort_msg("badness"); |
7c673cae FG |
4592 | } |
4593 | ||
4594 | ASSERT_EQ(objects_set.size(), available_objects.size()); | |
4595 | for (set<ghobject_t>::iterator i = objects_set.begin(); | |
4596 | i != objects_set.end(); | |
4597 | ++i) { | |
4598 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4599 | } | |
4600 | ||
f91f0fd5 TL |
4601 | int r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), |
4602 | INT_MAX, &objects, 0); | |
7c673cae FG |
4603 | ASSERT_EQ(r, 0); |
4604 | objects_set2.insert(objects.begin(), objects.end()); | |
4605 | ASSERT_EQ(objects_set2.size(), available_objects.size()); | |
4606 | for (set<ghobject_t>::iterator i = objects_set2.begin(); | |
4607 | i != objects_set2.end(); | |
4608 | ++i) { | |
4609 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4610 | if (available_objects.count(*i) == 0) { | |
4611 | cerr << "+ " << *i << std::endl; | |
4612 | } | |
4613 | } | |
4614 | } | |
4615 | ||
4616 | void stat() { | |
4617 | EnterExit ee("stat"); | |
4618 | ghobject_t hoid; | |
4619 | uint64_t expected; | |
4620 | { | |
9f95a23c | 4621 | std::unique_lock locker{lock}; |
7c673cae FG |
4622 | EnterExit ee("stat lock1"); |
4623 | if (!can_unlink()) | |
4624 | return ; | |
9f95a23c | 4625 | hoid = get_uniform_random_object(locker); |
7c673cae FG |
4626 | in_flight_objects.insert(hoid); |
4627 | available_objects.erase(hoid); | |
4628 | ++in_flight; | |
4629 | expected = contents[hoid].data.length(); | |
4630 | } | |
4631 | struct stat buf; | |
11fdf7f2 | 4632 | int r = store->stat(ch, hoid, &buf); |
7c673cae | 4633 | ASSERT_EQ(0, r); |
11fdf7f2 | 4634 | ceph_assert((uint64_t)buf.st_size == expected); |
7c673cae FG |
4635 | ASSERT_TRUE((uint64_t)buf.st_size == expected); |
4636 | { | |
9f95a23c | 4637 | std::lock_guard locker{lock}; |
7c673cae FG |
4638 | EnterExit ee("stat lock2"); |
4639 | --in_flight; | |
9f95a23c | 4640 | cond.notify_all(); |
7c673cae FG |
4641 | in_flight_objects.erase(hoid); |
4642 | available_objects.insert(hoid); | |
4643 | } | |
4644 | } | |
4645 | ||
4646 | int unlink() { | |
9f95a23c | 4647 | std::unique_lock locker{lock}; |
7c673cae FG |
4648 | EnterExit ee("unlink"); |
4649 | if (!can_unlink()) | |
4650 | return -ENOENT; | |
9f95a23c | 4651 | ghobject_t to_remove = get_uniform_random_object(locker); |
7c673cae FG |
4652 | ObjectStore::Transaction t; |
4653 | t.remove(cid, to_remove); | |
4654 | ++in_flight; | |
4655 | available_objects.erase(to_remove); | |
4656 | in_flight_objects.insert(to_remove); | |
4657 | contents.erase(to_remove); | |
11fdf7f2 TL |
4658 | t.register_on_applied(new C_SyntheticOnReadable(this, to_remove)); |
4659 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4660 | return status; |
4661 | } | |
4662 | ||
4663 | void print_internal_state() { | |
9f95a23c | 4664 | std::lock_guard locker{lock}; |
7c673cae FG |
4665 | cerr << "available_objects: " << available_objects.size() |
4666 | << " in_flight_objects: " << in_flight_objects.size() | |
4667 | << " total objects: " << in_flight_objects.size() + available_objects.size() | |
4668 | << " in_flight " << in_flight << std::endl; | |
4669 | } | |
4670 | }; | |
4671 | ||
4672 | ||
11fdf7f2 | 4673 | void StoreTest::doSyntheticTest( |
7c673cae FG |
4674 | int num_ops, |
4675 | uint64_t max_obj, uint64_t max_wr, uint64_t align) | |
4676 | { | |
7c673cae FG |
4677 | MixedGenerator gen(555); |
4678 | gen_type rng(time(NULL)); | |
4679 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
4680 | ||
11fdf7f2 TL |
4681 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); |
4682 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
4683 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 4684 | |
11fdf7f2 | 4685 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, |
7c673cae FG |
4686 | max_obj, max_wr, align); |
4687 | test_obj.init(); | |
4688 | for (int i = 0; i < num_ops/10; ++i) { | |
4689 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
4690 | test_obj.touch(); | |
4691 | } | |
4692 | for (int i = 0; i < num_ops; ++i) { | |
4693 | if (!(i % 1000)) { | |
4694 | cerr << "Op " << i << std::endl; | |
4695 | test_obj.print_internal_state(); | |
4696 | } | |
4697 | boost::uniform_int<> true_false(0, 999); | |
4698 | int val = true_false(rng); | |
4699 | if (val > 998) { | |
4700 | test_obj.fsck(true); | |
4701 | } else if (val > 997) { | |
4702 | test_obj.fsck(false); | |
4703 | } else if (val > 970) { | |
4704 | test_obj.scan(); | |
4705 | } else if (val > 950) { | |
4706 | test_obj.stat(); | |
4707 | } else if (val > 850) { | |
4708 | test_obj.zero(); | |
4709 | } else if (val > 800) { | |
4710 | test_obj.unlink(); | |
4711 | } else if (val > 550) { | |
4712 | test_obj.write(); | |
4713 | } else if (val > 500) { | |
4714 | test_obj.clone(); | |
4715 | } else if (val > 450) { | |
4716 | test_obj.clone_range(); | |
4717 | } else if (val > 300) { | |
4718 | test_obj.stash(); | |
4719 | } else if (val > 100) { | |
4720 | test_obj.read(); | |
4721 | } else { | |
4722 | test_obj.truncate(); | |
4723 | } | |
4724 | } | |
4725 | test_obj.wait_for_done(); | |
4726 | test_obj.shutdown(); | |
7c673cae FG |
4727 | } |
4728 | ||
4729 | TEST_P(StoreTest, Synthetic) { | |
11fdf7f2 | 4730 | doSyntheticTest(10000, 400*1024, 40*1024, 0); |
7c673cae FG |
4731 | } |
4732 | ||
9f95a23c | 4733 | #if defined(WITH_BLUESTORE) |
11fdf7f2 TL |
4734 | TEST_P(StoreTestSpecificAUSize, BlueFSExtenderTest) { |
4735 | if(string(GetParam()) != "bluestore") | |
4736 | return; | |
4737 | ||
4738 | SetVal(g_conf(), "bluestore_block_db_size", "0"); | |
4739 | SetVal(g_conf(), "bluestore_block_wal_size", "0"); | |
4740 | SetVal(g_conf(), "bluestore_bluefs_min", "12582912"); | |
4741 | SetVal(g_conf(), "bluestore_bluefs_min_free", "4194304"); | |
4742 | SetVal(g_conf(), "bluestore_bluefs_gift_ratio", "0"); | |
4743 | SetVal(g_conf(), "bluestore_bluefs_min_ratio", "0"); | |
4744 | SetVal(g_conf(), "bluestore_bluefs_balance_interval", "100000"); | |
4745 | SetVal(g_conf(), "bluestore_bluefs_db_compatibility", "false"); | |
4746 | ||
4747 | g_conf().apply_changes(nullptr); | |
4748 | ||
4749 | StartDeferred(4096); | |
7c673cae | 4750 | |
11fdf7f2 TL |
4751 | doSyntheticTest(10000, 400*1024, 40*1024, 0); |
4752 | ||
4753 | BlueStore* bstore = NULL; | |
4754 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
4755 | ||
4756 | // verify downgrades are broken and repair that | |
4757 | bstore->umount(); | |
4758 | ASSERT_EQ(bstore->fsck(false), 0); | |
4759 | ||
4760 | SetVal(g_conf(), "bluestore_bluefs_db_compatibility", "true"); | |
4761 | g_conf().apply_changes(nullptr); | |
4762 | ||
4763 | ASSERT_EQ(bstore->fsck(false), 1); | |
4764 | ASSERT_EQ(bstore->repair(false), 0); | |
4765 | ASSERT_EQ(bstore->fsck(false), 0); | |
4766 | bstore->mount(); | |
4767 | } | |
4768 | ||
7c673cae FG |
4769 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixSharding) { |
4770 | if (string(GetParam()) != "bluestore") | |
4771 | return; | |
4772 | ||
4773 | const char *m[][10] = { | |
4774 | { "bluestore_min_alloc_size", "4096", 0 }, // must be the first! | |
4775 | { "num_ops", "50000", 0 }, | |
4776 | { "max_write", "65536", 0 }, | |
4777 | { "max_size", "262144", 0 }, | |
4778 | { "alignment", "4096", 0 }, | |
4779 | { "bluestore_max_blob_size", "65536", 0 }, | |
4780 | { "bluestore_extent_map_shard_min_size", "60", 0 }, | |
4781 | { "bluestore_extent_map_shard_max_size", "300", 0 }, | |
4782 | { "bluestore_extent_map_shard_target_size", "150", 0 }, | |
4783 | { "bluestore_default_buffered_read", "true", 0 }, | |
4784 | { "bluestore_default_buffered_write", "true", 0 }, | |
4785 | { 0 }, | |
4786 | }; | |
11fdf7f2 | 4787 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4788 | } |
4789 | ||
4790 | TEST_P(StoreTestSpecificAUSize, ZipperPatternSharded) { | |
4791 | if(string(GetParam()) != "bluestore") | |
4792 | return; | |
4793 | StartDeferred(4096); | |
4794 | ||
4795 | int r; | |
7c673cae FG |
4796 | coll_t cid; |
4797 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 4798 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
4799 | { |
4800 | ObjectStore::Transaction t; | |
4801 | t.create_collection(cid, 0); | |
4802 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 4803 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4804 | ASSERT_EQ(r, 0); |
4805 | } | |
4806 | bufferlist bl; | |
4807 | int len = 4096; | |
4808 | bufferptr bp(len); | |
4809 | bp.zero(); | |
4810 | bl.append(bp); | |
4811 | for (int i=0; i<1000; ++i) { | |
4812 | ObjectStore::Transaction t; | |
4813 | t.write(cid, a, i*2*len, len, bl, 0); | |
11fdf7f2 | 4814 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4815 | ASSERT_EQ(r, 0); |
4816 | } | |
4817 | for (int i=0; i<1000; ++i) { | |
4818 | ObjectStore::Transaction t; | |
4819 | t.write(cid, a, i*2*len + 1, len, bl, 0); | |
11fdf7f2 | 4820 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4821 | ASSERT_EQ(r, 0); |
4822 | } | |
4823 | { | |
4824 | ObjectStore::Transaction t; | |
4825 | t.remove(cid, a); | |
4826 | t.remove_collection(cid); | |
4827 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 4828 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4829 | ASSERT_EQ(r, 0); |
4830 | } | |
4831 | } | |
4832 | ||
4833 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumAlgorithm) { | |
4834 | if (string(GetParam()) != "bluestore") | |
4835 | return; | |
4836 | ||
4837 | const char *m[][10] = { | |
4838 | { "bluestore_min_alloc_size", "65536", 0 }, // must be the first! | |
4839 | { "max_write", "65536", 0 }, | |
4840 | { "max_size", "1048576", 0 }, | |
4841 | { "alignment", "16", 0 }, | |
4842 | { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", | |
4843 | "xxhash64", "none", 0 }, | |
4844 | { "bluestore_default_buffered_write", "false", 0 }, | |
4845 | { 0 }, | |
4846 | }; | |
11fdf7f2 | 4847 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4848 | } |
4849 | ||
4850 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumVsCompression) { | |
4851 | if (string(GetParam()) != "bluestore") | |
4852 | return; | |
4853 | ||
4854 | const char *m[][10] = { | |
4855 | { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first! | |
4856 | { "max_write", "131072", 0 }, | |
4857 | { "max_size", "262144", 0 }, | |
4858 | { "alignment", "512", 0 }, | |
4859 | { "bluestore_compression_mode", "force", 0}, | |
4860 | { "bluestore_compression_algorithm", "snappy", "zlib", 0 }, | |
4861 | { "bluestore_csum_type", "crc32c", 0 }, | |
4862 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4863 | { "bluestore_default_buffered_write", "true", "false", 0 }, | |
4864 | { "bluestore_sync_submit_transaction", "false", 0 }, | |
4865 | { 0 }, | |
4866 | }; | |
11fdf7f2 | 4867 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4868 | } |
4869 | ||
4870 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompression) { | |
4871 | if (string(GetParam()) != "bluestore") | |
4872 | return; | |
4873 | ||
4874 | const char *m[][10] = { | |
4875 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4876 | { "max_write", "1048576", 0 }, | |
4877 | { "max_size", "4194304", 0 }, | |
4878 | { "alignment", "65536", 0 }, | |
4879 | { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0}, | |
4880 | { "bluestore_default_buffered_write", "false", 0 }, | |
4881 | { "bluestore_sync_submit_transaction", "true", 0 }, | |
4882 | { 0 }, | |
4883 | }; | |
11fdf7f2 | 4884 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4885 | } |
4886 | ||
4887 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompressionAlgorithm) { | |
4888 | if (string(GetParam()) != "bluestore") | |
4889 | return; | |
4890 | ||
4891 | const char *m[][10] = { | |
4892 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4893 | { "max_write", "1048576", 0 }, | |
4894 | { "max_size", "4194304", 0 }, | |
4895 | { "alignment", "65536", 0 }, | |
4896 | { "bluestore_compression_algorithm", "zlib", "snappy", 0 }, | |
4897 | { "bluestore_compression_mode", "force", 0 }, | |
4898 | { "bluestore_default_buffered_write", "false", 0 }, | |
4899 | { 0 }, | |
4900 | }; | |
11fdf7f2 | 4901 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4902 | } |
4903 | ||
4904 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixNoCsum) { | |
4905 | if (string(GetParam()) != "bluestore") | |
4906 | return; | |
4907 | ||
4908 | const char *m[][10] = { | |
4909 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4910 | { "max_write", "65536", 0 }, | |
4911 | { "max_size", "1048576", 0 }, | |
4912 | { "alignment", "512", 0 }, | |
4913 | { "bluestore_max_blob_size", "262144", 0 }, | |
4914 | { "bluestore_compression_mode", "force", "none", 0}, | |
4915 | { "bluestore_csum_type", "none", 0}, | |
4916 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4917 | { "bluestore_default_buffered_write", "true", 0 }, | |
4918 | { "bluestore_sync_submit_transaction", "true", "false", 0 }, | |
4919 | { 0 }, | |
4920 | }; | |
11fdf7f2 | 4921 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4922 | } |
4923 | ||
4924 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixPreferDeferred) { | |
4925 | if (string(GetParam()) != "bluestore") | |
4926 | return; | |
4927 | ||
4928 | const char *m[][10] = { | |
4929 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4930 | { "max_write", "65536", 0 }, | |
4931 | { "max_size", "1048576", 0 }, | |
4932 | { "alignment", "512", 0 }, | |
4933 | { "bluestore_max_blob_size", "262144", 0 }, | |
4934 | { "bluestore_compression_mode", "force", "none", 0}, | |
4935 | { "bluestore_prefer_deferred_size", "32768", "0", 0}, | |
4936 | { 0 }, | |
4937 | }; | |
11fdf7f2 | 4938 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae | 4939 | } |
11fdf7f2 | 4940 | #endif // WITH_BLUESTORE |
7c673cae FG |
4941 | |
4942 | TEST_P(StoreTest, AttrSynthetic) { | |
7c673cae FG |
4943 | MixedGenerator gen(447); |
4944 | gen_type rng(time(NULL)); | |
4945 | coll_t cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD)); | |
4946 | ||
11fdf7f2 | 4947 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, 40*1024, 4*1024, 0); |
7c673cae FG |
4948 | test_obj.init(); |
4949 | for (int i = 0; i < 500; ++i) { | |
4950 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; | |
4951 | test_obj.touch(); | |
4952 | } | |
4953 | for (int i = 0; i < 1000; ++i) { | |
4954 | if (!(i % 100)) { | |
4955 | cerr << "Op " << i << std::endl; | |
4956 | test_obj.print_internal_state(); | |
4957 | } | |
4958 | boost::uniform_int<> true_false(0, 99); | |
4959 | int val = true_false(rng); | |
4960 | if (val > 97) { | |
4961 | test_obj.scan(); | |
4962 | } else if (val > 93) { | |
4963 | test_obj.stat(); | |
4964 | } else if (val > 75) { | |
4965 | test_obj.rmattr(); | |
4966 | } else if (val > 47) { | |
4967 | test_obj.setattrs(); | |
4968 | } else if (val > 45) { | |
4969 | test_obj.clone(); | |
4970 | } else if (val > 37) { | |
4971 | test_obj.stash(); | |
4972 | } else if (val > 30) { | |
4973 | test_obj.getattrs(); | |
4974 | } else { | |
4975 | test_obj.getattr(); | |
4976 | } | |
4977 | } | |
4978 | test_obj.wait_for_done(); | |
4979 | test_obj.shutdown(); | |
4980 | } | |
4981 | ||
4982 | TEST_P(StoreTest, HashCollisionTest) { | |
7c673cae FG |
4983 | int64_t poolid = 11; |
4984 | coll_t cid(spg_t(pg_t(0,poolid),shard_id_t::NO_SHARD)); | |
4985 | int r; | |
11fdf7f2 | 4986 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
4987 | { |
4988 | ObjectStore::Transaction t; | |
4989 | t.create_collection(cid, 0); | |
11fdf7f2 | 4990 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4991 | ASSERT_EQ(r, 0); |
4992 | } | |
4993 | string base = ""; | |
4994 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
4995 | set<ghobject_t> created; | |
4996 | for (int n = 0; n < 10; ++n) { | |
4997 | char nbuf[100]; | |
4998 | sprintf(nbuf, "n%d", n); | |
4999 | for (int i = 0; i < 1000; ++i) { | |
5000 | char buf[100]; | |
5001 | sprintf(buf, "%d", i); | |
5002 | if (!(i % 100)) { | |
5003 | cerr << "Object n" << n << " "<< i << std::endl; | |
5004 | } | |
5005 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, poolid, string(nbuf))); | |
5006 | { | |
5007 | ObjectStore::Transaction t; | |
5008 | t.touch(cid, hoid); | |
11fdf7f2 | 5009 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5010 | ASSERT_EQ(r, 0); |
5011 | } | |
5012 | created.insert(hoid); | |
5013 | } | |
5014 | } | |
5015 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
5016 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5017 | &objects, 0); | |
7c673cae FG |
5018 | ASSERT_EQ(r, 0); |
5019 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5020 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5021 | ASSERT_TRUE(listed.size() == created.size()); | |
5022 | objects.clear(); | |
5023 | listed.clear(); | |
5024 | ghobject_t current, next; | |
5025 | while (1) { | |
f91f0fd5 TL |
5026 | r = collection_list(store, ch, current, ghobject_t::get_max(), 60, &objects, |
5027 | &next); | |
7c673cae FG |
5028 | ASSERT_EQ(r, 0); |
5029 | ASSERT_TRUE(sorted(objects)); | |
5030 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5031 | i != objects.end(); | |
5032 | ++i) { | |
5033 | if (listed.count(*i)) | |
5034 | cerr << *i << " repeated" << std::endl; | |
5035 | listed.insert(*i); | |
5036 | } | |
5037 | if (objects.size() < 50) { | |
5038 | ASSERT_TRUE(next.is_max()); | |
5039 | break; | |
5040 | } | |
5041 | objects.clear(); | |
5042 | current = next; | |
5043 | } | |
5044 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5045 | ASSERT_TRUE(listed.size() == created.size()); | |
5046 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5047 | i != listed.end(); | |
5048 | ++i) { | |
5049 | ASSERT_TRUE(created.count(*i)); | |
5050 | } | |
5051 | ||
5052 | for (set<ghobject_t>::iterator i = created.begin(); | |
5053 | i != created.end(); | |
5054 | ++i) { | |
5055 | ObjectStore::Transaction t; | |
5056 | t.remove(cid, *i); | |
11fdf7f2 | 5057 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5058 | ASSERT_EQ(r, 0); |
5059 | } | |
5060 | ObjectStore::Transaction t; | |
5061 | t.remove_collection(cid); | |
11fdf7f2 | 5062 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5063 | ASSERT_EQ(r, 0); |
5064 | } | |
5065 | ||
f91f0fd5 TL |
5066 | TEST_P(StoreTest, HashCollisionSorting) { |
5067 | bool disable_legacy = (string(GetParam()) == "bluestore"); | |
5068 | ||
5069 | char buf121664318_1[] = {18, -119, -121, -111, 0}; | |
5070 | char buf121664318_2[] = {19, 127, -121, 32, 0}; | |
5071 | char buf121664318_3[] = {19, -118, 15, 19, 0}; | |
5072 | char buf121664318_4[] = {28, 27, -116, -113, 0}; | |
5073 | char buf121664318_5[] = {28, 27, -115, -124, 0}; | |
5074 | ||
5075 | char buf121666222_1[] = {18, -119, -120, -111, 0}; | |
5076 | char buf121666222_2[] = {19, 127, -120, 32, 0}; | |
5077 | char buf121666222_3[] = {19, -118, 15, 30, 0}; | |
5078 | char buf121666222_4[] = {29, 17, -126, -113, 0}; | |
5079 | char buf121666222_5[] = {29, 17, -125, -124, 0}; | |
5080 | ||
5081 | std::map<uint32_t, std::vector<std::string>> object_names = { | |
5082 | {121664318, {{buf121664318_1}, | |
5083 | {buf121664318_2}, | |
5084 | {buf121664318_3}, | |
5085 | {buf121664318_4}, | |
5086 | {buf121664318_5}}}, | |
5087 | {121666222, {{buf121666222_1}, | |
5088 | {buf121666222_2}, | |
5089 | {buf121666222_3}, | |
5090 | {buf121666222_4}, | |
5091 | {buf121666222_5}}}}; | |
5092 | ||
5093 | int64_t poolid = 111; | |
5094 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
5095 | auto ch = store->create_new_collection(cid); | |
5096 | { | |
5097 | ObjectStore::Transaction t; | |
5098 | t.create_collection(cid, 0); | |
5099 | int r = queue_transaction(store, ch, std::move(t)); | |
5100 | ASSERT_EQ(r, 0); | |
5101 | } | |
5102 | ||
5103 | std::set<ghobject_t> created; | |
5104 | for (auto &[hash, names] : object_names) { | |
5105 | for (auto &name : names) { | |
5106 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP), | |
5107 | string(), | |
5108 | hash, | |
5109 | poolid, | |
5110 | string())); | |
5111 | ASSERT_EQ(hash, hoid.hobj.get_hash()); | |
5112 | ObjectStore::Transaction t; | |
5113 | t.touch(cid, hoid); | |
5114 | int r = queue_transaction(store, ch, std::move(t)); | |
5115 | ASSERT_EQ(r, 0); | |
5116 | created.insert(hoid); | |
5117 | } | |
5118 | } | |
5119 | ||
5120 | vector<ghobject_t> objects; | |
5121 | int r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), | |
5122 | INT_MAX, &objects, 0, disable_legacy); | |
5123 | ASSERT_EQ(r, 0); | |
5124 | ASSERT_EQ(created.size(), objects.size()); | |
5125 | auto it = objects.begin(); | |
5126 | for (auto &hoid : created) { | |
5127 | ASSERT_EQ(hoid, *it); | |
5128 | it++; | |
5129 | } | |
5130 | ||
5131 | for (auto i = created.begin(); i != created.end(); i++) { | |
5132 | auto j = i; | |
5133 | for (j++; j != created.end(); j++) { | |
5134 | std::set<ghobject_t> created_sub(i, j); | |
5135 | objects.clear(); | |
5136 | ghobject_t next; | |
5137 | r = collection_list(store, ch, *i, ghobject_t::get_max(), | |
5138 | created_sub.size(), &objects, &next, disable_legacy); | |
5139 | ASSERT_EQ(r, 0); | |
5140 | ASSERT_EQ(created_sub.size(), objects.size()); | |
5141 | it = objects.begin(); | |
5142 | for (auto &hoid : created_sub) { | |
5143 | ASSERT_EQ(hoid, *it); | |
5144 | it++; | |
5145 | } | |
5146 | if (j == created.end()) { | |
5147 | ASSERT_TRUE(next.is_max()); | |
5148 | } else { | |
5149 | ASSERT_EQ(*j, next); | |
5150 | } | |
5151 | } | |
5152 | } | |
5153 | ||
5154 | for (auto i = created.begin(); i != created.end(); i++) { | |
5155 | auto j = i; | |
5156 | for (j++; j != created.end(); j++) { | |
5157 | std::set<ghobject_t> created_sub(i, j); | |
5158 | objects.clear(); | |
5159 | ghobject_t next; | |
5160 | r = collection_list(store, ch, *i, *j, INT_MAX, &objects, &next, | |
5161 | disable_legacy); | |
5162 | ASSERT_EQ(r, 0); | |
5163 | ASSERT_EQ(created_sub.size(), objects.size()); | |
5164 | it = objects.begin(); | |
5165 | for (auto &hoid : created_sub) { | |
5166 | ASSERT_EQ(hoid, *it); | |
5167 | it++; | |
5168 | } | |
5169 | if (j == created.end()) { | |
5170 | ASSERT_TRUE(next.is_max()); | |
5171 | } else { | |
5172 | ASSERT_EQ(*j, next); | |
5173 | } | |
5174 | } | |
5175 | } | |
5176 | } | |
5177 | ||
7c673cae | 5178 | TEST_P(StoreTest, ScrubTest) { |
7c673cae FG |
5179 | int64_t poolid = 111; |
5180 | coll_t cid(spg_t(pg_t(0, poolid),shard_id_t(1))); | |
5181 | int r; | |
11fdf7f2 | 5182 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5183 | { |
5184 | ObjectStore::Transaction t; | |
5185 | t.create_collection(cid, 0); | |
11fdf7f2 | 5186 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5187 | ASSERT_EQ(r, 0); |
5188 | } | |
5189 | string base = "aaaaa"; | |
5190 | set<ghobject_t> created; | |
5191 | for (int i = 0; i < 1000; ++i) { | |
5192 | char buf[100]; | |
5193 | sprintf(buf, "%d", i); | |
5194 | if (!(i % 5)) { | |
5195 | cerr << "Object " << i << std::endl; | |
5196 | } | |
5197 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, i, | |
5198 | poolid, ""), | |
5199 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5200 | { | |
5201 | ObjectStore::Transaction t; | |
5202 | t.touch(cid, hoid); | |
11fdf7f2 | 5203 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5204 | ASSERT_EQ(r, 0); |
5205 | } | |
5206 | created.insert(hoid); | |
5207 | } | |
5208 | ||
5209 | // Add same hobject_t but different generation | |
5210 | { | |
5211 | ghobject_t hoid1(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), | |
5212 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5213 | ghobject_t hoid2(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)1, shard_id_t(1)); | |
5214 | ghobject_t hoid3(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)2, shard_id_t(1)); | |
5215 | ObjectStore::Transaction t; | |
5216 | t.touch(cid, hoid1); | |
5217 | t.touch(cid, hoid2); | |
5218 | t.touch(cid, hoid3); | |
11fdf7f2 | 5219 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5220 | created.insert(hoid1); |
5221 | created.insert(hoid2); | |
5222 | created.insert(hoid3); | |
5223 | ASSERT_EQ(r, 0); | |
5224 | } | |
5225 | ||
5226 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
5227 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5228 | &objects, 0); | |
7c673cae FG |
5229 | ASSERT_EQ(r, 0); |
5230 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5231 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5232 | ASSERT_TRUE(listed.size() == created.size()); | |
5233 | objects.clear(); | |
5234 | listed.clear(); | |
5235 | ghobject_t current, next; | |
5236 | while (1) { | |
f91f0fd5 TL |
5237 | r = collection_list(store, ch, current, ghobject_t::get_max(), 60, &objects, |
5238 | &next); | |
7c673cae FG |
5239 | ASSERT_EQ(r, 0); |
5240 | ASSERT_TRUE(sorted(objects)); | |
5241 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5242 | i != objects.end(); ++i) { | |
5243 | if (listed.count(*i)) | |
5244 | cerr << *i << " repeated" << std::endl; | |
5245 | listed.insert(*i); | |
5246 | } | |
5247 | if (objects.size() < 50) { | |
5248 | ASSERT_TRUE(next.is_max()); | |
5249 | break; | |
5250 | } | |
5251 | objects.clear(); | |
5252 | current = next.get_boundary(); | |
5253 | } | |
5254 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5255 | ASSERT_TRUE(listed.size() == created.size()); | |
5256 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5257 | i != listed.end(); | |
5258 | ++i) { | |
5259 | ASSERT_TRUE(created.count(*i)); | |
5260 | } | |
5261 | ||
5262 | for (set<ghobject_t>::iterator i = created.begin(); | |
5263 | i != created.end(); | |
5264 | ++i) { | |
5265 | ObjectStore::Transaction t; | |
5266 | t.remove(cid, *i); | |
11fdf7f2 | 5267 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5268 | ASSERT_EQ(r, 0); |
5269 | } | |
5270 | ObjectStore::Transaction t; | |
5271 | t.remove_collection(cid); | |
11fdf7f2 | 5272 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5273 | ASSERT_EQ(r, 0); |
5274 | } | |
5275 | ||
5276 | ||
5277 | TEST_P(StoreTest, OMapTest) { | |
7c673cae FG |
5278 | coll_t cid; |
5279 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 5280 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5281 | int r; |
5282 | { | |
5283 | ObjectStore::Transaction t; | |
5284 | t.create_collection(cid, 0); | |
11fdf7f2 | 5285 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5286 | ASSERT_EQ(r, 0); |
5287 | } | |
5288 | ||
5289 | map<string, bufferlist> attrs; | |
5290 | { | |
5291 | ObjectStore::Transaction t; | |
5292 | t.touch(cid, hoid); | |
5293 | t.omap_clear(cid, hoid); | |
5294 | map<string, bufferlist> start_set; | |
5295 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5296 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5297 | ASSERT_EQ(r, 0); |
7c673cae FG |
5298 | } |
5299 | ||
5300 | for (int i = 0; i < 100; i++) { | |
5301 | if (!(i%5)) { | |
5302 | std::cout << "On iteration " << i << std::endl; | |
5303 | } | |
5304 | ObjectStore::Transaction t; | |
5305 | bufferlist bl; | |
5306 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5307 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5308 | ASSERT_EQ(r, 0); |
5309 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5310 | j != attrs.end(); | |
5311 | ++j) { | |
5312 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5313 | if (!correct) { | |
5314 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5315 | if (cur_attrs.count(j->first) > 0) { | |
5316 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5317 | } | |
5318 | } | |
5319 | ASSERT_EQ(correct, true); | |
5320 | } | |
5321 | ASSERT_EQ(attrs.size(), cur_attrs.size()); | |
5322 | ||
5323 | char buf[100]; | |
5324 | snprintf(buf, sizeof(buf), "%d", i); | |
5325 | bl.clear(); | |
5326 | bufferptr bp(buf, strlen(buf) + 1); | |
5327 | bl.append(bp); | |
5328 | map<string, bufferlist> to_add; | |
5329 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5330 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5331 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5332 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5333 | ASSERT_EQ(r, 0); |
7c673cae FG |
5334 | } |
5335 | ||
5336 | int i = 0; | |
5337 | while (attrs.size()) { | |
5338 | if (!(i%5)) { | |
5339 | std::cout << "removal: On iteration " << i << std::endl; | |
5340 | } | |
5341 | ObjectStore::Transaction t; | |
5342 | bufferlist bl; | |
5343 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5344 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5345 | ASSERT_EQ(r, 0); |
5346 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5347 | j != attrs.end(); | |
5348 | ++j) { | |
5349 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5350 | if (!correct) { | |
5351 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5352 | if (cur_attrs.count(j->first) > 0) { | |
5353 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5354 | } | |
5355 | } | |
5356 | ASSERT_EQ(correct, true); | |
5357 | } | |
5358 | ||
5359 | string to_remove = attrs.begin()->first; | |
9f95a23c | 5360 | t.omap_rmkey(cid, hoid, to_remove); |
11fdf7f2 | 5361 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5362 | ASSERT_EQ(r, 0); |
7c673cae FG |
5363 | |
5364 | attrs.erase(to_remove); | |
5365 | ||
5366 | ++i; | |
5367 | } | |
5368 | ||
5369 | { | |
5370 | bufferlist bl1; | |
5371 | bl1.append("omap_header"); | |
5372 | ObjectStore::Transaction t; | |
5373 | t.omap_setheader(cid, hoid, bl1); | |
11fdf7f2 | 5374 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5375 | ASSERT_EQ(r, 0); |
7c673cae FG |
5376 | t = ObjectStore::Transaction(); |
5377 | ||
5378 | bufferlist bl2; | |
5379 | bl2.append("value"); | |
5380 | map<string, bufferlist> to_add; | |
5381 | to_add.insert(pair<string, bufferlist>("key", bl2)); | |
5382 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5383 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5384 | ASSERT_EQ(r, 0); |
7c673cae FG |
5385 | |
5386 | bufferlist bl3; | |
5387 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5388 | r = store->omap_get(ch, hoid, &bl3, &cur_attrs); |
7c673cae FG |
5389 | ASSERT_EQ(r, 0); |
5390 | ASSERT_EQ(cur_attrs.size(), size_t(1)); | |
5391 | ASSERT_TRUE(bl_eq(bl1, bl3)); | |
5392 | ||
5393 | set<string> keys; | |
11fdf7f2 | 5394 | r = store->omap_get_keys(ch, hoid, &keys); |
7c673cae FG |
5395 | ASSERT_EQ(r, 0); |
5396 | ASSERT_EQ(keys.size(), size_t(1)); | |
5397 | } | |
5398 | ||
5399 | // test omap_clear, omap_rmkey_range | |
5400 | { | |
5401 | { | |
5402 | map<string,bufferlist> to_set; | |
5403 | for (int n=0; n<10; ++n) { | |
5404 | to_set[stringify(n)].append("foo"); | |
5405 | } | |
5406 | bufferlist h; | |
5407 | h.append("header"); | |
5408 | ObjectStore::Transaction t; | |
5409 | t.remove(cid, hoid); | |
5410 | t.touch(cid, hoid); | |
5411 | t.omap_setheader(cid, hoid, h); | |
5412 | t.omap_setkeys(cid, hoid, to_set); | |
11fdf7f2 | 5413 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5414 | ASSERT_EQ(r, 0); |
7c673cae FG |
5415 | } |
5416 | { | |
5417 | ObjectStore::Transaction t; | |
5418 | t.omap_rmkeyrange(cid, hoid, "3", "7"); | |
11fdf7f2 | 5419 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5420 | ASSERT_EQ(r, 0); |
7c673cae FG |
5421 | } |
5422 | { | |
5423 | bufferlist hdr; | |
5424 | map<string,bufferlist> m; | |
11fdf7f2 | 5425 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5426 | ASSERT_EQ(6u, hdr.length()); |
5427 | ASSERT_TRUE(m.count("2")); | |
5428 | ASSERT_TRUE(!m.count("3")); | |
5429 | ASSERT_TRUE(!m.count("6")); | |
5430 | ASSERT_TRUE(m.count("7")); | |
5431 | ASSERT_TRUE(m.count("8")); | |
5432 | //cout << m << std::endl; | |
5433 | ASSERT_EQ(6u, m.size()); | |
5434 | } | |
5435 | { | |
5436 | ObjectStore::Transaction t; | |
5437 | t.omap_clear(cid, hoid); | |
11fdf7f2 | 5438 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5439 | ASSERT_EQ(r, 0); |
7c673cae FG |
5440 | } |
5441 | { | |
5442 | bufferlist hdr; | |
5443 | map<string,bufferlist> m; | |
11fdf7f2 | 5444 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5445 | ASSERT_EQ(0u, hdr.length()); |
5446 | ASSERT_EQ(0u, m.size()); | |
5447 | } | |
5448 | } | |
5449 | ||
5450 | ObjectStore::Transaction t; | |
5451 | t.remove(cid, hoid); | |
5452 | t.remove_collection(cid); | |
11fdf7f2 | 5453 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5454 | ASSERT_EQ(r, 0); |
5455 | } | |
5456 | ||
5457 | TEST_P(StoreTest, OMapIterator) { | |
7c673cae FG |
5458 | coll_t cid; |
5459 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5460 | int count = 0; | |
11fdf7f2 | 5461 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5462 | int r; |
5463 | { | |
5464 | ObjectStore::Transaction t; | |
5465 | t.create_collection(cid, 0); | |
11fdf7f2 | 5466 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5467 | ASSERT_EQ(r, 0); |
5468 | } | |
5469 | ||
5470 | map<string, bufferlist> attrs; | |
5471 | { | |
5472 | ObjectStore::Transaction t; | |
5473 | t.touch(cid, hoid); | |
5474 | t.omap_clear(cid, hoid); | |
5475 | map<string, bufferlist> start_set; | |
5476 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5477 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5478 | ASSERT_EQ(r, 0); |
7c673cae FG |
5479 | } |
5480 | ObjectMap::ObjectMapIterator iter; | |
5481 | bool correct; | |
5482 | //basic iteration | |
5483 | for (int i = 0; i < 100; i++) { | |
5484 | if (!(i%5)) { | |
5485 | std::cout << "On iteration " << i << std::endl; | |
5486 | } | |
5487 | bufferlist bl; | |
5488 | ||
5489 | // FileStore may deadlock two active iterators over the same data | |
5490 | iter = ObjectMap::ObjectMapIterator(); | |
5491 | ||
11fdf7f2 | 5492 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5493 | for (iter->seek_to_first(), count=0; iter->valid(); iter->next(), count++) { |
5494 | string key = iter->key(); | |
5495 | bufferlist value = iter->value(); | |
5496 | correct = attrs.count(key) && (string(value.c_str()) == string(attrs[key].c_str())); | |
5497 | if (!correct) { | |
5498 | if (attrs.count(key) > 0) { | |
5499 | std::cout << "key " << key << "in omap , " << value.c_str() << " : " << attrs[key].c_str() << std::endl; | |
5500 | } | |
5501 | else | |
5502 | std::cout << "key " << key << "should not exists in omap" << std::endl; | |
5503 | } | |
5504 | ASSERT_EQ(correct, true); | |
5505 | } | |
5506 | ASSERT_EQ((int)attrs.size(), count); | |
5507 | ||
11fdf7f2 | 5508 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5509 | iter = ObjectMap::ObjectMapIterator(); |
5510 | ||
5511 | char buf[100]; | |
5512 | snprintf(buf, sizeof(buf), "%d", i); | |
5513 | bl.clear(); | |
5514 | bufferptr bp(buf, strlen(buf) + 1); | |
5515 | bl.append(bp); | |
5516 | map<string, bufferlist> to_add; | |
5517 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5518 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5519 | ObjectStore::Transaction t; | |
5520 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5521 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5522 | ASSERT_EQ(r, 0); |
7c673cae FG |
5523 | } |
5524 | ||
11fdf7f2 | 5525 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5526 | //lower bound |
5527 | string bound_key = "key-5"; | |
5528 | iter->lower_bound(bound_key); | |
5529 | correct = bound_key <= iter->key(); | |
5530 | if (!correct) { | |
5531 | std::cout << "lower bound, bound key is " << bound_key << " < iter key is " << iter->key() << std::endl; | |
5532 | } | |
5533 | ASSERT_EQ(correct, true); | |
5534 | //upper bound | |
5535 | iter->upper_bound(bound_key); | |
5536 | correct = iter->key() > bound_key; | |
5537 | if (!correct) { | |
5538 | std::cout << "upper bound, bound key is " << bound_key << " >= iter key is " << iter->key() << std::endl; | |
5539 | } | |
5540 | ASSERT_EQ(correct, true); | |
5541 | ||
11fdf7f2 | 5542 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5543 | iter = ObjectMap::ObjectMapIterator(); |
5544 | { | |
5545 | ObjectStore::Transaction t; | |
5546 | t.remove(cid, hoid); | |
5547 | t.remove_collection(cid); | |
11fdf7f2 | 5548 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5549 | ASSERT_EQ(r, 0); |
5550 | } | |
5551 | } | |
5552 | ||
5553 | TEST_P(StoreTest, XattrTest) { | |
7c673cae FG |
5554 | coll_t cid; |
5555 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5556 | bufferlist big; | |
5557 | for (unsigned i = 0; i < 10000; ++i) { | |
5558 | big.append('\0'); | |
5559 | } | |
5560 | bufferlist small; | |
5561 | for (unsigned i = 0; i < 10; ++i) { | |
5562 | small.append('\0'); | |
5563 | } | |
5564 | int r; | |
11fdf7f2 | 5565 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5566 | { |
5567 | ObjectStore::Transaction t; | |
5568 | t.create_collection(cid, 0); | |
5569 | t.touch(cid, hoid); | |
11fdf7f2 | 5570 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5571 | ASSERT_EQ(r, 0); |
5572 | } | |
5573 | ||
5574 | map<string, bufferlist> attrs; | |
5575 | { | |
5576 | ObjectStore::Transaction t; | |
5577 | t.setattr(cid, hoid, "attr1", small); | |
5578 | attrs["attr1"] = small; | |
5579 | t.setattr(cid, hoid, "attr2", big); | |
5580 | attrs["attr2"] = big; | |
5581 | t.setattr(cid, hoid, "attr3", small); | |
5582 | attrs["attr3"] = small; | |
5583 | t.setattr(cid, hoid, "attr1", small); | |
5584 | attrs["attr1"] = small; | |
5585 | t.setattr(cid, hoid, "attr4", big); | |
5586 | attrs["attr4"] = big; | |
5587 | t.setattr(cid, hoid, "attr3", big); | |
5588 | attrs["attr3"] = big; | |
11fdf7f2 | 5589 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5590 | ASSERT_EQ(r, 0); |
5591 | } | |
5592 | ||
5593 | map<string, bufferptr> aset; | |
11fdf7f2 | 5594 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5595 | ASSERT_EQ(aset.size(), attrs.size()); |
5596 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5597 | i != aset.end(); | |
5598 | ++i) { | |
5599 | bufferlist bl; | |
5600 | bl.push_back(i->second); | |
5601 | ASSERT_TRUE(attrs[i->first] == bl); | |
5602 | } | |
5603 | ||
5604 | { | |
5605 | ObjectStore::Transaction t; | |
5606 | t.rmattr(cid, hoid, "attr2"); | |
5607 | attrs.erase("attr2"); | |
11fdf7f2 | 5608 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5609 | ASSERT_EQ(r, 0); |
5610 | } | |
5611 | ||
5612 | aset.clear(); | |
11fdf7f2 | 5613 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5614 | ASSERT_EQ(aset.size(), attrs.size()); |
5615 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5616 | i != aset.end(); | |
5617 | ++i) { | |
5618 | bufferlist bl; | |
5619 | bl.push_back(i->second); | |
5620 | ASSERT_TRUE(attrs[i->first] == bl); | |
5621 | } | |
5622 | ||
5623 | bufferptr bp; | |
11fdf7f2 | 5624 | r = store->getattr(ch, hoid, "attr2", bp); |
7c673cae FG |
5625 | ASSERT_EQ(r, -ENODATA); |
5626 | ||
11fdf7f2 | 5627 | r = store->getattr(ch, hoid, "attr3", bp); |
7c673cae FG |
5628 | ASSERT_EQ(r, 0); |
5629 | bufferlist bl2; | |
5630 | bl2.push_back(bp); | |
5631 | ASSERT_TRUE(bl2 == attrs["attr3"]); | |
5632 | ||
5633 | ObjectStore::Transaction t; | |
5634 | t.remove(cid, hoid); | |
5635 | t.remove_collection(cid); | |
11fdf7f2 | 5636 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5637 | ASSERT_EQ(r, 0); |
5638 | } | |
5639 | ||
5640 | void colsplittest( | |
5641 | ObjectStore *store, | |
5642 | unsigned num_objects, | |
5643 | unsigned common_suffix_size, | |
5644 | bool clones | |
5645 | ) { | |
7c673cae FG |
5646 | coll_t cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD)); |
5647 | coll_t tid(spg_t(pg_t(1<<common_suffix_size,52),shard_id_t::NO_SHARD)); | |
11fdf7f2 TL |
5648 | auto ch = store->create_new_collection(cid); |
5649 | auto tch = store->create_new_collection(tid); | |
7c673cae FG |
5650 | int r = 0; |
5651 | { | |
5652 | ObjectStore::Transaction t; | |
5653 | t.create_collection(cid, common_suffix_size); | |
11fdf7f2 | 5654 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5655 | ASSERT_EQ(r, 0); |
5656 | } | |
5657 | bufferlist small; | |
5658 | small.append("small"); | |
5659 | { | |
5660 | ObjectStore::Transaction t; | |
5661 | for (uint32_t i = 0; i < (2 - (int)clones)*num_objects; ++i) { | |
5662 | stringstream objname; | |
5663 | objname << "obj" << i; | |
5664 | ghobject_t a(hobject_t( | |
5665 | objname.str(), | |
5666 | "", | |
5667 | CEPH_NOSNAP, | |
5668 | i<<common_suffix_size, | |
5669 | 52, "")); | |
5670 | t.write(cid, a, 0, small.length(), small, | |
5671 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5672 | if (clones) { | |
5673 | objname << "-clone"; | |
5674 | ghobject_t b(hobject_t( | |
5675 | objname.str(), | |
5676 | "", | |
5677 | CEPH_NOSNAP, | |
5678 | i<<common_suffix_size, | |
5679 | 52, "")); | |
5680 | t.clone(cid, a, b); | |
5681 | } | |
5682 | if (i % 100) { | |
11fdf7f2 | 5683 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5684 | ASSERT_EQ(r, 0); |
5685 | t = ObjectStore::Transaction(); | |
5686 | } | |
5687 | } | |
11fdf7f2 | 5688 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5689 | ASSERT_EQ(r, 0); |
5690 | } | |
5691 | { | |
5692 | ObjectStore::Transaction t; | |
5693 | t.create_collection(tid, common_suffix_size + 1); | |
5694 | t.split_collection(cid, common_suffix_size+1, 1<<common_suffix_size, tid); | |
11fdf7f2 | 5695 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5696 | ASSERT_EQ(r, 0); |
5697 | } | |
11fdf7f2 | 5698 | ch->flush(); |
7c673cae | 5699 | |
11fdf7f2 | 5700 | // check |
7c673cae | 5701 | vector<ghobject_t> objects; |
f91f0fd5 TL |
5702 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5703 | &objects, 0); | |
7c673cae FG |
5704 | ASSERT_EQ(r, 0); |
5705 | ASSERT_EQ(objects.size(), num_objects); | |
7c673cae FG |
5706 | for (vector<ghobject_t>::iterator i = objects.begin(); |
5707 | i != objects.end(); | |
5708 | ++i) { | |
5709 | ASSERT_EQ(!!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
5710 | } |
5711 | ||
5712 | objects.clear(); | |
f91f0fd5 TL |
5713 | r = collection_list(store, tch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5714 | &objects, 0); | |
7c673cae FG |
5715 | ASSERT_EQ(r, 0); |
5716 | ASSERT_EQ(objects.size(), num_objects); | |
5717 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5718 | i != objects.end(); | |
5719 | ++i) { | |
5720 | ASSERT_EQ(!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
5721 | } |
5722 | ||
11fdf7f2 TL |
5723 | // merge them again! |
5724 | { | |
5725 | ObjectStore::Transaction t; | |
5726 | t.merge_collection(tid, cid, common_suffix_size); | |
5727 | r = queue_transaction(store, ch, std::move(t)); | |
5728 | ASSERT_EQ(r, 0); | |
5729 | } | |
5730 | ||
5731 | // check and clean up | |
5732 | ObjectStore::Transaction t; | |
5733 | { | |
5734 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
5735 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5736 | &objects, 0); | |
11fdf7f2 TL |
5737 | ASSERT_EQ(r, 0); |
5738 | ASSERT_EQ(objects.size(), num_objects * 2); // both halves | |
5739 | unsigned size = 0; | |
5740 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5741 | i != objects.end(); | |
5742 | ++i) { | |
5743 | t.remove(cid, *i); | |
5744 | if (++size > 100) { | |
5745 | size = 0; | |
5746 | r = queue_transaction(store, ch, std::move(t)); | |
5747 | ASSERT_EQ(r, 0); | |
5748 | t = ObjectStore::Transaction(); | |
5749 | } | |
5750 | } | |
5751 | } | |
7c673cae | 5752 | t.remove_collection(cid); |
11fdf7f2 | 5753 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 5754 | ASSERT_EQ(r, 0); |
11fdf7f2 TL |
5755 | |
5756 | ch->flush(); | |
5757 | ASSERT_TRUE(!store->collection_exists(tid)); | |
7c673cae FG |
5758 | } |
5759 | ||
11fdf7f2 TL |
5760 | TEST_P(StoreTest, ColSplitTest0) { |
5761 | colsplittest(store.get(), 10, 5, false); | |
5762 | } | |
7c673cae FG |
5763 | TEST_P(StoreTest, ColSplitTest1) { |
5764 | colsplittest(store.get(), 10000, 11, false); | |
5765 | } | |
5766 | TEST_P(StoreTest, ColSplitTest1Clones) { | |
5767 | colsplittest(store.get(), 10000, 11, true); | |
5768 | } | |
5769 | TEST_P(StoreTest, ColSplitTest2) { | |
5770 | colsplittest(store.get(), 100, 7, false); | |
5771 | } | |
5772 | TEST_P(StoreTest, ColSplitTest2Clones) { | |
5773 | colsplittest(store.get(), 100, 7, true); | |
5774 | } | |
5775 | ||
5776 | #if 0 | |
5777 | TEST_P(StoreTest, ColSplitTest3) { | |
5778 | colsplittest(store.get(), 100000, 25); | |
5779 | } | |
5780 | #endif | |
5781 | ||
11fdf7f2 TL |
5782 | void test_merge_skewed(ObjectStore *store, |
5783 | unsigned base, unsigned bits, | |
5784 | unsigned anum, unsigned bnum) | |
5785 | { | |
5786 | cout << __func__ << " 0x" << std::hex << base << std::dec | |
5787 | << " bits " << bits | |
5788 | << " anum " << anum << " bnum " << bnum << std::endl; | |
5789 | /* | |
5790 | make merge source pgs have radically different # of objects in them, | |
5791 | which should trigger different splitting in filestore, and verify that | |
5792 | post-merge all objects are accessible. | |
5793 | */ | |
5794 | int r; | |
5795 | coll_t a(spg_t(pg_t(base, 0), shard_id_t::NO_SHARD)); | |
5796 | coll_t b(spg_t(pg_t(base | (1<<bits), 0), shard_id_t::NO_SHARD)); | |
5797 | ||
5798 | auto cha = store->create_new_collection(a); | |
5799 | auto chb = store->create_new_collection(b); | |
5800 | { | |
5801 | ObjectStore::Transaction t; | |
5802 | t.create_collection(a, bits + 1); | |
5803 | r = queue_transaction(store, cha, std::move(t)); | |
5804 | ASSERT_EQ(r, 0); | |
5805 | } | |
5806 | { | |
5807 | ObjectStore::Transaction t; | |
5808 | t.create_collection(b, bits + 1); | |
5809 | r = queue_transaction(store, chb, std::move(t)); | |
5810 | ASSERT_EQ(r, 0); | |
5811 | } | |
5812 | ||
5813 | bufferlist small; | |
5814 | small.append("small"); | |
5815 | string suffix = "ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooaaaaaaaaaa"; | |
5816 | set<ghobject_t> aobjects, bobjects; | |
5817 | { | |
5818 | // fill a | |
5819 | ObjectStore::Transaction t; | |
5820 | for (unsigned i = 0; i < 1000; ++i) { | |
5821 | string objname = "a" + stringify(i) + suffix; | |
5822 | ghobject_t o(hobject_t( | |
5823 | objname, | |
5824 | "", | |
5825 | CEPH_NOSNAP, | |
5826 | i<<(bits+1) | base, | |
5827 | 52, "")); | |
5828 | aobjects.insert(o); | |
5829 | t.write(a, o, 0, small.length(), small, 0); | |
5830 | if (i % 100) { | |
5831 | r = queue_transaction(store, cha, std::move(t)); | |
5832 | ASSERT_EQ(r, 0); | |
5833 | t = ObjectStore::Transaction(); | |
5834 | } | |
5835 | } | |
5836 | r = queue_transaction(store, cha, std::move(t)); | |
5837 | ASSERT_EQ(r, 0); | |
5838 | } | |
5839 | { | |
5840 | // fill b | |
5841 | ObjectStore::Transaction t; | |
5842 | for (unsigned i = 0; i < 10; ++i) { | |
5843 | string objname = "b" + stringify(i) + suffix; | |
5844 | ghobject_t o(hobject_t( | |
5845 | objname, | |
5846 | "", | |
5847 | CEPH_NOSNAP, | |
5848 | (i<<(base+1)) | base | (1<<bits), | |
5849 | 52, "")); | |
5850 | bobjects.insert(o); | |
5851 | t.write(b, o, 0, small.length(), small, 0); | |
5852 | if (i % 100) { | |
5853 | r = queue_transaction(store, chb, std::move(t)); | |
5854 | ASSERT_EQ(r, 0); | |
5855 | t = ObjectStore::Transaction(); | |
5856 | } | |
5857 | } | |
5858 | r = queue_transaction(store, chb, std::move(t)); | |
5859 | ASSERT_EQ(r, 0); | |
5860 | } | |
5861 | ||
5862 | // merge b->a | |
5863 | { | |
5864 | ObjectStore::Transaction t; | |
5865 | t.merge_collection(b, a, bits); | |
5866 | r = queue_transaction(store, cha, std::move(t)); | |
5867 | ASSERT_EQ(r, 0); | |
5868 | } | |
5869 | ||
5870 | // verify | |
5871 | { | |
5872 | vector<ghobject_t> got; | |
f91f0fd5 TL |
5873 | collection_list(store, cha, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
5874 | &got, 0); | |
11fdf7f2 TL |
5875 | set<ghobject_t> gotset; |
5876 | for (auto& o : got) { | |
5877 | ASSERT_TRUE(aobjects.count(o) || bobjects.count(o)); | |
5878 | gotset.insert(o); | |
5879 | } | |
5880 | // check both listing and stat-ability (different code paths!) | |
5881 | struct stat st; | |
5882 | for (auto& o : aobjects) { | |
5883 | ASSERT_TRUE(gotset.count(o)); | |
5884 | int r = store->stat(cha, o, &st, false); | |
5885 | ASSERT_EQ(r, 0); | |
5886 | } | |
5887 | for (auto& o : bobjects) { | |
5888 | ASSERT_TRUE(gotset.count(o)); | |
5889 | int r = store->stat(cha, o, &st, false); | |
5890 | ASSERT_EQ(r, 0); | |
5891 | } | |
5892 | } | |
5893 | ||
5894 | // clean up | |
5895 | { | |
5896 | ObjectStore::Transaction t; | |
5897 | for (auto &o : aobjects) { | |
5898 | t.remove(a, o); | |
5899 | } | |
5900 | r = queue_transaction(store, cha, std::move(t)); | |
5901 | ASSERT_EQ(r, 0); | |
5902 | } | |
5903 | { | |
5904 | ObjectStore::Transaction t; | |
5905 | for (auto &o : bobjects) { | |
5906 | t.remove(a, o); | |
5907 | } | |
5908 | t.remove_collection(a); | |
5909 | r = queue_transaction(store, cha, std::move(t)); | |
5910 | ASSERT_EQ(r, 0); | |
5911 | } | |
5912 | } | |
5913 | ||
5914 | TEST_P(StoreTest, MergeSkewed) { | |
5915 | if (string(GetParam()) != "filestore") | |
5916 | return; | |
5917 | ||
5918 | // this is sufficient to exercise merges with different hashing levels | |
5919 | test_merge_skewed(store.get(), 0xf, 4, 10, 10000); | |
5920 | test_merge_skewed(store.get(), 0xf, 4, 10000, 10); | |
5921 | ||
5922 | /* | |
5923 | // this covers a zillion variations that all boil down to the same thing | |
5924 | for (unsigned base = 3; base < 0x1000; base *= 5) { | |
5925 | unsigned bits; | |
5926 | unsigned t = base; | |
5927 | for (bits = 0; t; t >>= 1) { | |
5928 | ++bits; | |
5929 | } | |
5930 | for (unsigned b = bits; b < bits + 10; b += 3) { | |
5931 | for (auto anum : { 10, 1000, 10000 }) { | |
5932 | for (auto bnum : { 10, 1000, 10000 }) { | |
5933 | if (anum == bnum) { | |
5934 | continue; | |
5935 | } | |
5936 | test_merge_skewed(store.get(), base, b, anum, bnum); | |
5937 | } | |
5938 | } | |
5939 | } | |
5940 | } | |
5941 | */ | |
5942 | } | |
5943 | ||
5944 | ||
7c673cae FG |
5945 | /** |
5946 | * This test tests adding two different groups | |
5947 | * of objects, each with 1 common prefix and 1 | |
5948 | * different prefix. We then remove half | |
5949 | * in order to verify that the merging correctly | |
5950 | * stops at the common prefix subdir. See bug | |
5951 | * #5273 */ | |
5952 | TEST_P(StoreTest, TwoHash) { | |
7c673cae FG |
5953 | coll_t cid; |
5954 | int r; | |
11fdf7f2 | 5955 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5956 | { |
5957 | ObjectStore::Transaction t; | |
5958 | t.create_collection(cid, 0); | |
11fdf7f2 | 5959 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5960 | ASSERT_EQ(r, 0); |
5961 | } | |
5962 | std::cout << "Making objects" << std::endl; | |
5963 | for (int i = 0; i < 360; ++i) { | |
5964 | ObjectStore::Transaction t; | |
5965 | ghobject_t o; | |
5966 | o.hobj.pool = -1; | |
5967 | if (i < 8) { | |
5968 | o.hobj.set_hash((i << 16) | 0xA1); | |
5969 | t.touch(cid, o); | |
5970 | } | |
5971 | o.hobj.set_hash((i << 16) | 0xB1); | |
5972 | t.touch(cid, o); | |
11fdf7f2 | 5973 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5974 | ASSERT_EQ(r, 0); |
5975 | } | |
5976 | std::cout << "Removing half" << std::endl; | |
5977 | for (int i = 1; i < 8; ++i) { | |
5978 | ObjectStore::Transaction t; | |
5979 | ghobject_t o; | |
5980 | o.hobj.pool = -1; | |
5981 | o.hobj.set_hash((i << 16) | 0xA1); | |
5982 | t.remove(cid, o); | |
11fdf7f2 | 5983 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5984 | ASSERT_EQ(r, 0); |
5985 | } | |
5986 | std::cout << "Checking" << std::endl; | |
5987 | for (int i = 1; i < 8; ++i) { | |
5988 | ObjectStore::Transaction t; | |
5989 | ghobject_t o; | |
5990 | o.hobj.set_hash((i << 16) | 0xA1); | |
5991 | o.hobj.pool = -1; | |
11fdf7f2 | 5992 | bool exists = store->exists(ch, o); |
7c673cae FG |
5993 | ASSERT_EQ(exists, false); |
5994 | } | |
5995 | { | |
5996 | ghobject_t o; | |
5997 | o.hobj.set_hash(0xA1); | |
5998 | o.hobj.pool = -1; | |
11fdf7f2 | 5999 | bool exists = store->exists(ch, o); |
7c673cae FG |
6000 | ASSERT_EQ(exists, true); |
6001 | } | |
6002 | std::cout << "Cleanup" << std::endl; | |
6003 | for (int i = 0; i < 360; ++i) { | |
6004 | ObjectStore::Transaction t; | |
6005 | ghobject_t o; | |
6006 | o.hobj.set_hash((i << 16) | 0xA1); | |
6007 | o.hobj.pool = -1; | |
6008 | t.remove(cid, o); | |
6009 | o.hobj.set_hash((i << 16) | 0xB1); | |
6010 | t.remove(cid, o); | |
11fdf7f2 | 6011 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6012 | ASSERT_EQ(r, 0); |
6013 | } | |
6014 | ObjectStore::Transaction t; | |
6015 | t.remove_collection(cid); | |
11fdf7f2 | 6016 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6017 | ASSERT_EQ(r, 0); |
6018 | } | |
6019 | ||
6020 | TEST_P(StoreTest, Rename) { | |
7c673cae FG |
6021 | coll_t cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD)); |
6022 | ghobject_t srcoid(hobject_t("src_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6023 | ghobject_t dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6024 | bufferlist a, b; | |
6025 | a.append("foo"); | |
6026 | b.append("bar"); | |
11fdf7f2 | 6027 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6028 | int r; |
6029 | { | |
6030 | ObjectStore::Transaction t; | |
6031 | t.create_collection(cid, 0); | |
6032 | t.write(cid, srcoid, 0, a.length(), a); | |
11fdf7f2 | 6033 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6034 | ASSERT_EQ(r, 0); |
6035 | } | |
11fdf7f2 | 6036 | ASSERT_TRUE(store->exists(ch, srcoid)); |
7c673cae FG |
6037 | { |
6038 | ObjectStore::Transaction t; | |
6039 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
6040 | t.write(cid, srcoid, 0, b.length(), b); | |
6041 | t.setattr(cid, srcoid, "attr", b); | |
11fdf7f2 | 6042 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6043 | ASSERT_EQ(r, 0); |
6044 | } | |
11fdf7f2 TL |
6045 | ASSERT_TRUE(store->exists(ch, srcoid)); |
6046 | ASSERT_TRUE(store->exists(ch, dstoid)); | |
7c673cae FG |
6047 | { |
6048 | bufferlist bl; | |
11fdf7f2 | 6049 | store->read(ch, srcoid, 0, 3, bl); |
7c673cae | 6050 | ASSERT_TRUE(bl_eq(b, bl)); |
11fdf7f2 | 6051 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
6052 | ASSERT_TRUE(bl_eq(a, bl)); |
6053 | } | |
6054 | { | |
6055 | ObjectStore::Transaction t; | |
6056 | t.remove(cid, dstoid); | |
6057 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
11fdf7f2 | 6058 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6059 | ASSERT_EQ(r, 0); |
6060 | } | |
11fdf7f2 TL |
6061 | ASSERT_TRUE(store->exists(ch, dstoid)); |
6062 | ASSERT_FALSE(store->exists(ch, srcoid)); | |
7c673cae FG |
6063 | { |
6064 | bufferlist bl; | |
11fdf7f2 | 6065 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
6066 | ASSERT_TRUE(bl_eq(b, bl)); |
6067 | } | |
6068 | { | |
6069 | ObjectStore::Transaction t; | |
6070 | t.remove(cid, dstoid); | |
6071 | t.remove_collection(cid); | |
11fdf7f2 | 6072 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6073 | ASSERT_EQ(r, 0); |
6074 | } | |
6075 | } | |
6076 | ||
6077 | TEST_P(StoreTest, MoveRename) { | |
7c673cae FG |
6078 | coll_t cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD)); |
6079 | ghobject_t temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
6080 | ghobject_t oid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 6081 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6082 | int r; |
6083 | { | |
6084 | ObjectStore::Transaction t; | |
6085 | t.create_collection(cid, 0); | |
6086 | t.touch(cid, oid); | |
11fdf7f2 | 6087 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6088 | ASSERT_EQ(r, 0); |
6089 | } | |
11fdf7f2 | 6090 | ASSERT_TRUE(store->exists(ch, oid)); |
7c673cae FG |
6091 | bufferlist data, attr; |
6092 | map<string, bufferlist> omap; | |
6093 | data.append("data payload"); | |
6094 | attr.append("attr value"); | |
6095 | omap["omap_key"].append("omap value"); | |
6096 | { | |
6097 | ObjectStore::Transaction t; | |
6098 | t.touch(cid, temp_oid); | |
6099 | t.write(cid, temp_oid, 0, data.length(), data); | |
6100 | t.setattr(cid, temp_oid, "attr", attr); | |
6101 | t.omap_setkeys(cid, temp_oid, omap); | |
11fdf7f2 | 6102 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6103 | ASSERT_EQ(r, 0); |
6104 | } | |
11fdf7f2 | 6105 | ASSERT_TRUE(store->exists(ch, temp_oid)); |
7c673cae FG |
6106 | { |
6107 | ObjectStore::Transaction t; | |
6108 | t.remove(cid, oid); | |
6109 | t.collection_move_rename(cid, temp_oid, cid, oid); | |
11fdf7f2 | 6110 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6111 | ASSERT_EQ(r, 0); |
6112 | } | |
11fdf7f2 TL |
6113 | ASSERT_TRUE(store->exists(ch, oid)); |
6114 | ASSERT_FALSE(store->exists(ch, temp_oid)); | |
7c673cae FG |
6115 | { |
6116 | bufferlist newdata; | |
11fdf7f2 | 6117 | r = store->read(ch, oid, 0, 1000, newdata); |
7c673cae FG |
6118 | ASSERT_GE(r, 0); |
6119 | ASSERT_TRUE(bl_eq(data, newdata)); | |
6120 | bufferlist newattr; | |
11fdf7f2 | 6121 | r = store->getattr(ch, oid, "attr", newattr); |
7c673cae FG |
6122 | ASSERT_EQ(r, 0); |
6123 | ASSERT_TRUE(bl_eq(attr, newattr)); | |
6124 | set<string> keys; | |
6125 | keys.insert("omap_key"); | |
6126 | map<string, bufferlist> newomap; | |
11fdf7f2 | 6127 | r = store->omap_get_values(ch, oid, keys, &newomap); |
7c673cae FG |
6128 | ASSERT_GE(r, 0); |
6129 | ASSERT_EQ(1u, newomap.size()); | |
6130 | ASSERT_TRUE(newomap.count("omap_key")); | |
6131 | ASSERT_TRUE(bl_eq(omap["omap_key"], newomap["omap_key"])); | |
6132 | } | |
6133 | { | |
6134 | ObjectStore::Transaction t; | |
6135 | t.remove(cid, oid); | |
6136 | t.remove_collection(cid); | |
11fdf7f2 | 6137 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6138 | ASSERT_EQ(r, 0); |
6139 | } | |
6140 | } | |
6141 | ||
6142 | TEST_P(StoreTest, BigRGWObjectName) { | |
7c673cae FG |
6143 | coll_t cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD)); |
6144 | ghobject_t oid( | |
6145 | hobject_t( | |
6146 | "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", | |
6147 | "", | |
6148 | CEPH_NOSNAP, | |
6149 | 0x81920472, | |
6150 | 12, | |
6151 | ""), | |
6152 | 15, | |
6153 | shard_id_t::NO_SHARD); | |
6154 | ghobject_t oid2(oid); | |
6155 | oid2.generation = 17; | |
6156 | ghobject_t oidhead(oid); | |
6157 | oidhead.generation = ghobject_t::NO_GEN; | |
6158 | ||
11fdf7f2 TL |
6159 | auto ch = store->create_new_collection(cid); |
6160 | ||
7c673cae FG |
6161 | int r; |
6162 | { | |
6163 | ObjectStore::Transaction t; | |
6164 | t.create_collection(cid, 0); | |
6165 | t.touch(cid, oidhead); | |
6166 | t.collection_move_rename(cid, oidhead, cid, oid); | |
6167 | t.touch(cid, oidhead); | |
6168 | t.collection_move_rename(cid, oidhead, cid, oid2); | |
11fdf7f2 | 6169 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6170 | ASSERT_EQ(r, 0); |
6171 | } | |
6172 | ||
6173 | { | |
6174 | ObjectStore::Transaction t; | |
6175 | t.remove(cid, oid); | |
11fdf7f2 | 6176 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6177 | ASSERT_EQ(r, 0); |
6178 | } | |
6179 | ||
6180 | { | |
6181 | vector<ghobject_t> objects; | |
f91f0fd5 TL |
6182 | r = collection_list(store, ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, |
6183 | &objects, 0); | |
7c673cae FG |
6184 | ASSERT_EQ(r, 0); |
6185 | ASSERT_EQ(objects.size(), 1u); | |
6186 | ASSERT_EQ(objects[0], oid2); | |
6187 | } | |
6188 | ||
11fdf7f2 | 6189 | ASSERT_FALSE(store->exists(ch, oid)); |
7c673cae FG |
6190 | |
6191 | { | |
6192 | ObjectStore::Transaction t; | |
6193 | t.remove(cid, oid2); | |
6194 | t.remove_collection(cid); | |
11fdf7f2 | 6195 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6196 | ASSERT_EQ(r, 0); |
6197 | ||
6198 | } | |
6199 | } | |
6200 | ||
6201 | TEST_P(StoreTest, SetAllocHint) { | |
7c673cae FG |
6202 | coll_t cid; |
6203 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 6204 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6205 | int r; |
6206 | { | |
6207 | ObjectStore::Transaction t; | |
6208 | t.create_collection(cid, 0); | |
6209 | t.touch(cid, hoid); | |
11fdf7f2 | 6210 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6211 | ASSERT_EQ(r, 0); |
6212 | } | |
6213 | { | |
6214 | ObjectStore::Transaction t; | |
6215 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6216 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6217 | ASSERT_EQ(r, 0); |
6218 | } | |
6219 | { | |
6220 | ObjectStore::Transaction t; | |
6221 | t.remove(cid, hoid); | |
11fdf7f2 | 6222 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6223 | ASSERT_EQ(r, 0); |
6224 | } | |
6225 | { | |
6226 | ObjectStore::Transaction t; | |
6227 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6228 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6229 | ASSERT_EQ(r, 0); |
6230 | } | |
6231 | { | |
6232 | ObjectStore::Transaction t; | |
6233 | t.remove_collection(cid); | |
11fdf7f2 | 6234 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6235 | ASSERT_EQ(r, 0); |
6236 | } | |
6237 | } | |
6238 | ||
6239 | TEST_P(StoreTest, TryMoveRename) { | |
7c673cae FG |
6240 | coll_t cid; |
6241 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6242 | ghobject_t hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP, 0, -1, "")); | |
11fdf7f2 | 6243 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6244 | int r; |
6245 | { | |
6246 | ObjectStore::Transaction t; | |
6247 | t.create_collection(cid, 0); | |
11fdf7f2 | 6248 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6249 | ASSERT_EQ(r, 0); |
6250 | } | |
6251 | { | |
6252 | ObjectStore::Transaction t; | |
6253 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6254 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6255 | ASSERT_EQ(r, 0); |
6256 | } | |
6257 | { | |
6258 | ObjectStore::Transaction t; | |
6259 | t.touch(cid, hoid); | |
11fdf7f2 | 6260 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6261 | ASSERT_EQ(r, 0); |
6262 | } | |
6263 | { | |
6264 | ObjectStore::Transaction t; | |
6265 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6266 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6267 | ASSERT_EQ(r, 0); |
6268 | } | |
6269 | struct stat st; | |
11fdf7f2 TL |
6270 | ASSERT_EQ(store->stat(ch, hoid, &st), -ENOENT); |
6271 | ASSERT_EQ(store->stat(ch, hoid2, &st), 0); | |
7c673cae FG |
6272 | } |
6273 | ||
11fdf7f2 | 6274 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6275 | TEST_P(StoreTest, BluestoreOnOffCSumTest) { |
6276 | if (string(GetParam()) != "bluestore") | |
6277 | return; | |
11fdf7f2 TL |
6278 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6279 | g_conf().apply_changes(nullptr); | |
7c673cae | 6280 | |
7c673cae FG |
6281 | int r; |
6282 | coll_t cid; | |
6283 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6284 | { | |
11fdf7f2 TL |
6285 | auto ch = store->open_collection(cid); |
6286 | ASSERT_FALSE(ch); | |
7c673cae | 6287 | } |
11fdf7f2 | 6288 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6289 | { |
6290 | ObjectStore::Transaction t; | |
6291 | t.create_collection(cid, 0); | |
6292 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 6293 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6294 | ASSERT_EQ(r, 0); |
6295 | } | |
6296 | { | |
6297 | //write with csum enabled followed by read with csum disabled | |
6298 | size_t block_size = 64*1024; | |
6299 | ObjectStore::Transaction t; | |
6300 | bufferlist bl, orig; | |
6301 | bl.append(std::string(block_size, 'a')); | |
6302 | orig = bl; | |
6303 | t.remove(cid, hoid); | |
6304 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6305 | t.write(cid, hoid, 0, bl.length(), bl); | |
6306 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6307 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6308 | ASSERT_EQ(r, 0); |
6309 | ||
11fdf7f2 TL |
6310 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6311 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6312 | |
6313 | bufferlist in; | |
11fdf7f2 | 6314 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6315 | ASSERT_EQ((int)block_size, r); |
6316 | ASSERT_TRUE(bl_eq(orig, in)); | |
6317 | ||
6318 | } | |
6319 | { | |
6320 | //write with csum disabled followed by read with csum enabled | |
6321 | ||
6322 | size_t block_size = 64*1024; | |
6323 | ObjectStore::Transaction t; | |
6324 | bufferlist bl, orig; | |
6325 | bl.append(std::string(block_size, 'a')); | |
6326 | orig = bl; | |
6327 | t.remove(cid, hoid); | |
6328 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6329 | t.write(cid, hoid, 0, bl.length(), bl); | |
6330 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6331 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6332 | ASSERT_EQ(r, 0); |
6333 | ||
11fdf7f2 TL |
6334 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6335 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6336 | |
6337 | bufferlist in; | |
11fdf7f2 | 6338 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6339 | ASSERT_EQ((int)block_size, r); |
6340 | ASSERT_TRUE(bl_eq(orig, in)); | |
6341 | } | |
6342 | { | |
6343 | //'mixed' non-overlapping writes to the same blob | |
6344 | ||
6345 | ObjectStore::Transaction t; | |
6346 | bufferlist bl, orig; | |
6347 | size_t block_size = 8000; | |
6348 | bl.append(std::string(block_size, 'a')); | |
6349 | orig = bl; | |
6350 | t.remove(cid, hoid); | |
6351 | t.write(cid, hoid, 0, bl.length(), bl); | |
6352 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6353 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6354 | ASSERT_EQ(r, 0); |
6355 | ||
11fdf7f2 TL |
6356 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6357 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6358 | |
6359 | ObjectStore::Transaction t2; | |
6360 | t2.write(cid, hoid, block_size*2, bl.length(), bl); | |
6361 | cerr << "Append 'unprotected'" << std::endl; | |
11fdf7f2 | 6362 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6363 | ASSERT_EQ(r, 0); |
6364 | ||
6365 | bufferlist in; | |
11fdf7f2 | 6366 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6367 | ASSERT_EQ((int)block_size, r); |
6368 | ASSERT_TRUE(bl_eq(orig, in)); | |
6369 | in.clear(); | |
11fdf7f2 | 6370 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6371 | ASSERT_EQ((int)block_size, r); |
6372 | ASSERT_TRUE(bl_eq(orig, in)); | |
6373 | ||
11fdf7f2 TL |
6374 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6375 | g_conf().apply_changes(nullptr); | |
7c673cae | 6376 | in.clear(); |
11fdf7f2 | 6377 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6378 | ASSERT_EQ((int)block_size, r); |
6379 | ASSERT_TRUE(bl_eq(orig, in)); | |
6380 | in.clear(); | |
11fdf7f2 | 6381 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6382 | ASSERT_EQ((int)block_size, r); |
6383 | ASSERT_TRUE(bl_eq(orig, in)); | |
6384 | } | |
6385 | { | |
6386 | //partially blob overwrite under a different csum enablement mode | |
6387 | ||
6388 | ObjectStore::Transaction t; | |
6389 | bufferlist bl, orig, orig2; | |
6390 | size_t block_size0 = 0x10000; | |
6391 | size_t block_size = 9000; | |
6392 | size_t block_size2 = 5000; | |
6393 | bl.append(std::string(block_size0, 'a')); | |
6394 | t.remove(cid, hoid); | |
6395 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6396 | t.write(cid, hoid, 0, bl.length(), bl); | |
6397 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6398 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6399 | ASSERT_EQ(r, 0); |
6400 | ||
11fdf7f2 TL |
6401 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6402 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6403 | |
6404 | ObjectStore::Transaction t2; | |
6405 | bl.clear(); | |
6406 | bl.append(std::string(block_size, 'b')); | |
6407 | t2.write(cid, hoid, 0, bl.length(), bl); | |
6408 | t2.write(cid, hoid, block_size0, bl.length(), bl); | |
6409 | cerr << "Overwrite with unprotected data" << std::endl; | |
11fdf7f2 | 6410 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6411 | ASSERT_EQ(r, 0); |
6412 | ||
6413 | orig = bl; | |
6414 | orig2 = bl; | |
6415 | orig.append( std::string(block_size0 - block_size, 'a')); | |
6416 | ||
6417 | bufferlist in; | |
11fdf7f2 | 6418 | r = store->read(ch, hoid, 0, block_size0, in); |
7c673cae FG |
6419 | ASSERT_EQ((int)block_size0, r); |
6420 | ASSERT_TRUE(bl_eq(orig, in)); | |
6421 | ||
11fdf7f2 | 6422 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6423 | ASSERT_EQ((int)block_size, r); |
6424 | ASSERT_TRUE(bl_eq(orig2, in)); | |
6425 | ||
11fdf7f2 TL |
6426 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6427 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6428 | |
6429 | ObjectStore::Transaction t3; | |
6430 | bl.clear(); | |
6431 | bl.append(std::string(block_size2, 'c')); | |
6432 | t3.write(cid, hoid, block_size0, bl.length(), bl); | |
6433 | cerr << "Overwrite with protected data" << std::endl; | |
11fdf7f2 | 6434 | r = queue_transaction(store, ch, std::move(t3)); |
7c673cae FG |
6435 | ASSERT_EQ(r, 0); |
6436 | ||
6437 | in.clear(); | |
6438 | orig = bl; | |
6439 | orig.append( std::string(block_size - block_size2, 'b')); | |
11fdf7f2 | 6440 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6441 | ASSERT_EQ((int)block_size, r); |
6442 | ASSERT_TRUE(bl_eq(orig, in)); | |
6443 | } | |
6444 | ||
6445 | { | |
6446 | ObjectStore::Transaction t; | |
6447 | t.remove(cid, hoid); | |
6448 | t.remove_collection(cid); | |
6449 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6450 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6451 | ASSERT_EQ(r, 0); |
6452 | } | |
6453 | } | |
6454 | #endif | |
6455 | ||
9f95a23c | 6456 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6457 | ObjectStore, |
6458 | StoreTest, | |
6459 | ::testing::Values( | |
6460 | "memstore", | |
6461 | "filestore", | |
11fdf7f2 | 6462 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6463 | "bluestore", |
6464 | #endif | |
6465 | "kstore")); | |
6466 | ||
6467 | // Note: instantiate all stores to preserve store numbering order only | |
9f95a23c | 6468 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6469 | ObjectStore, |
6470 | StoreTestSpecificAUSize, | |
6471 | ::testing::Values( | |
6472 | "memstore", | |
6473 | "filestore", | |
11fdf7f2 | 6474 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6475 | "bluestore", |
6476 | #endif | |
6477 | "kstore")); | |
6478 | ||
7c673cae FG |
6479 | void doMany4KWritesTest(boost::scoped_ptr<ObjectStore>& store, |
6480 | unsigned max_objects, | |
6481 | unsigned max_ops, | |
6482 | unsigned max_object_size, | |
6483 | unsigned max_write_size, | |
11fdf7f2 | 6484 | unsigned write_alignment) |
7c673cae | 6485 | { |
7c673cae FG |
6486 | MixedGenerator gen(555); |
6487 | gen_type rng(time(NULL)); | |
6488 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 6489 | store_statfs_t res_stat; |
7c673cae FG |
6490 | |
6491 | SyntheticWorkloadState test_obj(store.get(), | |
6492 | &gen, | |
6493 | &rng, | |
7c673cae FG |
6494 | cid, |
6495 | max_object_size, | |
6496 | max_write_size, | |
6497 | write_alignment); | |
6498 | test_obj.init(); | |
6499 | for (unsigned i = 0; i < max_objects; ++i) { | |
6500 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
6501 | test_obj.touch(); | |
6502 | } | |
6503 | for (unsigned i = 0; i < max_ops; ++i) { | |
6504 | if (!(i % 200)) { | |
6505 | cerr << "Op " << i << std::endl; | |
6506 | test_obj.print_internal_state(); | |
6507 | } | |
6508 | test_obj.write(); | |
6509 | } | |
6510 | test_obj.wait_for_done(); | |
11fdf7f2 TL |
6511 | test_obj.statfs(res_stat); |
6512 | if (!(res_stat.data_stored <= max_object_size) || | |
6513 | !(res_stat.allocated <= max_object_size)) { | |
6514 | // this will provide more insight on the mismatch and | |
6515 | // helps to avoid any races during stats collection | |
6516 | test_obj.fsck(false); | |
6517 | // retrieving stats once again and assert if still broken | |
6518 | test_obj.statfs(res_stat); | |
6519 | ASSERT_LE(res_stat.data_stored, max_object_size); | |
6520 | ASSERT_LE(res_stat.allocated, max_object_size); | |
7c673cae FG |
6521 | } |
6522 | test_obj.shutdown(); | |
6523 | } | |
6524 | ||
6525 | TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { | |
6526 | if (string(GetParam()) != "bluestore") | |
6527 | return; | |
6528 | ||
6529 | StartDeferred(0x10000); | |
6530 | ||
11fdf7f2 TL |
6531 | const unsigned max_object = 4*1024*1024; |
6532 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0); | |
7c673cae FG |
6533 | } |
6534 | ||
6535 | TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { | |
6536 | if (string(GetParam()) != "bluestore") | |
6537 | return; | |
6538 | StartDeferred(0x10000); | |
11fdf7f2 TL |
6539 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6540 | g_ceph_context->_conf.apply_changes(nullptr); | |
6541 | const unsigned max_object = 4*1024*1024; | |
7c673cae | 6542 | |
11fdf7f2 | 6543 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0 ); |
7c673cae FG |
6544 | } |
6545 | ||
6546 | TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { | |
6547 | if (string(GetParam()) != "bluestore") | |
6548 | return; | |
6549 | StartDeferred(0x10000); | |
11fdf7f2 TL |
6550 | const unsigned max_object = 4*1024*1024; |
6551 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0); | |
7c673cae FG |
6552 | } |
6553 | ||
11fdf7f2 | 6554 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6555 | void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items) |
6556 | { | |
f91f0fd5 | 6557 | uint64_t meta_allocated = mempool::bluestore_cache_meta::allocated_bytes(); |
31f18b77 FG |
6558 | uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes(); |
6559 | uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes(); | |
7c673cae | 6560 | |
f91f0fd5 | 6561 | uint64_t meta_items = mempool::bluestore_cache_meta::allocated_items(); |
31f18b77 FG |
6562 | uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items(); |
6563 | uint64_t other_items = mempool::bluestore_cache_other::allocated_items(); | |
f91f0fd5 TL |
6564 | cout << "meta(" << meta_allocated << "/" << meta_items |
6565 | << ") onode(" << onode_allocated << "/" << onode_items | |
7c673cae FG |
6566 | << ") other(" << other_allocated << "/" << other_items |
6567 | << ")" << std::endl; | |
f91f0fd5 | 6568 | *total_bytes = meta_allocated + onode_allocated + other_allocated; |
7c673cae FG |
6569 | *total_items = onode_items; |
6570 | } | |
6571 | ||
6572 | TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) { | |
6573 | ||
6574 | if (string(GetParam()) != "bluestore") | |
6575 | return; | |
6576 | ||
6577 | size_t block_size = 4096; | |
6578 | StartDeferred(block_size); | |
11fdf7f2 TL |
6579 | SetVal(g_conf(), "bluestore_compression_mode", "none"); |
6580 | SetVal(g_conf(), "bluestore_csum_type", "none"); | |
6581 | SetVal(g_conf(), "bluestore_cache_size_hdd", "400000000"); | |
6582 | SetVal(g_conf(), "bluestore_cache_size_ssd", "400000000"); | |
6583 | g_conf().apply_changes(nullptr); | |
7c673cae | 6584 | |
7c673cae FG |
6585 | int r; |
6586 | coll_t cid; | |
6587 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6588 | size_t obj_size = 4 * 1024 * 1024; | |
6589 | uint64_t total_bytes, total_bytes2; | |
6590 | uint64_t total_onodes; | |
6591 | get_mempool_stats(&total_bytes, &total_onodes); | |
6592 | ASSERT_EQ(total_onodes, 0u); | |
6593 | ||
11fdf7f2 | 6594 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6595 | { |
6596 | ObjectStore::Transaction t; | |
6597 | t.create_collection(cid, 0); | |
11fdf7f2 | 6598 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6599 | ASSERT_EQ(r, 0); |
6600 | } | |
6601 | { | |
6602 | ObjectStore::Transaction t; | |
6603 | bufferlist bl, orig, orig2; | |
6604 | ||
6605 | bl.append(std::string(obj_size, 'a')); | |
6606 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 6607 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6608 | ASSERT_EQ(r, 0); |
6609 | } | |
6610 | get_mempool_stats(&total_bytes, &total_onodes); | |
6611 | ASSERT_NE(total_bytes, 0u); | |
6612 | ASSERT_EQ(total_onodes, 1u); | |
6613 | ||
6614 | { | |
6615 | ObjectStore::Transaction t; | |
6616 | t.truncate(cid, hoid, 0); | |
11fdf7f2 | 6617 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6618 | ASSERT_EQ(r, 0); |
6619 | } | |
6620 | ||
6621 | for(size_t i = 0; i < 1; ++i) { | |
6622 | bufferlist bl; | |
6623 | bl.append(std::string(block_size * (i+1), 'a')); | |
6624 | for( size_t j = 0; j < obj_size; j+= bl.length()) { | |
6625 | ObjectStore::Transaction t; | |
6626 | t.write(cid, hoid, j, bl.length(), bl); | |
11fdf7f2 | 6627 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6628 | ASSERT_EQ(r, 0); |
6629 | } | |
6630 | get_mempool_stats(&total_bytes2, &total_onodes); | |
6631 | ASSERT_NE(total_bytes2, 0u); | |
6632 | ASSERT_EQ(total_onodes, 1u); | |
6633 | } | |
6634 | { | |
6635 | cout <<" mempool dump:\n"; | |
6636 | JSONFormatter f(true); | |
6637 | f.open_object_section("transaction"); | |
6638 | mempool::dump(&f); | |
6639 | f.close_section(); | |
6640 | f.flush(cout); | |
6641 | cout << std::endl; | |
6642 | } | |
6643 | { | |
6644 | bufferlist bl; | |
6645 | for (size_t i = 0; i < obj_size; i += 0x1000) { | |
11fdf7f2 | 6646 | store->read(ch, hoid, i, 0x1000, bl); |
7c673cae FG |
6647 | } |
6648 | } | |
6649 | get_mempool_stats(&total_bytes, &total_onodes); | |
6650 | ASSERT_NE(total_bytes, 0u); | |
6651 | ASSERT_EQ(total_onodes, 1u); | |
6652 | ||
6653 | { | |
6654 | cout <<" mempool dump:\n"; | |
6655 | JSONFormatter f(true); | |
6656 | f.open_object_section("transaction"); | |
6657 | mempool::dump(&f); | |
6658 | f.close_section(); | |
6659 | f.flush(cout); | |
6660 | cout << std::endl; | |
6661 | } | |
6662 | { | |
6663 | ObjectStore::Transaction t; | |
6664 | t.remove(cid, hoid); | |
6665 | t.remove_collection(cid); | |
6666 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6667 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6668 | ASSERT_EQ(r, 0); |
6669 | } | |
7c673cae FG |
6670 | } |
6671 | ||
6672 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) { | |
6673 | ||
6674 | if (string(GetParam()) != "bluestore") | |
6675 | return; | |
6676 | ||
6677 | size_t block_size = 4096; | |
6678 | StartDeferred(block_size); | |
11fdf7f2 TL |
6679 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
6680 | g_conf().apply_changes(nullptr); | |
7c673cae | 6681 | |
7c673cae FG |
6682 | int r; |
6683 | coll_t cid; | |
6684 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6685 | ||
6686 | const PerfCounters* logger = store->get_perf_counters(); | |
6687 | ||
11fdf7f2 | 6688 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6689 | { |
6690 | ObjectStore::Transaction t; | |
6691 | t.create_collection(cid, 0); | |
11fdf7f2 | 6692 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6693 | ASSERT_EQ(r, 0); |
6694 | } | |
6695 | { | |
6696 | ObjectStore::Transaction t; | |
6697 | bufferlist bl; | |
6698 | ||
6699 | bl.append(std::string(block_size * 2, 'a')); | |
6700 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6701 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6702 | ASSERT_EQ(r, 0); |
6703 | } | |
6704 | { | |
6705 | // overwrite at the beginning | |
6706 | ObjectStore::Transaction t; | |
6707 | bufferlist bl; | |
6708 | ||
6709 | bl.append(std::string(block_size, 'b')); | |
6710 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6711 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6712 | ASSERT_EQ(r, 0); |
6713 | } | |
6714 | { | |
6715 | // append | |
6716 | ObjectStore::Transaction t; | |
6717 | bufferlist bl; | |
6718 | ||
6719 | bl.append(std::string(block_size * 2, 'c')); | |
6720 | t.write(cid, hoid, block_size * 2, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6721 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6722 | ASSERT_EQ(r, 0); |
6723 | } | |
6724 | { | |
6725 | // append with a gap | |
6726 | ObjectStore::Transaction t; | |
6727 | bufferlist bl; | |
6728 | ||
6729 | bl.append(std::string(block_size * 2, 'd')); | |
6730 | t.write(cid, hoid, block_size * 5, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6731 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6732 | ASSERT_EQ(r, 0); |
6733 | } | |
6734 | { | |
6735 | // We need to issue a read to trigger cache stat update that refresh | |
6736 | // perf counters. additionally we need to wait some time for mempool | |
6737 | // thread to update stats. | |
6738 | sleep(1); | |
6739 | bufferlist bl, expected; | |
11fdf7f2 | 6740 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6741 | ASSERT_EQ(r, (int)block_size); |
6742 | expected.append(string(block_size, 'b')); | |
6743 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6744 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6745 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6746 | } | |
6747 | { | |
6748 | // overwrite at end | |
6749 | ObjectStore::Transaction t; | |
6750 | bufferlist bl; | |
6751 | ||
6752 | bl.append(std::string(block_size * 2, 'e')); | |
6753 | ||
6754 | // Currently we are unable to reuse blob when overwriting in a single step | |
6755 | t.write(cid, hoid, block_size * 6, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6756 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6757 | ASSERT_EQ(r, 0); |
6758 | } | |
6759 | { | |
6760 | // We need to issue a read to trigger cache stat update that refresh | |
6761 | // perf counters. additionally we need to wait some time for mempool | |
6762 | // thread to update stats. | |
6763 | sleep(1); | |
6764 | bufferlist bl, expected; | |
11fdf7f2 | 6765 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6766 | ASSERT_EQ(r, (int)block_size); |
6767 | expected.append(string(block_size, 'b')); | |
6768 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6769 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6770 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6771 | } | |
6772 | { | |
6773 | // fill the gap | |
6774 | ObjectStore::Transaction t; | |
6775 | bufferlist bl; | |
6776 | ||
6777 | bl.append(std::string(block_size, 'f')); | |
6778 | ||
6779 | t.write(cid, hoid, block_size * 4, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6780 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6781 | ASSERT_EQ(r, 0); |
6782 | } | |
6783 | { | |
6784 | // we need to wait some time for mempool | |
6785 | // thread to update stats to be able to check blob/extent numbers from | |
6786 | // perf counters. | |
6787 | sleep(1); | |
6788 | ||
6789 | bufferlist bl, expected; | |
11fdf7f2 | 6790 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6791 | ASSERT_EQ(r, (int)block_size); |
6792 | expected.append(string(block_size, 'b')); | |
6793 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6794 | ||
6795 | bl.clear(); | |
6796 | expected.clear(); | |
11fdf7f2 | 6797 | r = store->read(ch, hoid, block_size, block_size, bl); |
7c673cae FG |
6798 | ASSERT_EQ(r, (int)block_size); |
6799 | expected.append(string(block_size, 'a')); | |
6800 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6801 | ||
6802 | bl.clear(); | |
6803 | expected.clear(); | |
11fdf7f2 | 6804 | r = store->read(ch, hoid, block_size * 2, block_size * 2, bl); |
7c673cae FG |
6805 | ASSERT_EQ(r, (int)block_size * 2); |
6806 | expected.append(string(block_size * 2, 'c')); | |
6807 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6808 | ||
6809 | bl.clear(); | |
6810 | expected.clear(); | |
11fdf7f2 | 6811 | r = store->read(ch, hoid, block_size * 4, block_size, bl); |
7c673cae FG |
6812 | ASSERT_EQ(r, (int)block_size); |
6813 | expected.append(string(block_size, 'f')); | |
6814 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6815 | ||
6816 | bl.clear(); | |
6817 | expected.clear(); | |
11fdf7f2 | 6818 | r = store->read(ch, hoid, block_size * 5, block_size, bl); |
7c673cae FG |
6819 | ASSERT_EQ(r, (int)block_size); |
6820 | expected.append(string(block_size, 'd')); | |
6821 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6822 | ||
6823 | bl.clear(); | |
6824 | expected.clear(); | |
11fdf7f2 | 6825 | r = store->read(ch, hoid, block_size * 5, block_size * 3, bl); |
7c673cae FG |
6826 | ASSERT_EQ(r, (int)block_size * 3); |
6827 | expected.append(string(block_size, 'd')); | |
6828 | expected.append(string(block_size * 2, 'e')); | |
6829 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6830 | } | |
6831 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6832 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6833 | ||
6834 | ||
6835 | { | |
6836 | ObjectStore::Transaction t; | |
6837 | t.remove(cid, hoid); | |
6838 | t.remove_collection(cid); | |
6839 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6840 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6841 | ASSERT_EQ(r, 0); |
6842 | } | |
7c673cae FG |
6843 | } |
6844 | ||
6845 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) { | |
6846 | ||
6847 | if (string(GetParam()) != "bluestore") | |
6848 | return; | |
6849 | ||
6850 | size_t block_size = 4096; | |
6851 | StartDeferred(block_size); | |
11fdf7f2 TL |
6852 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
6853 | g_conf().apply_changes(nullptr); | |
7c673cae | 6854 | |
7c673cae FG |
6855 | int r; |
6856 | coll_t cid; | |
6857 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6858 | ||
11fdf7f2 TL |
6859 | auto ch = store->create_new_collection(cid); |
6860 | ||
7c673cae FG |
6861 | const PerfCounters* logger = store->get_perf_counters(); |
6862 | { | |
6863 | ObjectStore::Transaction t; | |
6864 | t.create_collection(cid, 0); | |
11fdf7f2 | 6865 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6866 | ASSERT_EQ(r, 0); |
6867 | } | |
6868 | { | |
6869 | ObjectStore::Transaction t; | |
6870 | bufferlist bl; | |
6871 | ||
6872 | bl.append(std::string(block_size * 2, 'a')); | |
6873 | t.write(cid, hoid, block_size * 10, bl.length(), bl, | |
6874 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6875 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6876 | ASSERT_EQ(r, 0); |
6877 | } | |
6878 | { | |
6879 | // prepend existing | |
6880 | ObjectStore::Transaction t; | |
6881 | bufferlist bl; | |
6882 | ||
6883 | bl.append(std::string(block_size, 'b')); | |
6884 | t.write(cid, hoid, block_size * 9, bl.length(), bl, | |
6885 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6886 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6887 | ASSERT_EQ(r, 0); |
6888 | } | |
6889 | { | |
6890 | // We need to issue a read to trigger cache stat update that refresh | |
6891 | // perf counters. additionally we need to wait some time for mempool | |
6892 | // thread to update stats. | |
6893 | sleep(1); | |
6894 | bufferlist bl, expected; | |
11fdf7f2 | 6895 | r = store->read(ch, hoid, block_size * 9, block_size * 2, bl); |
7c673cae FG |
6896 | ASSERT_EQ(r, (int)block_size * 2); |
6897 | expected.append(string(block_size, 'b')); | |
6898 | expected.append(string(block_size, 'a')); | |
6899 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6900 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6901 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6902 | } | |
6903 | ||
6904 | ||
6905 | { | |
6906 | // prepend existing with a gap | |
6907 | ObjectStore::Transaction t; | |
6908 | bufferlist bl; | |
6909 | ||
6910 | bl.append(std::string(block_size, 'c')); | |
6911 | t.write(cid, hoid, block_size * 7, bl.length(), bl, | |
6912 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6913 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6914 | ASSERT_EQ(r, 0); |
6915 | } | |
6916 | { | |
6917 | // We need to issue a read to trigger cache stat update that refresh | |
6918 | // perf counters. additionally we need to wait some time for mempool | |
6919 | // thread to update stats. | |
6920 | sleep(1); | |
6921 | bufferlist bl, expected; | |
11fdf7f2 | 6922 | r = store->read(ch, hoid, block_size * 7, block_size * 3, bl); |
7c673cae FG |
6923 | ASSERT_EQ(r, (int)block_size * 3); |
6924 | expected.append(string(block_size, 'c')); | |
6925 | expected.append(string(block_size, 0)); | |
6926 | expected.append(string(block_size, 'b')); | |
6927 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6928 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6929 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6930 | } | |
6931 | ||
6932 | { | |
6933 | // append after existing with a gap | |
6934 | ObjectStore::Transaction t; | |
6935 | bufferlist bl; | |
6936 | ||
6937 | bl.append(std::string(block_size, 'd')); | |
6938 | t.write(cid, hoid, block_size * 13, bl.length(), bl, | |
6939 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6940 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6941 | ASSERT_EQ(r, 0); |
6942 | } | |
6943 | { | |
6944 | // We need to issue a read to trigger cache stat update that refresh | |
6945 | // perf counters. additionally we need to wait some time for mempool | |
6946 | // thread to update stats. | |
6947 | sleep(1); | |
6948 | bufferlist bl, expected; | |
11fdf7f2 | 6949 | r = store->read(ch, hoid, block_size * 11, block_size * 3, bl); |
7c673cae FG |
6950 | ASSERT_EQ(r, (int)block_size * 3); |
6951 | expected.append(string(block_size, 'a')); | |
6952 | expected.append(string(block_size, 0)); | |
6953 | expected.append(string(block_size, 'd')); | |
6954 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6955 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6956 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
6957 | } | |
6958 | ||
6959 | { | |
6960 | // append twice to the next max_blob slot | |
6961 | ObjectStore::Transaction t; | |
6962 | bufferlist bl; | |
6963 | ||
6964 | bl.append(std::string(block_size, 'e')); | |
6965 | t.write(cid, hoid, block_size * 17, bl.length(), bl, | |
6966 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6967 | t.write(cid, hoid, block_size * 19, bl.length(), bl, | |
6968 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6969 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6970 | ASSERT_EQ(r, 0); |
6971 | } | |
6972 | { | |
6973 | // We need to issue a read to trigger cache stat update that refresh | |
6974 | // perf counters. additionally we need to wait some time for mempool | |
6975 | // thread to update stats. | |
6976 | sleep(1); | |
6977 | bufferlist bl, expected; | |
11fdf7f2 | 6978 | r = store->read(ch, hoid, block_size * 17, block_size * 3, bl); |
7c673cae FG |
6979 | ASSERT_EQ(r, (int)block_size * 3); |
6980 | expected.append(string(block_size, 'e')); | |
6981 | expected.append(string(block_size, 0)); | |
6982 | expected.append(string(block_size, 'e')); | |
6983 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6984 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
6985 | ASSERT_EQ(logger->get(l_bluestore_extents), 5u); | |
6986 | } | |
6987 | { | |
6988 | // fill gaps at the second slot | |
6989 | ObjectStore::Transaction t; | |
6990 | bufferlist bl; | |
6991 | ||
6992 | bl.append(std::string(block_size, 'f')); | |
6993 | t.write(cid, hoid, block_size * 16, bl.length(), bl, | |
6994 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6995 | t.write(cid, hoid, block_size * 18, bl.length(), bl, | |
6996 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6997 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6998 | ASSERT_EQ(r, 0); |
6999 | } | |
7000 | { | |
7001 | // We need to issue a read to trigger cache stat update that refresh | |
7002 | // perf counters. additionally we need to wait some time for mempool | |
7003 | // thread to update stats. | |
7004 | sleep(1); | |
7005 | bufferlist bl, expected; | |
11fdf7f2 | 7006 | r = store->read(ch, hoid, block_size * 16, block_size * 4, bl); |
7c673cae FG |
7007 | ASSERT_EQ(r, (int)block_size * 4); |
7008 | expected.append(string(block_size, 'f')); | |
7009 | expected.append(string(block_size, 'e')); | |
7010 | expected.append(string(block_size, 'f')); | |
7011 | expected.append(string(block_size, 'e')); | |
7012 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7013 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
7014 | ASSERT_EQ(logger->get(l_bluestore_extents), 4u); | |
7015 | } | |
7016 | { | |
7017 | ObjectStore::Transaction t; | |
7018 | t.remove(cid, hoid); | |
7019 | t.remove_collection(cid); | |
7020 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7021 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7022 | ASSERT_EQ(r, 0); |
7023 | } | |
7c673cae FG |
7024 | } |
7025 | ||
7026 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) { | |
7027 | ||
7028 | if (string(GetParam()) != "bluestore") | |
7029 | return; | |
7030 | ||
7031 | size_t block_size = 4096; | |
7032 | StartDeferred(block_size); | |
11fdf7f2 TL |
7033 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
7034 | g_conf().apply_changes(nullptr); | |
7c673cae | 7035 | |
7c673cae FG |
7036 | int r; |
7037 | coll_t cid; | |
7038 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
7039 | ||
7040 | const PerfCounters* logger = store->get_perf_counters(); | |
11fdf7f2 | 7041 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7042 | |
7043 | { | |
7044 | ObjectStore::Transaction t; | |
7045 | t.create_collection(cid, 0); | |
11fdf7f2 | 7046 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7047 | ASSERT_EQ(r, 0); |
7048 | } | |
7049 | { | |
7050 | ObjectStore::Transaction t; | |
7051 | bufferlist bl; | |
7052 | ||
7053 | bl.append(std::string(block_size, 'a')); | |
7054 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
7055 | t.write(cid, hoid, block_size * 2, bl.length(), bl, | |
7056 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7057 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7058 | ASSERT_EQ(r, 0); |
7059 | } | |
7060 | { | |
7061 | // write small into the gap | |
7062 | ObjectStore::Transaction t; | |
7063 | bufferlist bl; | |
7064 | ||
7065 | bl.append(std::string(3, 'b')); | |
7066 | t.write(cid, hoid, block_size + 1, bl.length(), bl, | |
7067 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 7068 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7069 | ASSERT_EQ(r, 0); |
7070 | } | |
7071 | { | |
7072 | // We need to issue a read to trigger cache stat update that refresh | |
7073 | // perf counters. additionally we need to wait some time for mempool | |
7074 | // thread to update stats. | |
7075 | sleep(1); | |
7076 | bufferlist bl, expected; | |
11fdf7f2 | 7077 | r = store->read(ch, hoid, 0, block_size * 3, bl); |
7c673cae FG |
7078 | ASSERT_EQ(r, (int)block_size * 3); |
7079 | expected.append(string(block_size, 'a')); | |
7080 | expected.append(string(1, 0)); | |
7081 | expected.append(string(3, 'b')); | |
7082 | expected.append(string(block_size - 4, 0)); | |
7083 | expected.append(string(block_size, 'a')); | |
7084 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7085 | ||
7086 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
7087 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
7088 | } | |
7089 | { | |
7090 | ObjectStore::Transaction t; | |
7091 | t.remove(cid, hoid); | |
7092 | t.remove_collection(cid); | |
7093 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7094 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7095 | ASSERT_EQ(r, 0); |
7096 | } | |
7c673cae FG |
7097 | } |
7098 | ||
7099 | // The test case to reproduce an issue when write happens | |
7100 | // to a zero space between the extents sharing the same spanning blob | |
7101 | // with unloaded shard map. | |
7102 | // Second extent might be filled with zeros this way due to wrong result | |
7103 | // returned by has_any_extents() call in do_write_small. The latter is caused | |
7104 | // by incompletly loaded extent map. | |
7105 | TEST_P(StoreTestSpecificAUSize, SmallWriteOnShardedExtents) { | |
7106 | if (string(GetParam()) != "bluestore") | |
7107 | return; | |
7108 | ||
7109 | size_t block_size = 0x10000; | |
7110 | StartDeferred(block_size); | |
7111 | ||
11fdf7f2 TL |
7112 | SetVal(g_conf(), "bluestore_csum_type", "xxhash64"); |
7113 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); // for sure | |
7c673cae | 7114 | |
11fdf7f2 | 7115 | g_conf().apply_changes(nullptr); |
7c673cae | 7116 | |
7c673cae FG |
7117 | int r; |
7118 | coll_t cid; | |
7119 | ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 7120 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7121 | |
7122 | { | |
7123 | ObjectStore::Transaction t; | |
7124 | t.create_collection(cid, 0); | |
11fdf7f2 | 7125 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7126 | ASSERT_EQ(r, 0); |
7127 | } | |
7128 | { | |
7129 | //doing some tricks to have sharded extents/spanning objects | |
7130 | ObjectStore::Transaction t; | |
7131 | bufferlist bl, bl2; | |
7132 | ||
7133 | bl.append(std::string(0x80000, 'a')); | |
7134 | t.write(cid, hoid1, 0, bl.length(), bl, 0); | |
7135 | t.zero(cid, hoid1, 0x719e0, 0x75b0 ); | |
11fdf7f2 | 7136 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7137 | ASSERT_EQ(r, 0); |
7138 | ||
7139 | bl2.append(std::string(0x70000, 'b')); | |
7140 | t.write(cid, hoid1, 0, bl2.length(), bl2, 0); | |
7141 | t.zero(cid, hoid1, 0, 0x50000); | |
11fdf7f2 | 7142 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 7143 | ASSERT_EQ(r, 0); |
7c673cae FG |
7144 | |
7145 | } | |
11fdf7f2 | 7146 | ch.reset(); |
7c673cae FG |
7147 | store->umount(); |
7148 | store->mount(); | |
11fdf7f2 | 7149 | ch = store->open_collection(cid); |
7c673cae FG |
7150 | |
7151 | { | |
7152 | // do a write to zero space in between some extents sharing the same blob | |
7153 | ObjectStore::Transaction t; | |
7154 | bufferlist bl, bl2; | |
7155 | ||
7156 | bl.append(std::string(0x6520, 'c')); | |
7157 | t.write(cid, hoid1, 0x71c00, bl.length(), bl, 0); | |
7158 | ||
11fdf7f2 | 7159 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7160 | ASSERT_EQ(r, 0); |
7161 | } | |
7162 | ||
7163 | { | |
7164 | ObjectStore::Transaction t; | |
7165 | bufferlist bl, expected; | |
7166 | ||
11fdf7f2 | 7167 | r = store->read(ch, hoid1, 0x70000, 0x9c00, bl); |
7c673cae FG |
7168 | ASSERT_EQ(r, (int)0x9c00); |
7169 | expected.append(string(0x19e0, 'a')); | |
7170 | expected.append(string(0x220, 0)); | |
7171 | expected.append(string(0x6520, 'c')); | |
7172 | expected.append(string(0xe70, 0)); | |
7173 | expected.append(string(0xc70, 'a')); | |
7174 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7175 | bl.clear(); | |
7176 | ||
7177 | } | |
7178 | ||
7179 | { | |
7180 | ObjectStore::Transaction t; | |
7181 | t.remove(cid, hoid1); | |
7182 | t.remove_collection(cid); | |
7183 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7184 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7185 | ASSERT_EQ(r, 0); |
7186 | } | |
7c673cae FG |
7187 | } |
7188 | ||
11fdf7f2 | 7189 | #endif //#if defined(WITH_BLUESTORE) |
7c673cae FG |
7190 | |
7191 | TEST_P(StoreTest, KVDBHistogramTest) { | |
7192 | if (string(GetParam()) != "bluestore") | |
7193 | return; | |
7194 | ||
7c673cae FG |
7195 | int NUM_OBJS = 200; |
7196 | int r = 0; | |
7197 | coll_t cid; | |
7198 | string base("testobj."); | |
7199 | bufferlist a; | |
7200 | bufferptr ap(0x1000); | |
7201 | memset(ap.c_str(), 'a', 0x1000); | |
7202 | a.append(ap); | |
11fdf7f2 | 7203 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7204 | { |
7205 | ObjectStore::Transaction t; | |
7206 | t.create_collection(cid, 0); | |
11fdf7f2 | 7207 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7208 | ASSERT_EQ(r, 0); |
7209 | } | |
7210 | for (int i = 0; i < NUM_OBJS; ++i) { | |
7211 | ObjectStore::Transaction t; | |
7212 | char buf[100]; | |
7213 | snprintf(buf, sizeof(buf), "%d", i); | |
7214 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
7215 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 7216 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7217 | ASSERT_EQ(r, 0); |
7218 | } | |
7219 | ||
7220 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7221 | store->generate_db_histogram(f); | |
7222 | f->flush(cout); | |
7223 | cout << std::endl; | |
7224 | } | |
7225 | ||
7226 | TEST_P(StoreTest, KVDBStatsTest) { | |
7227 | if (string(GetParam()) != "bluestore") | |
7228 | return; | |
7229 | ||
11fdf7f2 TL |
7230 | SetVal(g_conf(), "rocksdb_perf", "true"); |
7231 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
7232 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
7233 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
7234 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae FG |
7235 | int r = store->umount(); |
7236 | ASSERT_EQ(r, 0); | |
7237 | r = store->mount(); //to force rocksdb stats | |
7238 | ASSERT_EQ(r, 0); | |
7239 | ||
7c673cae FG |
7240 | int NUM_OBJS = 200; |
7241 | coll_t cid; | |
7242 | string base("testobj."); | |
7243 | bufferlist a; | |
7244 | bufferptr ap(0x1000); | |
7245 | memset(ap.c_str(), 'a', 0x1000); | |
7246 | a.append(ap); | |
11fdf7f2 | 7247 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7248 | { |
7249 | ObjectStore::Transaction t; | |
7250 | t.create_collection(cid, 0); | |
11fdf7f2 | 7251 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7252 | ASSERT_EQ(r, 0); |
7253 | } | |
7254 | for (int i = 0; i < NUM_OBJS; ++i) { | |
7255 | ObjectStore::Transaction t; | |
7256 | char buf[100]; | |
7257 | snprintf(buf, sizeof(buf), "%d", i); | |
7258 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
7259 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 7260 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7261 | ASSERT_EQ(r, 0); |
7262 | } | |
7263 | ||
7264 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7265 | store->get_db_statistics(f); | |
7266 | f->flush(cout); | |
7267 | cout << std::endl; | |
7c673cae FG |
7268 | } |
7269 | ||
11fdf7f2 | 7270 | #if defined(WITH_BLUESTORE) |
7c673cae | 7271 | TEST_P(StoreTestSpecificAUSize, garbageCollection) { |
7c673cae FG |
7272 | int r; |
7273 | coll_t cid; | |
7274 | int buf_len = 256 * 1024; | |
7275 | int overlap_offset = 64 * 1024; | |
7276 | int write_offset = buf_len; | |
7277 | if (string(GetParam()) != "bluestore") | |
7278 | return; | |
7279 | ||
7280 | #define WRITE_AT(offset, _length) {\ | |
7281 | ObjectStore::Transaction t;\ | |
7282 | if ((uint64_t)_length != bl.length()) { \ | |
7283 | buffer::ptr p(bl.c_str(), _length);\ | |
7284 | bufferlist bl_tmp;\ | |
7285 | bl_tmp.push_back(p);\ | |
7286 | t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\ | |
7287 | } else {\ | |
7288 | t.write(cid, hoid, offset, bl.length(), bl);\ | |
7289 | }\ | |
11fdf7f2 | 7290 | r = queue_transaction(store, ch, std::move(t));\ |
7c673cae FG |
7291 | ASSERT_EQ(r, 0);\ |
7292 | } | |
7293 | ||
7294 | StartDeferred(65536); | |
7295 | ||
11fdf7f2 TL |
7296 | SetVal(g_conf(), "bluestore_compression_max_blob_size", "524288"); |
7297 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
7298 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
7299 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
7300 | g_conf().apply_changes(nullptr); | |
7301 | ||
7302 | auto ch = store->create_new_collection(cid); | |
7c673cae FG |
7303 | |
7304 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
7305 | { | |
7306 | bufferlist in; | |
11fdf7f2 | 7307 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
7308 | ASSERT_EQ(-ENOENT, r); |
7309 | } | |
7310 | { | |
7311 | ObjectStore::Transaction t; | |
7312 | t.create_collection(cid, 0); | |
7313 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 7314 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7315 | ASSERT_EQ(r, 0); |
7316 | } | |
7317 | ||
7318 | std::string data; | |
7319 | data.resize(buf_len); | |
7320 | ||
7321 | { | |
7322 | { | |
11fdf7f2 | 7323 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
7324 | ASSERT_TRUE(!exists); |
7325 | ||
7326 | ObjectStore::Transaction t; | |
7327 | t.touch(cid, hoid); | |
7328 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 7329 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7330 | ASSERT_EQ(r, 0); |
7331 | ||
11fdf7f2 | 7332 | exists = store->exists(ch, hoid); |
7c673cae FG |
7333 | ASSERT_EQ(true, exists); |
7334 | } | |
7335 | bufferlist bl; | |
7336 | ||
7337 | for(size_t i = 0; i < data.size(); i++) | |
7338 | data[i] = i % 256; | |
7339 | ||
7340 | bl.append(data); | |
7341 | ||
7342 | { | |
7343 | struct store_statfs_t statfs; | |
7344 | WRITE_AT(0, buf_len); | |
7345 | int r = store->statfs(&statfs); | |
7346 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7347 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7348 | } |
7349 | { | |
7350 | struct store_statfs_t statfs; | |
7351 | WRITE_AT(write_offset - 2 * overlap_offset, buf_len); | |
7352 | int r = store->statfs(&statfs); | |
7353 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7354 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7355 | const PerfCounters* counters = store->get_perf_counters(); |
7356 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0u); | |
7357 | } | |
7358 | ||
7359 | { | |
7360 | struct store_statfs_t statfs; | |
7361 | WRITE_AT(write_offset - overlap_offset, buf_len); | |
7362 | int r = store->statfs(&statfs); | |
7363 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7364 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7365 | const PerfCounters* counters = store->get_perf_counters(); |
7366 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x10000u); | |
7367 | } | |
7368 | { | |
7369 | struct store_statfs_t statfs; | |
7370 | WRITE_AT(write_offset - 3 * overlap_offset, buf_len); | |
7371 | int r = store->statfs(&statfs); | |
7372 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7373 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7374 | const PerfCounters* counters = store->get_perf_counters(); |
7375 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
7376 | } | |
7377 | { | |
7378 | struct store_statfs_t statfs; | |
7379 | WRITE_AT(write_offset + 1, overlap_offset-1); | |
7380 | int r = store->statfs(&statfs); | |
7381 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7382 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7383 | const PerfCounters* counters = store->get_perf_counters(); |
7384 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
7385 | } | |
7386 | { | |
7387 | struct store_statfs_t statfs; | |
7388 | WRITE_AT(write_offset + 1, overlap_offset); | |
7389 | int r = store->statfs(&statfs); | |
7390 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7391 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7392 | const PerfCounters* counters = store->get_perf_counters(); |
7393 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x3ffffu); | |
7394 | } | |
7395 | { | |
7396 | struct store_statfs_t statfs; | |
7397 | WRITE_AT(0, buf_len-1); | |
7398 | int r = store->statfs(&statfs); | |
7399 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7400 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7401 | const PerfCounters* counters = store->get_perf_counters(); |
7402 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
7403 | } | |
11fdf7f2 | 7404 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0 |
7c673cae FG |
7405 | { |
7406 | struct store_statfs_t statfs; | |
7407 | WRITE_AT(1, overlap_offset-2); | |
7408 | WRITE_AT(overlap_offset * 2 + 1, overlap_offset-2); | |
7409 | int r = store->statfs(&statfs); | |
7410 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7411 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7412 | const PerfCounters* counters = store->get_perf_counters(); |
7413 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
7414 | } | |
7415 | { | |
7416 | struct store_statfs_t statfs; | |
7417 | WRITE_AT(overlap_offset + 1, overlap_offset-2); | |
7418 | int r = store->statfs(&statfs); | |
7419 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7420 | ASSERT_EQ(statfs.data_compressed_allocated, 0x0); |
7c673cae FG |
7421 | const PerfCounters* counters = store->get_perf_counters(); |
7422 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40007u); | |
7423 | } | |
7424 | { | |
7425 | ObjectStore::Transaction t; | |
7426 | t.remove(cid, hoid); | |
7427 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7428 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7429 | ASSERT_EQ(r, 0); |
7430 | } | |
7431 | } | |
7c673cae | 7432 | } |
7c673cae | 7433 | |
b32b8144 FG |
7434 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice) { |
7435 | if (string(GetParam()) != "bluestore") | |
7436 | return; | |
7437 | ||
11fdf7f2 TL |
7438 | SetVal(g_conf(), "bluestore_block_size", |
7439 | stringify(0x280005000).c_str()); //10 Gb + 4K | |
7440 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7441 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
7442 | StartDeferred(0x4000); |
7443 | store->umount(); | |
7444 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7445 | store->mount(); | |
7446 | ||
b32b8144 FG |
7447 | } |
7448 | ||
7449 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice2) { | |
7450 | if (string(GetParam()) != "bluestore") | |
7451 | return; | |
7452 | ||
11fdf7f2 TL |
7453 | SetVal(g_conf(), "bluestore_block_size", |
7454 | stringify(0x280005000).c_str()); //10 Gb + 20K | |
7455 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7456 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
7457 | StartDeferred(0x1000); |
7458 | store->umount(); | |
7459 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7460 | store->mount(); | |
11fdf7f2 TL |
7461 | } |
7462 | ||
7463 | namespace { | |
7464 | ghobject_t make_object(const char* name, int64_t pool) { | |
7465 | sobject_t soid{name, CEPH_NOSNAP}; | |
7466 | uint32_t hash = std::hash<sobject_t>{}(soid); | |
7467 | return ghobject_t{hobject_t{soid, "", hash, pool, ""}}; | |
7468 | } | |
7469 | } | |
7470 | ||
7471 | TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) { | |
7472 | if (string(GetParam()) != "bluestore") | |
7473 | return; | |
7474 | const size_t offs_base = 65536 / 2; | |
7475 | ||
7476 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7477 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
7478 | SetVal(g_conf(), "bluestore_max_blob_size", | |
7479 | stringify(2 * offs_base).c_str()); | |
7480 | SetVal(g_conf(), "bluestore_extent_map_shard_max_size", "12000"); | |
eafe8130 | 7481 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "false"); |
11fdf7f2 TL |
7482 | |
7483 | StartDeferred(0x10000); | |
7484 | ||
7485 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7486 | ||
7487 | // fill the store with some data | |
7488 | const uint64_t pool = 555; | |
7489 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7490 | auto ch = store->create_new_collection(cid); | |
7491 | ||
7492 | ghobject_t hoid = make_object("Object 1", pool); | |
7493 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7494 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7495 | ghobject_t hoid_cloned = hoid2; | |
7496 | hoid_cloned.hobj.snap = 1; | |
7497 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7498 | ghobject_t hoid3_cloned = hoid3; | |
7499 | hoid3_cloned.hobj.snap = 1; | |
7500 | bufferlist bl; | |
7501 | bl.append("1234512345"); | |
7502 | int r; | |
7503 | const size_t repeats = 16; | |
7504 | { | |
7505 | auto ch = store->create_new_collection(cid); | |
7506 | cerr << "create collection + write" << std::endl; | |
7507 | ObjectStore::Transaction t; | |
7508 | t.create_collection(cid, 0); | |
7509 | for( auto i = 0ul; i < repeats; ++i ) { | |
7510 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7511 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7512 | } | |
7513 | for( auto i = 0ul; i < repeats; ++i ) { | |
7514 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7515 | } | |
7516 | t.clone(cid, hoid2, hoid_cloned); | |
7517 | ||
7518 | r = queue_transaction(store, ch, std::move(t)); | |
7519 | ASSERT_EQ(r, 0); | |
7520 | } | |
7521 | ||
7522 | bstore->umount(); | |
7523 | //////////// leaked pextent fix //////////// | |
7524 | cerr << "fix leaked pextents" << std::endl; | |
7525 | ASSERT_EQ(bstore->fsck(false), 0); | |
7526 | ASSERT_EQ(bstore->repair(false), 0); | |
7527 | bstore->mount(); | |
7528 | bstore->inject_leaked(0x30000); | |
7529 | bstore->umount(); | |
7530 | ASSERT_EQ(bstore->fsck(false), 1); | |
7531 | ASSERT_EQ(bstore->repair(false), 0); | |
7532 | ASSERT_EQ(bstore->fsck(false), 0); | |
7533 | ||
7534 | //////////// false free fix //////////// | |
7535 | cerr << "fix false free pextents" << std::endl; | |
7536 | bstore->mount(); | |
7537 | bstore->inject_false_free(cid, hoid); | |
7538 | bstore->umount(); | |
7539 | ASSERT_EQ(bstore->fsck(false), 2); | |
7540 | ASSERT_EQ(bstore->repair(false), 0); | |
7541 | ASSERT_EQ(bstore->fsck(false), 0); | |
7542 | ||
7543 | //////////// verify invalid statfs /////////// | |
7544 | cerr << "fix invalid statfs" << std::endl; | |
7545 | store_statfs_t statfs0, statfs; | |
7546 | bstore->mount(); | |
7547 | ASSERT_EQ(bstore->statfs(&statfs0), 0); | |
7548 | statfs = statfs0; | |
7549 | statfs.allocated += 0x10000; | |
7550 | statfs.data_stored += 0x10000; | |
7551 | ASSERT_FALSE(statfs0 == statfs); | |
7552 | bstore->inject_statfs("bluestore_statfs", statfs); | |
7553 | bstore->umount(); | |
7554 | ||
eafe8130 | 7555 | ASSERT_EQ(bstore->fsck(false), 2); |
11fdf7f2 TL |
7556 | ASSERT_EQ(bstore->repair(false), 0); |
7557 | ASSERT_EQ(bstore->fsck(false), 0); | |
7558 | ASSERT_EQ(bstore->mount(), 0); | |
7559 | ASSERT_EQ(bstore->statfs(&statfs), 0); | |
7560 | // adjust free space to success in comparison | |
7561 | statfs0.available = statfs.available; | |
7562 | ASSERT_EQ(statfs0, statfs); | |
7563 | ||
7564 | ///////// undecodable shared blob key / stray shared blob records /////// | |
7565 | cerr << "undecodable shared blob key" << std::endl; | |
7566 | bstore->inject_broken_shared_blob_key("undec1", | |
7567 | bufferlist()); | |
7568 | bstore->inject_broken_shared_blob_key("undecodable key 2", | |
7569 | bufferlist()); | |
7570 | bstore->inject_broken_shared_blob_key("undecodable key 3", | |
7571 | bufferlist()); | |
7572 | bstore->umount(); | |
7573 | ASSERT_EQ(bstore->fsck(false), 3); | |
7574 | ASSERT_EQ(bstore->repair(false), 0); | |
7575 | ASSERT_EQ(bstore->fsck(false), 0); | |
7576 | ||
7577 | cerr << "misreferencing" << std::endl; | |
7578 | bstore->mount(); | |
7579 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, 0); | |
7580 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, (offs_base * repeats) / 2); | |
7581 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, offs_base * (repeats -1) ); | |
7582 | ||
7583 | bstore->umount(); | |
7584 | ASSERT_EQ(bstore->fsck(false), 6); | |
7585 | ASSERT_EQ(bstore->repair(false), 0); | |
7586 | ||
7587 | ASSERT_EQ(bstore->fsck(true), 0); | |
7588 | ||
7589 | // reproducing issues #21040 & 20983 | |
7590 | SetVal(g_conf(), "bluestore_debug_inject_bug21040", "true"); | |
7591 | g_ceph_context->_conf.apply_changes(nullptr); | |
7592 | bstore->mount(); | |
7593 | ||
7594 | cerr << "repro bug #21040" << std::endl; | |
7595 | { | |
7596 | auto ch = store->open_collection(cid); | |
7597 | { | |
7598 | ObjectStore::Transaction t; | |
7599 | bl.append("0123456789012345"); | |
7600 | t.write(cid, hoid3, offs_base, bl.length(), bl); | |
7601 | bl.clear(); | |
7602 | bl.append('!'); | |
7603 | t.write(cid, hoid3, 0, bl.length(), bl); | |
7604 | ||
7605 | r = queue_transaction(store, ch, std::move(t)); | |
7606 | ASSERT_EQ(r, 0); | |
7607 | } | |
7608 | { | |
7609 | ObjectStore::Transaction t; | |
7610 | t.clone(cid, hoid3, hoid3_cloned); | |
7611 | r = queue_transaction(store, ch, std::move(t)); | |
7612 | ASSERT_EQ(r, 0); | |
7613 | } | |
7614 | ||
7615 | bstore->umount(); | |
7616 | ASSERT_EQ(bstore->fsck(false), 3); | |
7617 | ASSERT_LE(bstore->repair(false), 0); | |
7618 | ASSERT_EQ(bstore->fsck(false), 0); | |
7619 | } | |
7620 | ||
adb31ebb TL |
7621 | cerr << "Zombie spanning blob" << std::endl; |
7622 | { | |
7623 | bstore->mount(); | |
7624 | ghobject_t hoid4 = make_object("Object 4", pool); | |
7625 | auto ch = store->open_collection(cid); | |
7626 | { | |
7627 | bufferlist bl; | |
7628 | string s(0x1000, 'a'); | |
7629 | bl.append(s); | |
7630 | ObjectStore::Transaction t; | |
7631 | for(size_t i = 0; i < 0x10; i++) { | |
7632 | t.write(cid, hoid4, i * bl.length(), bl.length(), bl); | |
7633 | } | |
7634 | r = queue_transaction(store, ch, std::move(t)); | |
7635 | ASSERT_EQ(r, 0); | |
7636 | } | |
7637 | sleep(5); | |
7638 | { | |
7639 | bstore->inject_zombie_spanning_blob(cid, hoid4, 12345); | |
7640 | bstore->inject_zombie_spanning_blob(cid, hoid4, 23456); | |
7641 | bstore->inject_zombie_spanning_blob(cid, hoid4, 23457); | |
7642 | } | |
7643 | ||
7644 | bstore->umount(); | |
7645 | ASSERT_EQ(bstore->fsck(false), 1); | |
7646 | ASSERT_LE(bstore->repair(false), 0); | |
7647 | ASSERT_EQ(bstore->fsck(false), 0); | |
7648 | } | |
7649 | ||
eafe8130 TL |
7650 | cerr << "Completing" << std::endl; |
7651 | bstore->mount(); | |
7652 | ||
7653 | } | |
7654 | ||
ec96510d FG |
7655 | TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) { |
7656 | if (string(GetParam()) != "bluestore") | |
7657 | return; | |
7658 | ||
7659 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7660 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
7661 | ||
7662 | StartDeferred(0x10000); | |
7663 | ||
7664 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7665 | ||
7666 | int r; | |
7667 | ||
7668 | cerr << "initializing" << std::endl; | |
7669 | { | |
7670 | const size_t col_count = 16; | |
7671 | const size_t obj_count = 1024; | |
7672 | ObjectStore::CollectionHandle ch[col_count]; | |
7673 | ghobject_t hoid[col_count][obj_count]; | |
7674 | ||
7675 | unique_ptr<coll_t> cid[col_count]; | |
7676 | ||
7677 | for (size_t i = 0; i < col_count; i++) { | |
7678 | cid[i].reset(new coll_t(spg_t(pg_t(0, i), shard_id_t::NO_SHARD))); | |
7679 | ch[i] = store->create_new_collection(*cid[i]); | |
7680 | for (size_t j = 0; j < obj_count; j++) { | |
7681 | hoid[i][j] = make_object(stringify(j).c_str(), i); | |
7682 | } | |
7683 | } | |
7684 | ||
7685 | for (size_t i = 0; i < col_count; i++) { | |
7686 | ObjectStore::Transaction t; | |
7687 | t.create_collection(*cid[i], 0); | |
7688 | r = queue_transaction(store, ch[i], std::move(t)); | |
7689 | ASSERT_EQ(r, 0); | |
7690 | } | |
7691 | cerr << "onode preparing" << std::endl; | |
7692 | bufferlist bl; | |
7693 | string s(0x1000, 'a'); | |
7694 | bl.append(s); | |
7695 | ||
7696 | for (size_t i = 0; i < col_count; i++) { | |
7697 | for (size_t j = 0; j < obj_count; j++) { | |
7698 | ObjectStore::Transaction t; | |
7699 | t.write(*cid[i], hoid[i][j], bl.length(), bl.length(), bl); | |
7700 | r = queue_transaction(store, ch[i], std::move(t)); | |
7701 | ASSERT_EQ(r, 0); | |
7702 | } | |
7703 | } | |
7704 | cerr << "Zombie spanning blob injection" << std::endl; | |
7705 | ||
7706 | sleep(5); | |
7707 | ||
7708 | for (size_t i = 0; i < col_count; i++) { | |
7709 | for (size_t j = 0; j < obj_count; j++) { | |
7710 | bstore->inject_zombie_spanning_blob(*cid[i], hoid[i][j], 12345); | |
7711 | } | |
7712 | } | |
7713 | ||
7714 | cerr << "fscking/fixing" << std::endl; | |
7715 | bstore->umount(); | |
7716 | ASSERT_EQ(bstore->fsck(false), col_count * obj_count); | |
7717 | ASSERT_LE(bstore->quick_fix(), 0); | |
7718 | ASSERT_EQ(bstore->fsck(false), 0); | |
7719 | } | |
7720 | ||
7721 | cerr << "Completing" << std::endl; | |
7722 | bstore->mount(); | |
7723 | } | |
7724 | ||
adb31ebb | 7725 | TEST_P(StoreTest, BluestoreRepairGlobalStats) { |
eafe8130 TL |
7726 | if (string(GetParam()) != "bluestore") |
7727 | return; | |
7728 | const size_t offs_base = 65536 / 2; | |
7729 | ||
7730 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7731 | ||
7732 | // start with global stats | |
7733 | bstore->inject_global_statfs({}); | |
7734 | bstore->umount(); | |
7735 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7736 | bstore->mount(); | |
7737 | ||
7738 | // fill the store with some data | |
7739 | const uint64_t pool = 555; | |
7740 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7741 | auto ch = store->create_new_collection(cid); | |
7742 | ||
7743 | ghobject_t hoid = make_object("Object 1", pool); | |
7744 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7745 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7746 | ghobject_t hoid_cloned = hoid2; | |
7747 | hoid_cloned.hobj.snap = 1; | |
7748 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7749 | ghobject_t hoid3_cloned = hoid3; | |
7750 | hoid3_cloned.hobj.snap = 1; | |
7751 | bufferlist bl; | |
7752 | bl.append("1234512345"); | |
7753 | int r; | |
7754 | const size_t repeats = 16; | |
7755 | { | |
7756 | auto ch = store->create_new_collection(cid); | |
7757 | cerr << "create collection + write" << std::endl; | |
7758 | ObjectStore::Transaction t; | |
7759 | t.create_collection(cid, 0); | |
7760 | for( auto i = 0ul; i < repeats; ++i ) { | |
7761 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7762 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7763 | } | |
7764 | for( auto i = 0ul; i < repeats; ++i ) { | |
7765 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7766 | } | |
7767 | t.clone(cid, hoid2, hoid_cloned); | |
7768 | ||
7769 | r = queue_transaction(store, ch, std::move(t)); | |
7770 | ASSERT_EQ(r, 0); | |
7771 | } | |
7772 | ||
7773 | bstore->umount(); | |
7774 | ||
11fdf7f2 TL |
7775 | // enable per-pool stats collection hence causing fsck to fail |
7776 | cerr << "per-pool statfs" << std::endl; | |
eafe8130 | 7777 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); |
11fdf7f2 TL |
7778 | g_ceph_context->_conf.apply_changes(nullptr); |
7779 | ||
eafe8130 | 7780 | ASSERT_EQ(bstore->fsck(false), 1); |
11fdf7f2 TL |
7781 | ASSERT_EQ(bstore->repair(false), 0); |
7782 | ASSERT_EQ(bstore->fsck(false), 0); | |
7783 | ||
11fdf7f2 | 7784 | bstore->mount(); |
eafe8130 TL |
7785 | } |
7786 | ||
adb31ebb | 7787 | TEST_P(StoreTest, BluestoreRepairGlobalStatsFixOnMount) { |
eafe8130 TL |
7788 | if (string(GetParam()) != "bluestore") |
7789 | return; | |
7790 | const size_t offs_base = 65536 / 2; | |
7791 | ||
7792 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7793 | ||
7794 | // start with global stats | |
7795 | bstore->inject_global_statfs({}); | |
7796 | bstore->umount(); | |
7797 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7798 | bstore->mount(); | |
7799 | ||
7800 | // fill the store with some data | |
7801 | const uint64_t pool = 555; | |
7802 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7803 | auto ch = store->create_new_collection(cid); | |
7804 | ||
7805 | ghobject_t hoid = make_object("Object 1", pool); | |
7806 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7807 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7808 | ghobject_t hoid_cloned = hoid2; | |
7809 | hoid_cloned.hobj.snap = 1; | |
7810 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7811 | ghobject_t hoid3_cloned = hoid3; | |
7812 | hoid3_cloned.hobj.snap = 1; | |
7813 | bufferlist bl; | |
7814 | bl.append("1234512345"); | |
7815 | int r; | |
7816 | const size_t repeats = 16; | |
7817 | { | |
7818 | auto ch = store->create_new_collection(cid); | |
7819 | cerr << "create collection + write" << std::endl; | |
7820 | ObjectStore::Transaction t; | |
7821 | t.create_collection(cid, 0); | |
7822 | for( auto i = 0ul; i < repeats; ++i ) { | |
7823 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7824 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7825 | } | |
7826 | for( auto i = 0ul; i < repeats; ++i ) { | |
7827 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7828 | } | |
7829 | t.clone(cid, hoid2, hoid_cloned); | |
7830 | ||
7831 | r = queue_transaction(store, ch, std::move(t)); | |
7832 | ASSERT_EQ(r, 0); | |
7833 | } | |
7834 | ||
7835 | bstore->umount(); | |
7836 | ||
7837 | // enable per-pool stats collection hence causing fsck to fail | |
7838 | cerr << "per-pool statfs" << std::endl; | |
7839 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); | |
7840 | g_ceph_context->_conf.apply_changes(nullptr); | |
7841 | ||
7842 | ASSERT_EQ(bstore->fsck(false), 1); | |
7843 | ||
7844 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7845 | bstore->mount(); | |
7846 | bstore->umount(); | |
7847 | ASSERT_EQ(bstore->fsck(false), 0); | |
b32b8144 | 7848 | |
eafe8130 | 7849 | bstore->mount(); |
11fdf7f2 TL |
7850 | } |
7851 | ||
7852 | TEST_P(StoreTest, BluestoreStatistics) { | |
7853 | if (string(GetParam()) != "bluestore") | |
7854 | return; | |
7855 | ||
7856 | SetVal(g_conf(), "rocksdb_perf", "true"); | |
7857 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
7858 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
7859 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
7860 | ||
7861 | // disable cache | |
7862 | SetVal(g_conf(), "bluestore_cache_size_ssd", "0"); | |
7863 | SetVal(g_conf(), "bluestore_cache_size_hdd", "0"); | |
7864 | SetVal(g_conf(), "bluestore_cache_size", "0"); | |
7865 | g_ceph_context->_conf.apply_changes(nullptr); | |
7866 | ||
7867 | int r = store->umount(); | |
7868 | ASSERT_EQ(r, 0); | |
7869 | r = store->mount(); | |
7870 | ASSERT_EQ(r, 0); | |
7871 | ||
7872 | BlueStore* bstore = NULL; | |
7873 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
7874 | ||
7875 | coll_t cid; | |
7876 | ghobject_t hoid(hobject_t("test_db_statistics", "", CEPH_NOSNAP, 0, 0, "")); | |
7877 | auto ch = bstore->create_new_collection(cid); | |
7878 | bufferlist bl; | |
7879 | bl.append("0123456789abcdefghi"); | |
7880 | { | |
7881 | ObjectStore::Transaction t; | |
7882 | t.create_collection(cid, 0); | |
7883 | t.touch(cid, hoid); | |
7884 | t.write(cid, hoid, 0, bl.length(), bl); | |
7885 | cerr << "Write object" << std::endl; | |
7886 | r = queue_transaction(bstore, ch, std::move(t)); | |
7887 | ASSERT_EQ(r, 0); | |
7888 | } | |
7889 | { | |
7890 | bufferlist readback; | |
7891 | r = store->read(ch, hoid, 0, bl.length(), readback); | |
7892 | ASSERT_EQ(static_cast<int>(bl.length()), r); | |
7893 | ASSERT_TRUE(bl_eq(bl, readback)); | |
7894 | } | |
7895 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7896 | EXPECT_NO_THROW(store->get_db_statistics(f)); | |
7897 | f->flush(cout); | |
7898 | cout << std::endl; | |
7899 | } | |
7900 | ||
9f95a23c TL |
7901 | TEST_P(StoreTest, BluestorePerPoolOmapFixOnMount) |
7902 | { | |
7903 | if (string(GetParam()) != "bluestore") | |
7904 | return; | |
7905 | ||
7906 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7907 | const uint64_t pool = 555; | |
7908 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7909 | ghobject_t oid = make_object("Object 1", pool); | |
7910 | ghobject_t oid2 = make_object("Object 2", pool); | |
7911 | // fill the store with some data | |
7912 | auto ch = store->create_new_collection(cid); | |
7913 | map<string, bufferlist> omap; | |
7914 | bufferlist h; | |
7915 | h.append("header"); | |
7916 | { | |
7917 | omap["omap_key"].append("omap value"); | |
7918 | ObjectStore::Transaction t; | |
7919 | t.create_collection(cid, 0); | |
7920 | t.touch(cid, oid); | |
7921 | t.omap_setheader(cid, oid, h); | |
7922 | t.touch(cid, oid2); | |
7923 | t.omap_setheader(cid, oid2, h); | |
7924 | int r = queue_transaction(store, ch, std::move(t)); | |
7925 | ASSERT_EQ(r, 0); | |
7926 | } | |
7927 | ||
7928 | // inject legacy omaps | |
7929 | bstore->inject_legacy_omap(); | |
7930 | bstore->inject_legacy_omap(cid, oid); | |
7931 | bstore->inject_legacy_omap(cid, oid2); | |
7932 | ||
7933 | bstore->umount(); | |
7934 | ||
7935 | // check we injected an issue | |
7936 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7937 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
7938 | g_ceph_context->_conf.apply_changes(nullptr); | |
7939 | ASSERT_EQ(bstore->fsck(false), 3); | |
7940 | ||
7941 | // set autofix and mount | |
7942 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7943 | g_ceph_context->_conf.apply_changes(nullptr); | |
7944 | bstore->mount(); | |
7945 | bstore->umount(); | |
7946 | ||
7947 | // check we fixed it.. | |
7948 | ASSERT_EQ(bstore->fsck(false), 0); | |
7949 | bstore->mount(); | |
7950 | ||
7951 | // | |
7952 | // Now repro https://tracker.ceph.com/issues/43824 | |
7953 | // | |
7954 | // inject legacy omaps again | |
7955 | bstore->inject_legacy_omap(); | |
7956 | bstore->inject_legacy_omap(cid, oid); | |
7957 | bstore->inject_legacy_omap(cid, oid2); | |
7958 | bstore->umount(); | |
7959 | ||
7960 | // check we injected an issue | |
7961 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7962 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
7963 | g_ceph_context->_conf.apply_changes(nullptr); | |
7964 | bstore->mount(); | |
7965 | ch = store->open_collection(cid); | |
7966 | ||
7967 | { | |
7968 | // write to onode which will partiall revert per-pool | |
7969 | // omap repair done on mount due to #43824. | |
7970 | // And object removal will leave stray per-pool omap recs | |
7971 | // | |
7972 | ObjectStore::Transaction t; | |
7973 | bufferlist bl; | |
7974 | bl.append("data"); | |
7975 | //this triggers onode rec update and hence legacy omap | |
7976 | t.write(cid, oid, 0, bl.length(), bl); | |
7977 | t.remove(cid, oid2); // this will trigger stray per-pool omap | |
7978 | int r = queue_transaction(store, ch, std::move(t)); | |
7979 | ASSERT_EQ(r, 0); | |
7980 | } | |
7981 | bstore->umount(); | |
7982 | // check omap's been fixed. | |
7983 | ASSERT_EQ(bstore->fsck(false), 0); // this will fail without fix for #43824 | |
7984 | ||
7985 | bstore->mount(); | |
7986 | } | |
7987 | ||
11fdf7f2 TL |
7988 | TEST_P(StoreTestSpecificAUSize, BluestoreTinyDevFailure) { |
7989 | if (string(GetParam()) != "bluestore") | |
7990 | return; | |
7991 | // This caused superblock overwrite by bluefs, see | |
7992 | // https://tracker.ceph.com/issues/24480 | |
7993 | SetVal(g_conf(), "bluestore_block_size", | |
7994 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
7995 | SetVal(g_conf(), "bluestore_block_db_size", "0"); | |
7996 | SetVal(g_conf(), "bluestore_block_db_create", "false"); | |
7997 | SetVal(g_conf(), "bluestore_bluefs_min", | |
7998 | stringify(1024 * 1024 * 1024).c_str()); | |
7999 | StartDeferred(0x1000); | |
8000 | store->umount(); | |
8001 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
8002 | store->mount(); | |
8003 | } | |
8004 | ||
8005 | TEST_P(StoreTestSpecificAUSize, BluestoreTinyDevFailure2) { | |
8006 | if (string(GetParam()) != "bluestore") | |
8007 | return; | |
8008 | ||
8009 | // This caused assert in allocator as initial bluefs extent as slow device | |
8010 | // overlaped with superblock | |
8011 | // https://tracker.ceph.com/issues/24480 | |
8012 | SetVal(g_conf(), "bluestore_block_size", | |
8013 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
8014 | SetVal(g_conf(), "bluestore_block_db_size", | |
8015 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
8016 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8017 | SetVal(g_conf(), "bluestore_bluefs_min", | |
8018 | stringify(1024 * 1024 * 1024).c_str()); | |
8019 | StartDeferred(0x1000); | |
8020 | store->umount(); | |
8021 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
8022 | store->mount(); | |
b32b8144 FG |
8023 | } |
8024 | ||
f64942e4 AA |
8025 | TEST_P(StoreTest, SpuriousReadErrorTest) { |
8026 | if (string(GetParam()) != "bluestore") | |
8027 | return; | |
8028 | ||
f64942e4 AA |
8029 | int r; |
8030 | auto logger = store->get_perf_counters(); | |
8031 | coll_t cid; | |
11fdf7f2 | 8032 | auto ch = store->create_new_collection(cid); |
f64942e4 AA |
8033 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
8034 | { | |
8035 | ObjectStore::Transaction t; | |
8036 | t.create_collection(cid, 0); | |
8037 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 8038 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
8039 | ASSERT_EQ(r, 0); |
8040 | } | |
8041 | bufferlist test_data; | |
8042 | bufferptr ap(0x2000); | |
8043 | memset(ap.c_str(), 'a', 0x2000); | |
8044 | test_data.append(ap); | |
8045 | { | |
8046 | ObjectStore::Transaction t; | |
8047 | t.write(cid, hoid, 0, 0x2000, test_data); | |
11fdf7f2 | 8048 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
8049 | ASSERT_EQ(r, 0); |
8050 | // force cache clear | |
8051 | EXPECT_EQ(store->umount(), 0); | |
8052 | EXPECT_EQ(store->mount(), 0); | |
8053 | } | |
f6b5b4d7 | 8054 | ch = store->open_collection(cid); |
f64942e4 AA |
8055 | |
8056 | cerr << "Injecting CRC error with no retry, expecting EIO" << std::endl; | |
11fdf7f2 TL |
8057 | SetVal(g_conf(), "bluestore_retry_disk_reads", "0"); |
8058 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "1"); | |
8059 | g_ceph_context->_conf.apply_changes(nullptr); | |
f64942e4 AA |
8060 | { |
8061 | bufferlist in; | |
11fdf7f2 | 8062 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
8063 | ASSERT_EQ(-EIO, r); |
8064 | ASSERT_EQ(logger->get(l_bluestore_read_eio), 1u); | |
8065 | ASSERT_EQ(logger->get(l_bluestore_reads_with_retries), 0u); | |
8066 | } | |
8067 | ||
8068 | cerr << "Injecting CRC error with retries, expecting success after several retries" << std::endl; | |
11fdf7f2 TL |
8069 | SetVal(g_conf(), "bluestore_retry_disk_reads", "255"); |
8070 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "0.8"); | |
f64942e4 AA |
8071 | /** |
8072 | * Probabilistic test: 25 reads, each has a 80% chance of failing with 255 retries | |
8073 | * Probability of at least one retried read: 1 - (0.2 ** 25) = 100% - 3e-18 | |
8074 | * Probability of a random test failure: 1 - ((1 - (0.8 ** 255)) ** 25) ~= 5e-24 | |
8075 | */ | |
11fdf7f2 | 8076 | g_ceph_context->_conf.apply_changes(nullptr); |
f64942e4 AA |
8077 | { |
8078 | for (int i = 0; i < 25; ++i) { | |
8079 | bufferlist in; | |
11fdf7f2 | 8080 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
8081 | ASSERT_EQ(0x2000, r); |
8082 | ASSERT_TRUE(bl_eq(test_data, in)); | |
8083 | } | |
8084 | ASSERT_GE(logger->get(l_bluestore_reads_with_retries), 1u); | |
8085 | } | |
11fdf7f2 TL |
8086 | } |
8087 | ||
8088 | TEST_P(StoreTest, allocateBlueFSTest) { | |
8089 | if (string(GetParam()) != "bluestore") | |
8090 | return; | |
8091 | ||
8092 | BlueStore* bstore = NULL; | |
8093 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
8094 | ||
8095 | struct store_statfs_t statfs; | |
8096 | store->statfs(&statfs); | |
8097 | ||
8098 | uint64_t to_alloc = g_conf().get_val<Option::size_t>("bluefs_alloc_size"); | |
8099 | ||
8100 | int r = bstore->allocate_bluefs_freespace(to_alloc, to_alloc, nullptr); | |
8101 | ASSERT_EQ(r, 0); | |
8102 | r = bstore->allocate_bluefs_freespace(statfs.total, statfs.total, nullptr); | |
8103 | ASSERT_EQ(r, -ENOSPC); | |
8104 | r = bstore->allocate_bluefs_freespace(to_alloc * 16, to_alloc * 16, nullptr); | |
8105 | ASSERT_EQ(r, 0); | |
8106 | store->umount(); | |
8107 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
8108 | r = store->mount(); | |
8109 | ASSERT_EQ(r, 0); | |
8110 | } | |
8111 | ||
8112 | TEST_P(StoreTest, mergeRegionTest) { | |
8113 | if (string(GetParam()) != "bluestore") | |
8114 | return; | |
8115 | ||
8116 | SetVal(g_conf(), "bluestore_fsck_on_mount", "true"); | |
8117 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
8118 | SetVal(g_conf(), "bdev_debug_inflight_ios", "true"); | |
8119 | g_ceph_context->_conf.apply_changes(nullptr); | |
8120 | ||
8121 | uint32_t chunk_size = g_ceph_context->_conf->bdev_block_size; | |
8122 | int r = -1; | |
8123 | coll_t cid; | |
8124 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
8125 | auto ch = store->create_new_collection(cid); | |
8126 | { | |
8127 | ObjectStore::Transaction t; | |
8128 | t.create_collection(cid, 0); | |
8129 | r = queue_transaction(store, ch, std::move(t)); | |
8130 | ASSERT_EQ(r, 0); | |
8131 | } | |
8132 | { | |
8133 | ObjectStore::Transaction t; | |
8134 | t.touch(cid, hoid); | |
8135 | cerr << "Creating object " << hoid << std::endl; | |
8136 | r = queue_transaction(store, ch, std::move(t)); | |
8137 | ASSERT_EQ(r, 0); | |
8138 | } | |
8139 | bufferlist bl5; | |
8140 | bl5.append("abcde"); | |
8141 | uint64_t offset = 0; | |
8142 | { // 1. same region | |
8143 | ObjectStore::Transaction t; | |
8144 | t.write(cid, hoid, offset, 5, bl5); | |
8145 | t.write(cid, hoid, 0xa + offset, 5, bl5); | |
8146 | t.write(cid, hoid, 0x14 + offset, 5, bl5); | |
8147 | r = queue_transaction(store, ch, std::move(t)); | |
8148 | ASSERT_EQ(r, 0); | |
8149 | } | |
8150 | { // 2. adjacent regions | |
8151 | ObjectStore::Transaction t; | |
8152 | offset = chunk_size; | |
8153 | t.write(cid, hoid, offset, 5, bl5); | |
8154 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
8155 | r = queue_transaction(store, ch, std::move(t)); | |
8156 | ASSERT_EQ(r, 0); | |
8157 | } | |
8158 | { // 3. front merge | |
8159 | ObjectStore::Transaction t; | |
8160 | offset = chunk_size * 2; | |
8161 | t.write(cid, hoid, offset, 5, bl5); | |
8162 | t.write(cid, hoid, offset + chunk_size - 2, 5, bl5); | |
8163 | r = queue_transaction(store, ch, std::move(t)); | |
8164 | ASSERT_EQ(r, 0); | |
8165 | } | |
8166 | { // 4. back merge | |
8167 | ObjectStore::Transaction t; | |
8168 | bufferlist blc2; | |
8169 | blc2.append_zero(chunk_size + 2); | |
f64942e4 | 8170 | |
11fdf7f2 TL |
8171 | offset = chunk_size * 3; |
8172 | t.write(cid, hoid, offset, chunk_size + 2, blc2); | |
8173 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
8174 | r = queue_transaction(store, ch, std::move(t)); | |
8175 | ASSERT_EQ(r, 0); | |
8176 | } | |
8177 | { // 5. overlapping | |
8178 | ObjectStore::Transaction t; | |
8179 | uint64_t final_len = 0; | |
8180 | offset = chunk_size * 10; | |
8181 | bufferlist bl2c2; | |
8182 | bl2c2.append_zero(chunk_size * 2); | |
8183 | t.write(cid, hoid, offset + chunk_size * 3 - 3, chunk_size * 2, bl2c2); | |
8184 | bl2c2.append_zero(2); | |
8185 | t.write(cid, hoid, offset + chunk_size - 2, chunk_size * 2 + 2, bl2c2); | |
8186 | r = queue_transaction(store, ch, std::move(t)); | |
8187 | ASSERT_EQ(r, 0); | |
8188 | ||
8189 | final_len = (offset + chunk_size * 3 - 3) + (chunk_size * 2); | |
8190 | bufferlist bl; | |
8191 | r = store->read(ch, hoid, 0, final_len, bl); | |
8192 | ASSERT_EQ(final_len, static_cast<uint64_t>(r)); | |
8193 | } | |
f64942e4 | 8194 | } |
9f95a23c TL |
8195 | |
8196 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { | |
8197 | if (string(GetParam()) != "bluestore") | |
8198 | return; | |
8199 | ||
8200 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
8201 | StartDeferred(0x1000); | |
8202 | ||
8203 | int r; | |
8204 | coll_t cid; | |
8205 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
8206 | auto ch = store->create_new_collection(cid); | |
8207 | { | |
8208 | ObjectStore::Transaction t; | |
8209 | t.create_collection(cid, 0); | |
8210 | cerr << "Creating collection " << cid << std::endl; | |
8211 | r = queue_transaction(store, ch, std::move(t)); | |
8212 | ASSERT_EQ(r, 0); | |
8213 | } | |
8214 | { | |
8215 | ObjectStore::Transaction t; | |
8216 | bufferlist bl, orig; | |
8217 | string s(g_ceph_context->_conf->bluestore_max_blob_size_hdd, '0'); | |
8218 | bl.append(s); | |
8219 | t.write(cid, hoid, 0, bl.length(), bl); | |
8220 | cerr << "write" << std::endl; | |
8221 | r = queue_transaction(store, ch, std::move(t)); | |
8222 | ASSERT_EQ(r, 0); | |
8223 | ||
8224 | const PerfCounters* logger = store->get_perf_counters(); | |
8225 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
8226 | } | |
8227 | } | |
8228 | ||
8229 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsSsd) { | |
8230 | if (string(GetParam()) != "bluestore") | |
8231 | return; | |
8232 | ||
8233 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "ssd"); | |
8234 | StartDeferred(0x1000); | |
8235 | ||
8236 | int r; | |
8237 | coll_t cid; | |
8238 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
8239 | auto ch = store->create_new_collection(cid); | |
8240 | { | |
8241 | ObjectStore::Transaction t; | |
8242 | t.create_collection(cid, 0); | |
8243 | cerr << "Creating collection " << cid << std::endl; | |
8244 | r = queue_transaction(store, ch, std::move(t)); | |
8245 | ASSERT_EQ(r, 0); | |
8246 | } | |
8247 | { | |
8248 | ObjectStore::Transaction t; | |
8249 | bufferlist bl, orig; | |
8250 | string s(g_ceph_context->_conf->bluestore_max_blob_size_ssd * 8, '0'); | |
8251 | bl.append(s); | |
8252 | t.write(cid, hoid, 0, bl.length(), bl); | |
8253 | cerr << "write" << std::endl; | |
8254 | r = queue_transaction(store, ch, std::move(t)); | |
8255 | ASSERT_EQ(r, 0); | |
8256 | ||
8257 | const PerfCounters* logger = store->get_perf_counters(); | |
8258 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 8u); | |
8259 | } | |
8260 | } | |
eafe8130 TL |
8261 | |
8262 | TEST_P(StoreTestSpecificAUSize, ReproNoBlobMultiTest) { | |
8263 | ||
8264 | if(string(GetParam()) != "bluestore") | |
8265 | return; | |
8266 | ||
8267 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8268 | SetVal(g_conf(), "bluestore_block_db_size", "4294967296"); | |
8269 | SetVal(g_conf(), "bluestore_block_size", "12884901888"); | |
8270 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
8271 | ||
8272 | g_conf().apply_changes(nullptr); | |
8273 | ||
8274 | StartDeferred(65536); | |
8275 | ||
8276 | int r; | |
8277 | coll_t cid; | |
8278 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
8279 | ghobject_t hoid2 = hoid; | |
8280 | hoid2.hobj.snap = 1; | |
8281 | ||
8282 | auto ch = store->create_new_collection(cid); | |
8283 | { | |
8284 | ObjectStore::Transaction t; | |
8285 | t.create_collection(cid, 0); | |
8286 | cerr << "Creating collection " << cid << std::endl; | |
8287 | r = queue_transaction(store, ch, std::move(t)); | |
8288 | ASSERT_EQ(r, 0); | |
8289 | } | |
8290 | { | |
8291 | bool exists = store->exists(ch, hoid); | |
8292 | ASSERT_TRUE(!exists); | |
8293 | ||
8294 | ObjectStore::Transaction t; | |
8295 | t.touch(cid, hoid); | |
8296 | cerr << "Creating object " << hoid << std::endl; | |
8297 | r = queue_transaction(store, ch, std::move(t)); | |
8298 | ASSERT_EQ(r, 0); | |
8299 | ||
8300 | exists = store->exists(ch, hoid); | |
8301 | ASSERT_EQ(true, exists); | |
8302 | } | |
8303 | { | |
8304 | uint64_t offs = 0; | |
8305 | bufferlist bl; | |
8306 | const int size = 0x100; | |
8307 | bufferptr ap(size); | |
8308 | memset(ap.c_str(), 'a', size); | |
8309 | bl.append(ap); | |
8310 | int i = 0; | |
8311 | uint64_t blob_size = 524288; | |
8312 | uint64_t total = 0; | |
8313 | for (i = 0; i <= 512; i++) { | |
8314 | offs = 0 + i * size; | |
8315 | ObjectStore::Transaction t; | |
8316 | ghobject_t hoid2 = hoid; | |
8317 | hoid2.hobj.snap = i + 1; | |
8318 | while (offs < 128 * 1024 * 1024) { | |
8319 | ||
8320 | t.write(cid, hoid, offs, ap.length(), bl); | |
8321 | offs += blob_size; | |
8322 | total += ap.length(); | |
8323 | } | |
8324 | t.clone(cid, hoid, hoid2); | |
8325 | r = queue_transaction(store, ch, std::move(t)); | |
8326 | ASSERT_EQ(r, 0); | |
8327 | } | |
8328 | cerr << "Total written = " << total << std::endl; | |
8329 | } | |
8330 | { | |
8331 | cerr << "Finalizing" << std::endl; | |
8332 | const PerfCounters* logger = store->get_perf_counters(); | |
8333 | ASSERT_GE(logger->get(l_bluestore_gc_merged), 1024*1024*1024); | |
8334 | } | |
8335 | } | |
8336 | ||
9f95a23c TL |
8337 | void doManySetAttr(ObjectStore* store, |
8338 | std::function<void(ObjectStore*)> do_check_fn) | |
8339 | { | |
8340 | MixedGenerator gen(447); | |
8341 | gen_type rng(time(NULL)); | |
8342 | coll_t cid(spg_t(pg_t(0, 447), shard_id_t::NO_SHARD)); | |
8343 | ||
f6b5b4d7 | 8344 | SyntheticWorkloadState test_obj(store, &gen, &rng, cid, 0, 0, 0); |
9f95a23c | 8345 | test_obj.init(); |
f6b5b4d7 TL |
8346 | size_t object_count = 256; |
8347 | for (size_t i = 0; i < object_count; ++i) { | |
9f95a23c TL |
8348 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; |
8349 | test_obj.touch(); | |
8350 | } | |
f6b5b4d7 | 8351 | for (size_t i = 0; i < object_count; ++i) { |
9f95a23c TL |
8352 | if (!(i % 100)) { |
8353 | cerr << "Op " << i << std::endl; | |
8354 | test_obj.print_internal_state(); | |
8355 | } | |
f6b5b4d7 | 8356 | test_obj.set_fixed_attrs(1024, 64, 4096); // 1024 attributes, 64 bytes name and 4K value |
9f95a23c TL |
8357 | } |
8358 | test_obj.wait_for_done(); | |
8359 | ||
f6b5b4d7 TL |
8360 | std::cout << "done" << std::endl; |
8361 | do_check_fn(store); | |
9f95a23c TL |
8362 | AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); |
8363 | ceph_assert(admin_socket); | |
8364 | ||
8365 | ceph::bufferlist in, out; | |
8366 | ostringstream err; | |
8367 | ||
f6b5b4d7 TL |
8368 | auto r = admin_socket->execute_command( |
8369 | { "{\"prefix\": \"bluefs stats\"}" }, | |
9f95a23c | 8370 | in, err, &out); |
f6b5b4d7 TL |
8371 | if (r != 0) { |
8372 | cerr << "failure querying: " << cpp_strerror(r) << std::endl; | |
8373 | } else { | |
8374 | std::cout << std::string(out.c_str(), out.length()) << std::endl; | |
9f95a23c | 8375 | } |
9f95a23c TL |
8376 | do_check_fn(store); |
8377 | test_obj.shutdown(); | |
8378 | } | |
8379 | ||
8380 | TEST_P(StoreTestSpecificAUSize, SpilloverTest) { | |
8381 | if (string(GetParam()) != "bluestore") | |
8382 | return; | |
8383 | ||
8384 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8385 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8386 | SetVal(g_conf(), "bluestore_volume_selection_policy", "rocksdb_original"); | |
8387 | ||
8388 | g_conf().apply_changes(nullptr); | |
8389 | ||
8390 | StartDeferred(65536); | |
8391 | doManySetAttr(store.get(), | |
8392 | [&](ObjectStore* _store) { | |
8393 | ||
8394 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8395 | ceph_assert(bstore); | |
f6b5b4d7 | 8396 | bstore->compact(); |
9f95a23c TL |
8397 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8398 | //experimentally it was discovered that this case results in 400+MB spillover | |
8399 | //using lower 300MB threshold just to be safe enough | |
f6b5b4d7 TL |
8400 | std::cout << "db_used:" << logger->get(l_bluefs_db_used_bytes) << std::endl; |
8401 | std::cout << "slow_used:" << logger->get(l_bluefs_slow_used_bytes) << std::endl; | |
9f95a23c | 8402 | |
f6b5b4d7 TL |
8403 | // Disabling any validation/assertion for now as it looks like |
8404 | // we're unable to 100% force RocksDB to spillover. | |
8405 | // Leaving test case hoping to fix that one day though. | |
8406 | //ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 16 * 1024 * 1024); | |
9f95a23c TL |
8407 | } |
8408 | ); | |
8409 | } | |
8410 | ||
8411 | TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) { | |
8412 | if (string(GetParam()) != "bluestore") | |
8413 | return; | |
8414 | ||
8415 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8416 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8417 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
8418 | SetVal(g_conf(), "bluestore_volume_selection_reserved", "1"); // just use non-zero to enable | |
8419 | ||
8420 | g_conf().apply_changes(nullptr); | |
8421 | ||
8422 | StartDeferred(65536); | |
8423 | doManySetAttr(store.get(), | |
8424 | [&](ObjectStore* _store) { | |
8425 | ||
8426 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8427 | ceph_assert(bstore); | |
f6b5b4d7 | 8428 | bstore->compact(); |
9f95a23c TL |
8429 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8430 | ASSERT_EQ(0, logger->get(l_bluefs_slow_used_bytes)); | |
8431 | } | |
8432 | ); | |
8433 | } | |
8434 | ||
8435 | TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) { | |
8436 | if (string(GetParam()) != "bluestore") | |
8437 | return; | |
8438 | ||
8439 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8440 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8441 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
8442 | //default 2.0 factor results in too high threshold, using less value | |
8443 | // that results in less but still present spillover. | |
8444 | SetVal(g_conf(), "bluestore_volume_selection_reserved_factor", "0.5"); | |
8445 | ||
8446 | g_conf().apply_changes(nullptr); | |
8447 | ||
8448 | StartDeferred(65536); | |
8449 | doManySetAttr(store.get(), | |
8450 | [&](ObjectStore* _store) { | |
8451 | ||
8452 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8453 | ceph_assert(bstore); | |
f6b5b4d7 | 8454 | bstore->compact(); |
9f95a23c TL |
8455 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8456 | ASSERT_LE(logger->get(l_bluefs_slow_used_bytes), 300 * 1024 * 1024); // see SpilloverTest for 300MB choice rationale | |
8457 | } | |
8458 | ); | |
8459 | } | |
8460 | ||
11fdf7f2 | 8461 | #endif // WITH_BLUESTORE |
f64942e4 | 8462 | |
7c673cae FG |
8463 | int main(int argc, char **argv) { |
8464 | vector<const char*> args; | |
8465 | argv_to_vec(argc, (const char **)argv, args); | |
7c673cae FG |
8466 | |
8467 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
11fdf7f2 TL |
8468 | CODE_ENVIRONMENT_UTILITY, |
8469 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae FG |
8470 | common_init_finish(g_ceph_context); |
8471 | ||
11fdf7f2 TL |
8472 | // make sure we can adjust any config settings |
8473 | g_ceph_context->_conf._clear_safe_to_start_threads(); | |
8474 | ||
8475 | g_ceph_context->_conf.set_val_or_die("osd_journal_size", "400"); | |
8476 | g_ceph_context->_conf.set_val_or_die("filestore_index_retry_probability", "0.5"); | |
8477 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_timeout", "1000"); | |
8478 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_suicide_timeout", "10000"); | |
8479 | //g_ceph_context->_conf.set_val_or_die("filestore_fiemap", "true"); | |
8480 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mkfs", "false"); | |
8481 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mount", "false"); | |
8482 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_umount", "false"); | |
8483 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_misc", "true"); | |
8484 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_small_allocations", "4"); | |
8485 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_freelist", "true"); | |
8486 | g_ceph_context->_conf.set_val_or_die("bluestore_clone_cow", "true"); | |
8487 | g_ceph_context->_conf.set_val_or_die("bluestore_max_alloc_size", "196608"); | |
7c673cae FG |
8488 | |
8489 | // set small cache sizes so we see trimming during Synthetic tests | |
11fdf7f2 TL |
8490 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_hdd", "4000000"); |
8491 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_ssd", "4000000"); | |
7c673cae FG |
8492 | |
8493 | // very short *_max prealloc so that we fall back to async submits | |
11fdf7f2 TL |
8494 | g_ceph_context->_conf.set_val_or_die("bluestore_blobid_prealloc", "10"); |
8495 | g_ceph_context->_conf.set_val_or_die("bluestore_nid_prealloc", "10"); | |
8496 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_randomize_serial_transaction", | |
7c673cae FG |
8497 | "10"); |
8498 | ||
11fdf7f2 | 8499 | g_ceph_context->_conf.set_val_or_die("bdev_debug_aio", "true"); |
7c673cae FG |
8500 | |
8501 | // specify device size | |
11fdf7f2 | 8502 | g_ceph_context->_conf.set_val_or_die("bluestore_block_size", |
94b18763 | 8503 | stringify(DEF_STORE_TEST_BLOCKDEV_SIZE)); |
7c673cae | 8504 | |
11fdf7f2 | 8505 | g_ceph_context->_conf.set_val_or_die( |
7c673cae | 8506 | "enable_experimental_unrecoverable_data_corrupting_features", "*"); |
11fdf7f2 | 8507 | g_ceph_context->_conf.apply_changes(nullptr); |
7c673cae FG |
8508 | |
8509 | ::testing::InitGoogleTest(&argc, argv); | |
8510 | return RUN_ALL_TESTS(); | |
8511 | } | |
8512 | ||
8513 | /* | |
8514 | * Local Variables: | |
8515 | * compile-command: "cd ../.. ; make ceph_test_objectstore && | |
8516 | * ./ceph_test_objectstore \ | |
8517 | * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20 | |
8518 | * " | |
8519 | * End: | |
8520 | */ |