]>
Commit | Line | Data |
---|---|---|
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- | |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <glob.h> | |
16 | #include <stdio.h> | |
17 | #include <string.h> | |
18 | #include <iostream> | |
19 | #include <time.h> | |
20 | #include <sys/mount.h> | |
21 | #include <boost/scoped_ptr.hpp> | |
22 | #include <boost/random/mersenne_twister.hpp> | |
23 | #include <boost/random/uniform_int.hpp> | |
24 | #include <boost/random/binomial_distribution.hpp> | |
25 | #include <gtest/gtest.h> | |
26 | ||
27 | #include "os/ObjectStore.h" | |
28 | #include "os/filestore/FileStore.h" | |
29 | #if defined(HAVE_LIBAIO) | |
30 | #include "os/bluestore/BlueStore.h" | |
31 | #endif | |
32 | #include "include/Context.h" | |
33 | #include "common/ceph_argparse.h" | |
34 | #include "global/global_init.h" | |
35 | #include "common/Mutex.h" | |
36 | #include "common/Cond.h" | |
37 | #include "common/errno.h" | |
38 | #include "include/stringify.h" | |
39 | #include "include/coredumpctl.h" | |
40 | ||
41 | #include "include/unordered_map.h" | |
42 | #include "store_test_fixture.h" | |
43 | ||
44 | ||
45 | typedef boost::mt11213b gen_type; | |
46 | ||
47 | const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000; | |
48 | #define dout_context g_ceph_context | |
49 | ||
50 | #if GTEST_HAS_PARAM_TEST | |
51 | ||
52 | static bool bl_eq(bufferlist& expected, bufferlist& actual) | |
53 | { | |
54 | if (expected.contents_equal(actual)) | |
55 | return true; | |
56 | ||
57 | unsigned first = 0; | |
58 | if(expected.length() != actual.length()) { | |
59 | cout << "--- buffer lengths mismatch " << std::hex | |
60 | << "expected 0x" << expected.length() << " != actual 0x" | |
61 | << actual.length() << std::dec << std::endl; | |
62 | derr << "--- buffer lengths mismatch " << std::hex | |
63 | << "expected 0x" << expected.length() << " != actual 0x" | |
64 | << actual.length() << std::dec << dendl; | |
65 | } | |
66 | auto len = MIN(expected.length(), actual.length()); | |
67 | while ( first<len && expected[first] == actual[first]) | |
68 | ++first; | |
69 | unsigned last = len; | |
70 | while (last > 0 && expected[last-1] == actual[last-1]) | |
71 | --last; | |
72 | if(len > 0) { | |
73 | cout << "--- buffer mismatch between offset 0x" << std::hex << first | |
74 | << " and 0x" << last << ", total 0x" << len << std::dec | |
75 | << std::endl; | |
76 | derr << "--- buffer mismatch between offset 0x" << std::hex << first | |
77 | << " and 0x" << last << ", total 0x" << len << std::dec | |
78 | << dendl; | |
79 | cout << "--- expected:\n"; | |
80 | expected.hexdump(cout); | |
81 | cout << "--- actual:\n"; | |
82 | actual.hexdump(cout); | |
83 | } | |
84 | return false; | |
85 | } | |
86 | ||
87 | ||
88 | ||
89 | template <typename T> | |
90 | int apply_transaction( | |
91 | T &store, | |
92 | ObjectStore::Sequencer *osr, | |
93 | ObjectStore::Transaction &&t) { | |
94 | if (rand() % 2) { | |
95 | ObjectStore::Transaction t2; | |
96 | t2.append(t); | |
97 | return store->apply_transaction(osr, std::move(t2)); | |
98 | } else { | |
99 | return store->apply_transaction(osr, std::move(t)); | |
100 | } | |
101 | } | |
102 | ||
103 | ||
104 | bool sorted(const vector<ghobject_t> &in) { | |
105 | ghobject_t start; | |
106 | for (vector<ghobject_t>::const_iterator i = in.begin(); | |
107 | i != in.end(); | |
108 | ++i) { | |
109 | if (start > *i) { | |
110 | cout << start << " should follow " << *i << std::endl; | |
111 | return false; | |
112 | } | |
113 | start = *i; | |
114 | } | |
115 | return true; | |
116 | } | |
117 | ||
118 | class StoreTest : public StoreTestFixture, | |
119 | public ::testing::WithParamInterface<const char*> { | |
120 | public: | |
121 | StoreTest() | |
122 | : StoreTestFixture(GetParam()) | |
123 | {} | |
124 | }; | |
125 | ||
126 | class StoreTestDeferredSetup : public StoreTest { | |
127 | void SetUp() override { | |
128 | //do nothing | |
129 | } | |
130 | ||
131 | protected: | |
132 | void DeferredSetup() { | |
133 | StoreTest::SetUp(); | |
134 | } | |
135 | ||
136 | public: | |
137 | }; | |
138 | ||
139 | class StoreTestSpecificAUSize : public StoreTestDeferredSetup { | |
140 | ||
141 | public: | |
142 | typedef | |
143 | std::function<void( | |
144 | boost::scoped_ptr<ObjectStore>& store, | |
145 | uint64_t num_ops, | |
146 | uint64_t max_obj, | |
147 | uint64_t max_wr, | |
148 | uint64_t align)> MatrixTest; | |
149 | ||
150 | void StartDeferred(size_t min_alloc_size) { | |
151 | g_conf->set_val("bluestore_min_alloc_size", stringify(min_alloc_size)); | |
152 | DeferredSetup(); | |
153 | } | |
154 | ||
155 | void TearDown() override { | |
156 | g_conf->set_val("bluestore_min_alloc_size", "0"); | |
157 | StoreTestDeferredSetup::TearDown(); | |
158 | } | |
159 | ||
160 | private: | |
161 | // bluestore matrix testing | |
162 | uint64_t max_write = 40 * 1024; | |
163 | uint64_t max_size = 400 * 1024; | |
164 | uint64_t alignment = 0; | |
165 | uint64_t num_ops = 10000; | |
166 | ||
167 | protected: | |
168 | string matrix_get(const char *k) { | |
169 | if (string(k) == "max_write") { | |
170 | return stringify(max_write); | |
171 | } else if (string(k) == "max_size") { | |
172 | return stringify(max_size); | |
173 | } else if (string(k) == "alignment") { | |
174 | return stringify(alignment); | |
175 | } else if (string(k) == "num_ops") { | |
176 | return stringify(num_ops); | |
177 | } else { | |
178 | char *buf; | |
179 | g_conf->get_val(k, &buf, -1); | |
180 | string v = buf; | |
181 | free(buf); | |
182 | return v; | |
183 | } | |
184 | } | |
185 | ||
186 | void matrix_set(const char *k, const char *v) { | |
187 | if (string(k) == "max_write") { | |
188 | max_write = atoll(v); | |
189 | } else if (string(k) == "max_size") { | |
190 | max_size = atoll(v); | |
191 | } else if (string(k) == "alignment") { | |
192 | alignment = atoll(v); | |
193 | } else if (string(k) == "num_ops") { | |
194 | num_ops = atoll(v); | |
195 | } else { | |
196 | g_conf->set_val(k, v); | |
197 | } | |
198 | } | |
199 | ||
200 | void do_matrix_choose(const char *matrix[][10], | |
201 | int i, int pos, int num, | |
202 | boost::scoped_ptr<ObjectStore>& store, | |
203 | MatrixTest fn) { | |
204 | if (matrix[i][0]) { | |
205 | int count; | |
206 | for (count = 0; matrix[i][count+1]; ++count) ; | |
207 | for (int j = 1; matrix[i][j]; ++j) { | |
208 | matrix_set(matrix[i][0], matrix[i][j]); | |
209 | do_matrix_choose(matrix, | |
210 | i + 1, | |
211 | pos * count + j - 1, | |
212 | num * count, | |
213 | store, | |
214 | fn); | |
215 | } | |
216 | } else { | |
217 | cout << "---------------------- " << (pos + 1) << " / " << num | |
218 | << " ----------------------" << std::endl; | |
219 | for (unsigned k=0; matrix[k][0]; ++k) { | |
220 | cout << " " << matrix[k][0] << " = " << matrix_get(matrix[k][0]) | |
221 | << std::endl; | |
222 | } | |
223 | g_ceph_context->_conf->apply_changes(NULL); | |
224 | fn(store, num_ops, max_size, max_write, alignment); | |
225 | } | |
226 | } | |
227 | ||
228 | void do_matrix(const char *matrix[][10], | |
229 | boost::scoped_ptr<ObjectStore>& store, | |
230 | MatrixTest fn) { | |
231 | map<string,string> old; | |
232 | for (unsigned i=0; matrix[i][0]; ++i) { | |
233 | old[matrix[i][0]] = matrix_get(matrix[i][0]); | |
234 | } | |
235 | cout << "saved config options " << old << std::endl; | |
236 | ||
237 | if (strcmp(matrix[0][0], "bluestore_min_alloc_size") == 0) { | |
238 | int count; | |
239 | for (count = 0; matrix[0][count+1]; ++count) ; | |
240 | for (size_t j = 1; matrix[0][j]; ++j) { | |
241 | if (j > 1) { | |
242 | TearDown(); | |
243 | } | |
244 | StartDeferred(strtoll(matrix[0][j], NULL, 10)); | |
245 | do_matrix_choose(matrix, 1, j - 1, count, store, fn); | |
246 | } | |
247 | } else { | |
248 | StartDeferred(0); | |
249 | do_matrix_choose(matrix, 0, 0, 1, store, fn); | |
250 | } | |
251 | ||
252 | cout << "restoring config options " << old << std::endl; | |
253 | for (auto p : old) { | |
254 | cout << " " << p.first << " = " << p.second << std::endl; | |
255 | matrix_set(p.first.c_str(), p.second.c_str()); | |
256 | } | |
257 | g_ceph_context->_conf->apply_changes(NULL); | |
258 | } | |
259 | ||
260 | }; | |
261 | ||
262 | TEST_P(StoreTest, collect_metadata) { | |
263 | map<string,string> pm; | |
264 | store->collect_metadata(&pm); | |
265 | if (GetParam() == string("filestore")) { | |
266 | ASSERT_NE(pm.count("filestore_backend"), 0u); | |
267 | ASSERT_NE(pm.count("filestore_f_type"), 0u); | |
268 | ASSERT_NE(pm.count("backend_filestore_partition_path"), 0u); | |
269 | ASSERT_NE(pm.count("backend_filestore_dev_node"), 0u); | |
270 | } | |
271 | } | |
272 | ||
273 | TEST_P(StoreTest, Trivial) { | |
274 | } | |
275 | ||
276 | TEST_P(StoreTest, TrivialRemount) { | |
277 | int r = store->umount(); | |
278 | ASSERT_EQ(0, r); | |
279 | r = store->mount(); | |
280 | ASSERT_EQ(0, r); | |
281 | } | |
282 | ||
283 | TEST_P(StoreTest, SimpleRemount) { | |
284 | ObjectStore::Sequencer osr("test"); | |
285 | coll_t cid; | |
286 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
287 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
288 | bufferlist bl; | |
289 | bl.append("1234512345"); | |
290 | int r; | |
291 | { | |
292 | cerr << "create collection + write" << std::endl; | |
293 | ObjectStore::Transaction t; | |
294 | t.create_collection(cid, 0); | |
295 | t.write(cid, hoid, 0, bl.length(), bl); | |
296 | r = apply_transaction(store, &osr, std::move(t)); | |
297 | ASSERT_EQ(r, 0); | |
298 | } | |
299 | r = store->umount(); | |
300 | ASSERT_EQ(0, r); | |
301 | r = store->mount(); | |
302 | ASSERT_EQ(0, r); | |
303 | { | |
304 | ObjectStore::Transaction t; | |
305 | t.write(cid, hoid2, 0, bl.length(), bl); | |
306 | r = apply_transaction(store, &osr, std::move(t)); | |
307 | ASSERT_EQ(r, 0); | |
308 | } | |
309 | { | |
310 | ObjectStore::Transaction t; | |
311 | t.remove(cid, hoid); | |
312 | t.remove(cid, hoid2); | |
313 | t.remove_collection(cid); | |
314 | cerr << "remove collection" << std::endl; | |
315 | r = apply_transaction(store, &osr, std::move(t)); | |
316 | ASSERT_EQ(r, 0); | |
317 | } | |
318 | r = store->umount(); | |
319 | ASSERT_EQ(0, r); | |
320 | r = store->mount(); | |
321 | ASSERT_EQ(0, r); | |
322 | { | |
323 | ObjectStore::Transaction t; | |
324 | t.create_collection(cid, 0); | |
325 | r = apply_transaction(store, &osr, std::move(t)); | |
326 | ASSERT_EQ(r, 0); | |
327 | bool exists = store->exists(cid, hoid); | |
328 | ASSERT_TRUE(!exists); | |
329 | } | |
330 | { | |
331 | ObjectStore::Transaction t; | |
332 | t.remove_collection(cid); | |
333 | cerr << "remove collection" << std::endl; | |
334 | r = apply_transaction(store, &osr, std::move(t)); | |
335 | ASSERT_EQ(r, 0); | |
336 | } | |
337 | } | |
338 | ||
339 | TEST_P(StoreTest, IORemount) { | |
340 | ObjectStore::Sequencer osr("test"); | |
341 | coll_t cid; | |
342 | bufferlist bl; | |
343 | bl.append("1234512345"); | |
344 | int r; | |
345 | { | |
346 | cerr << "create collection + objects" << std::endl; | |
347 | ObjectStore::Transaction t; | |
348 | t.create_collection(cid, 0); | |
349 | for (int n=1; n<=100; ++n) { | |
350 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
351 | t.write(cid, hoid, 0, bl.length(), bl); | |
352 | } | |
353 | r = apply_transaction(store, &osr, std::move(t)); | |
354 | ASSERT_EQ(r, 0); | |
355 | } | |
356 | // overwrites | |
357 | { | |
358 | cout << "overwrites" << std::endl; | |
359 | for (int n=1; n<=100; ++n) { | |
360 | ObjectStore::Transaction t; | |
361 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
362 | t.write(cid, hoid, 1, bl.length(), bl); | |
363 | r = apply_transaction(store, &osr, std::move(t)); | |
364 | ASSERT_EQ(r, 0); | |
365 | } | |
366 | } | |
367 | r = store->umount(); | |
368 | ASSERT_EQ(0, r); | |
369 | r = store->mount(); | |
370 | ASSERT_EQ(0, r); | |
371 | { | |
372 | ObjectStore::Transaction t; | |
373 | for (int n=1; n<=100; ++n) { | |
374 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
375 | t.remove(cid, hoid); | |
376 | } | |
377 | t.remove_collection(cid); | |
378 | r = apply_transaction(store, &osr, std::move(t)); | |
379 | ASSERT_EQ(r, 0); | |
380 | } | |
381 | } | |
382 | ||
383 | TEST_P(StoreTest, UnprintableCharsName) { | |
384 | ObjectStore::Sequencer osr("test"); | |
385 | coll_t cid; | |
386 | string name = "funnychars_"; | |
387 | for (unsigned i = 0; i < 256; ++i) { | |
388 | name.push_back(i); | |
389 | } | |
390 | ghobject_t oid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
391 | int r; | |
392 | { | |
393 | cerr << "create collection + object" << std::endl; | |
394 | ObjectStore::Transaction t; | |
395 | t.create_collection(cid, 0); | |
396 | t.touch(cid, oid); | |
397 | r = apply_transaction(store, &osr, std::move(t)); | |
398 | ASSERT_EQ(r, 0); | |
399 | } | |
400 | r = store->umount(); | |
401 | ASSERT_EQ(0, r); | |
402 | r = store->mount(); | |
403 | ASSERT_EQ(0, r); | |
404 | { | |
405 | cout << "removing" << std::endl; | |
406 | ObjectStore::Transaction t; | |
407 | t.remove(cid, oid); | |
408 | t.remove_collection(cid); | |
409 | r = apply_transaction(store, &osr, std::move(t)); | |
410 | ASSERT_EQ(r, 0); | |
411 | } | |
412 | } | |
413 | ||
414 | TEST_P(StoreTest, FiemapEmpty) { | |
415 | ObjectStore::Sequencer osr("test"); | |
416 | coll_t cid; | |
417 | int r = 0; | |
418 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
419 | { | |
420 | ObjectStore::Transaction t; | |
421 | t.create_collection(cid, 0); | |
422 | t.touch(cid, oid); | |
423 | t.truncate(cid, oid, 100000); | |
424 | r = apply_transaction(store, &osr, std::move(t)); | |
425 | ASSERT_EQ(r, 0); | |
426 | } | |
427 | { | |
428 | bufferlist bl; | |
429 | store->fiemap(cid, oid, 0, 100000, bl); | |
430 | map<uint64_t,uint64_t> m, e; | |
431 | bufferlist::iterator p = bl.begin(); | |
432 | ::decode(m, p); | |
433 | cout << " got " << m << std::endl; | |
434 | e[0] = 100000; | |
435 | EXPECT_TRUE(m == e || m.empty()); | |
436 | } | |
437 | { | |
438 | ObjectStore::Transaction t; | |
439 | t.remove(cid, oid); | |
440 | t.remove_collection(cid); | |
441 | cerr << "remove collection" << std::endl; | |
442 | r = apply_transaction(store, &osr, std::move(t)); | |
443 | ASSERT_EQ(r, 0); | |
444 | } | |
445 | } | |
446 | ||
447 | TEST_P(StoreTest, FiemapHoles) { | |
448 | ObjectStore::Sequencer osr("test"); | |
449 | const uint64_t MAX_EXTENTS = 4000; | |
450 | const uint64_t SKIP_STEP = 65536; | |
451 | coll_t cid; | |
452 | int r = 0; | |
453 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
454 | bufferlist bl; | |
455 | bl.append("foo"); | |
456 | { | |
457 | ObjectStore::Transaction t; | |
458 | t.create_collection(cid, 0); | |
459 | t.touch(cid, oid); | |
460 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) | |
461 | t.write(cid, oid, SKIP_STEP * i, 3, bl); | |
462 | r = apply_transaction(store, &osr, std::move(t)); | |
463 | ASSERT_EQ(r, 0); | |
464 | } | |
465 | { | |
466 | //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3 | |
467 | bufferlist bl; | |
468 | store->fiemap(cid, oid, 0, SKIP_STEP * (MAX_EXTENTS - 1) + 3, bl); | |
469 | map<uint64_t,uint64_t> m, e; | |
470 | bufferlist::iterator p = bl.begin(); | |
471 | ::decode(m, p); | |
472 | cout << " got " << m << std::endl; | |
473 | ASSERT_TRUE(!m.empty()); | |
474 | ASSERT_GE(m[0], 3u); | |
475 | bool extents_exist = true; | |
476 | if (m.size() == MAX_EXTENTS) { | |
477 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) | |
478 | extents_exist = extents_exist && m.count(SKIP_STEP*i); | |
479 | } | |
480 | ASSERT_TRUE((m.size() == 1 && | |
481 | m[0] > SKIP_STEP * (MAX_EXTENTS - 1)) || | |
482 | (m.size() == MAX_EXTENTS && extents_exist)); | |
483 | ||
484 | // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3 | |
485 | // reset bufferlist and map | |
486 | bl.clear(); | |
487 | m.clear(); | |
488 | e.clear(); | |
489 | store->fiemap(cid, oid, SKIP_STEP, SKIP_STEP * (MAX_EXTENTS - 2) + 3, bl); | |
490 | p = bl.begin(); | |
491 | ::decode(m, p); | |
492 | cout << " got " << m << std::endl; | |
493 | ASSERT_TRUE(!m.empty()); | |
494 | ASSERT_GE(m[SKIP_STEP], 3u); | |
495 | extents_exist = true; | |
496 | if (m.size() == (MAX_EXTENTS - 2)) { | |
497 | for (uint64_t i = 1; i < MAX_EXTENTS - 1; i++) | |
498 | extents_exist = extents_exist && m.count(SKIP_STEP*i); | |
499 | } | |
500 | ASSERT_TRUE((m.size() == 1 && | |
501 | m[SKIP_STEP] > SKIP_STEP * (MAX_EXTENTS - 2)) || | |
502 | (m.size() == (MAX_EXTENTS - 1) && extents_exist)); | |
503 | } | |
504 | { | |
505 | ObjectStore::Transaction t; | |
506 | t.remove(cid, oid); | |
507 | t.remove_collection(cid); | |
508 | cerr << "remove collection" << std::endl; | |
509 | r = apply_transaction(store, &osr, std::move(t)); | |
510 | ASSERT_EQ(r, 0); | |
511 | } | |
512 | } | |
513 | ||
514 | TEST_P(StoreTest, SimpleMetaColTest) { | |
515 | ObjectStore::Sequencer osr("test"); | |
516 | coll_t cid; | |
517 | int r = 0; | |
518 | { | |
519 | ObjectStore::Transaction t; | |
520 | t.create_collection(cid, 0); | |
521 | cerr << "create collection" << std::endl; | |
522 | r = apply_transaction(store, &osr, std::move(t)); | |
523 | ASSERT_EQ(r, 0); | |
524 | } | |
525 | { | |
526 | ObjectStore::Transaction t; | |
527 | t.remove_collection(cid); | |
528 | cerr << "remove collection" << std::endl; | |
529 | r = apply_transaction(store, &osr, std::move(t)); | |
530 | ASSERT_EQ(r, 0); | |
531 | } | |
532 | { | |
533 | ObjectStore::Transaction t; | |
534 | t.create_collection(cid, 0); | |
535 | cerr << "add collection" << std::endl; | |
536 | r = apply_transaction(store, &osr, std::move(t)); | |
537 | ASSERT_EQ(r, 0); | |
538 | } | |
539 | { | |
540 | ObjectStore::Transaction t; | |
541 | t.remove_collection(cid); | |
542 | cerr << "remove collection" << std::endl; | |
543 | r = apply_transaction(store, &osr, std::move(t)); | |
544 | ASSERT_EQ(r, 0); | |
545 | } | |
546 | } | |
547 | ||
548 | TEST_P(StoreTest, SimplePGColTest) { | |
549 | ObjectStore::Sequencer osr("test"); | |
550 | coll_t cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD)); | |
551 | int r = 0; | |
552 | { | |
553 | ObjectStore::Transaction t; | |
554 | t.create_collection(cid, 4); | |
555 | cerr << "create collection" << std::endl; | |
556 | r = apply_transaction(store, &osr, std::move(t)); | |
557 | ASSERT_EQ(r, 0); | |
558 | } | |
559 | { | |
560 | ObjectStore::Transaction t; | |
561 | t.remove_collection(cid); | |
562 | cerr << "remove collection" << std::endl; | |
563 | r = apply_transaction(store, &osr, std::move(t)); | |
564 | ASSERT_EQ(r, 0); | |
565 | } | |
566 | { | |
567 | ObjectStore::Transaction t; | |
568 | t.create_collection(cid, 4); | |
569 | cerr << "add collection" << std::endl; | |
570 | r = apply_transaction(store, &osr, std::move(t)); | |
571 | ASSERT_EQ(r, 0); | |
572 | } | |
573 | { | |
574 | ObjectStore::Transaction t; | |
575 | t.remove_collection(cid); | |
576 | cerr << "remove collection" << std::endl; | |
577 | r = apply_transaction(store, &osr, std::move(t)); | |
578 | ASSERT_EQ(r, 0); | |
579 | } | |
580 | } | |
581 | ||
582 | TEST_P(StoreTest, SimpleColPreHashTest) { | |
583 | ObjectStore::Sequencer osr("test"); | |
584 | // Firstly we will need to revert the value making sure | |
585 | // collection hint actually works | |
586 | int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold; | |
587 | std::ostringstream oss; | |
588 | if (merge_threshold > 0) { | |
589 | oss << "-" << merge_threshold; | |
590 | g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str()); | |
591 | } | |
592 | ||
593 | uint32_t pg_num = 128; | |
594 | ||
595 | boost::uniform_int<> pg_id_range(0, pg_num); | |
596 | gen_type rng(time(NULL)); | |
597 | int pg_id = pg_id_range(rng); | |
598 | ||
599 | int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple; | |
600 | boost::uniform_int<> folders_range(5, 256); | |
601 | uint64_t expected_num_objs = (uint64_t)objs_per_folder * (uint64_t)folders_range(rng); | |
602 | ||
603 | coll_t cid(spg_t(pg_t(pg_id, 15), shard_id_t::NO_SHARD)); | |
604 | int r; | |
605 | { | |
606 | // Create a collection along with a hint | |
607 | ObjectStore::Transaction t; | |
608 | t.create_collection(cid, 5); | |
609 | cerr << "create collection" << std::endl; | |
610 | bufferlist hint; | |
611 | ::encode(pg_num, hint); | |
612 | ::encode(expected_num_objs, hint); | |
613 | t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); | |
614 | cerr << "collection hint" << std::endl; | |
615 | r = apply_transaction(store, &osr, std::move(t)); | |
616 | ASSERT_EQ(r, 0); | |
617 | } | |
618 | { | |
619 | // Remove the collection | |
620 | ObjectStore::Transaction t; | |
621 | t.remove_collection(cid); | |
622 | cerr << "remove collection" << std::endl; | |
623 | r = apply_transaction(store, &osr, std::move(t)); | |
624 | ASSERT_EQ(r, 0); | |
625 | } | |
626 | // Revert the config change so that it does not affect the split/merge tests | |
627 | if (merge_threshold > 0) { | |
628 | oss.str(""); | |
629 | oss << merge_threshold; | |
630 | g_ceph_context->_conf->set_val("filestore_merge_threshold", oss.str().c_str()); | |
631 | } | |
632 | } | |
633 | ||
634 | TEST_P(StoreTest, SmallBlockWrites) { | |
635 | ObjectStore::Sequencer osr("test"); | |
636 | int r; | |
637 | coll_t cid; | |
638 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
639 | { | |
640 | ObjectStore::Transaction t; | |
641 | t.create_collection(cid, 0); | |
642 | cerr << "Creating collection " << cid << std::endl; | |
643 | r = apply_transaction(store, &osr, std::move(t)); | |
644 | ASSERT_EQ(r, 0); | |
645 | } | |
646 | bufferlist a; | |
647 | bufferptr ap(0x1000); | |
648 | memset(ap.c_str(), 'a', 0x1000); | |
649 | a.append(ap); | |
650 | bufferlist b; | |
651 | bufferptr bp(0x1000); | |
652 | memset(bp.c_str(), 'b', 0x1000); | |
653 | b.append(bp); | |
654 | bufferlist c; | |
655 | bufferptr cp(0x1000); | |
656 | memset(cp.c_str(), 'c', 0x1000); | |
657 | c.append(cp); | |
658 | bufferptr zp(0x1000); | |
659 | zp.zero(); | |
660 | bufferlist z; | |
661 | z.append(zp); | |
662 | { | |
663 | ObjectStore::Transaction t; | |
664 | t.write(cid, hoid, 0, 0x1000, a); | |
665 | r = apply_transaction(store, &osr, std::move(t)); | |
666 | ASSERT_EQ(r, 0); | |
667 | ||
668 | bufferlist in, exp; | |
669 | r = store->read(cid, hoid, 0, 0x4000, in); | |
670 | ASSERT_EQ(0x1000, r); | |
671 | exp.append(a); | |
672 | ASSERT_TRUE(bl_eq(exp, in)); | |
673 | } | |
674 | { | |
675 | ObjectStore::Transaction t; | |
676 | t.write(cid, hoid, 0x1000, 0x1000, b); | |
677 | r = apply_transaction(store, &osr, std::move(t)); | |
678 | ASSERT_EQ(r, 0); | |
679 | ||
680 | bufferlist in, exp; | |
681 | r = store->read(cid, hoid, 0, 0x4000, in); | |
682 | ASSERT_EQ(0x2000, r); | |
683 | exp.append(a); | |
684 | exp.append(b); | |
685 | ASSERT_TRUE(bl_eq(exp, in)); | |
686 | } | |
687 | { | |
688 | ObjectStore::Transaction t; | |
689 | t.write(cid, hoid, 0x3000, 0x1000, c); | |
690 | r = apply_transaction(store, &osr, std::move(t)); | |
691 | ASSERT_EQ(r, 0); | |
692 | ||
693 | bufferlist in, exp; | |
694 | r = store->read(cid, hoid, 0, 0x4000, in); | |
695 | ASSERT_EQ(0x4000, r); | |
696 | exp.append(a); | |
697 | exp.append(b); | |
698 | exp.append(z); | |
699 | exp.append(c); | |
700 | ASSERT_TRUE(bl_eq(exp, in)); | |
701 | } | |
702 | { | |
703 | ObjectStore::Transaction t; | |
704 | t.write(cid, hoid, 0x2000, 0x1000, a); | |
705 | r = apply_transaction(store, &osr, std::move(t)); | |
706 | ASSERT_EQ(r, 0); | |
707 | ||
708 | bufferlist in, exp; | |
709 | r = store->read(cid, hoid, 0, 0x4000, in); | |
710 | ASSERT_EQ(0x4000, r); | |
711 | exp.append(a); | |
712 | exp.append(b); | |
713 | exp.append(a); | |
714 | exp.append(c); | |
715 | ASSERT_TRUE(bl_eq(exp, in)); | |
716 | } | |
717 | { | |
718 | ObjectStore::Transaction t; | |
719 | t.write(cid, hoid, 0, 0x1000, c); | |
720 | r = apply_transaction(store, &osr, std::move(t)); | |
721 | ASSERT_EQ(r, 0); | |
722 | } | |
723 | { | |
724 | bufferlist in, exp; | |
725 | r = store->read(cid, hoid, 0, 0x4000, in); | |
726 | ASSERT_EQ(0x4000, r); | |
727 | exp.append(c); | |
728 | exp.append(b); | |
729 | exp.append(a); | |
730 | exp.append(c); | |
731 | ASSERT_TRUE(bl_eq(exp, in)); | |
732 | } | |
733 | { | |
734 | ObjectStore::Transaction t; | |
735 | t.remove(cid, hoid); | |
736 | t.remove_collection(cid); | |
737 | cerr << "Cleaning" << std::endl; | |
738 | r = apply_transaction(store, &osr, std::move(t)); | |
739 | ASSERT_EQ(r, 0); | |
740 | } | |
741 | } | |
742 | ||
743 | TEST_P(StoreTest, BufferCacheReadTest) { | |
744 | ObjectStore::Sequencer osr("test"); | |
745 | int r; | |
746 | coll_t cid; | |
747 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
748 | { | |
749 | bufferlist in; | |
750 | r = store->read(cid, hoid, 0, 5, in); | |
751 | ASSERT_EQ(-ENOENT, r); | |
752 | } | |
753 | { | |
754 | ObjectStore::Transaction t; | |
755 | t.create_collection(cid, 0); | |
756 | cerr << "Creating collection " << cid << std::endl; | |
757 | r = apply_transaction(store, &osr, std::move(t)); | |
758 | ASSERT_EQ(r, 0); | |
759 | } | |
760 | { | |
761 | bool exists = store->exists(cid, hoid); | |
762 | ASSERT_TRUE(!exists); | |
763 | ||
764 | ObjectStore::Transaction t; | |
765 | t.touch(cid, hoid); | |
766 | cerr << "Creating object " << hoid << std::endl; | |
767 | r = apply_transaction(store, &osr, std::move(t)); | |
768 | ASSERT_EQ(r, 0); | |
769 | ||
770 | exists = store->exists(cid, hoid); | |
771 | ASSERT_EQ(true, exists); | |
772 | } | |
773 | { | |
774 | ObjectStore::Transaction t; | |
775 | bufferlist bl, newdata; | |
776 | bl.append("abcde"); | |
777 | t.write(cid, hoid, 0, 5, bl); | |
778 | t.write(cid, hoid, 10, 5, bl); | |
779 | cerr << "TwinWrite" << std::endl; | |
780 | r = apply_transaction(store, &osr, std::move(t)); | |
781 | ASSERT_EQ(r, 0); | |
782 | ||
783 | r = store->read(cid, hoid, 0, 15, newdata); | |
784 | ASSERT_EQ(r, 15); | |
785 | { | |
786 | bufferlist expected; | |
787 | expected.append(bl); | |
788 | expected.append_zero(5); | |
789 | expected.append(bl); | |
790 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
791 | } | |
792 | } | |
793 | //overwrite over the same extents | |
794 | { | |
795 | ObjectStore::Transaction t; | |
796 | bufferlist bl, newdata; | |
797 | bl.append("edcba"); | |
798 | t.write(cid, hoid, 0, 5, bl); | |
799 | t.write(cid, hoid, 10, 5, bl); | |
800 | cerr << "TwinWrite" << std::endl; | |
801 | r = apply_transaction(store, &osr, std::move(t)); | |
802 | ASSERT_EQ(r, 0); | |
803 | ||
804 | r = store->read(cid, hoid, 0, 15, newdata); | |
805 | ASSERT_EQ(r, 15); | |
806 | { | |
807 | bufferlist expected; | |
808 | expected.append(bl); | |
809 | expected.append_zero(5); | |
810 | expected.append(bl); | |
811 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
812 | } | |
813 | } | |
814 | //additional write to an unused region of some blob | |
815 | { | |
816 | ObjectStore::Transaction t; | |
817 | bufferlist bl2, newdata; | |
818 | bl2.append("1234567890"); | |
819 | ||
820 | t.write(cid, hoid, 20, bl2.length(), bl2); | |
821 | cerr << "Append" << std::endl; | |
822 | r = apply_transaction(store, &osr, std::move(t)); | |
823 | ASSERT_EQ(r, 0); | |
824 | ||
825 | r = store->read(cid, hoid, 0, 30, newdata); | |
826 | ASSERT_EQ(r, 30); | |
827 | { | |
828 | bufferlist expected; | |
829 | expected.append("edcba"); | |
830 | expected.append_zero(5); | |
831 | expected.append("edcba"); | |
832 | expected.append_zero(5); | |
833 | expected.append(bl2); | |
834 | ||
835 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
836 | } | |
837 | } | |
838 | //additional write to an unused region of some blob and partial owerite over existing extents | |
839 | { | |
840 | ObjectStore::Transaction t; | |
841 | bufferlist bl, bl2, bl3, newdata; | |
842 | bl.append("DCB"); | |
843 | bl2.append("1234567890"); | |
844 | bl3.append("BA"); | |
845 | ||
846 | t.write(cid, hoid, 30, bl2.length(), bl2); | |
847 | t.write(cid, hoid, 1, bl.length(), bl); | |
848 | t.write(cid, hoid, 13, bl3.length(), bl3); | |
849 | cerr << "TripleWrite" << std::endl; | |
850 | r = apply_transaction(store, &osr, std::move(t)); | |
851 | ASSERT_EQ(r, 0); | |
852 | ||
853 | r = store->read(cid, hoid, 0, 40, newdata); | |
854 | ASSERT_EQ(r, 40); | |
855 | { | |
856 | bufferlist expected; | |
857 | expected.append("eDCBa"); | |
858 | expected.append_zero(5); | |
859 | expected.append("edcBA"); | |
860 | expected.append_zero(5); | |
861 | expected.append(bl2); | |
862 | expected.append(bl2); | |
863 | ||
864 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
865 | } | |
866 | } | |
867 | } | |
868 | ||
869 | void doCompressionTest( boost::scoped_ptr<ObjectStore>& store) | |
870 | { | |
871 | ObjectStore::Sequencer osr("test"); | |
872 | int r; | |
873 | coll_t cid; | |
874 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
875 | { | |
876 | bufferlist in; | |
877 | r = store->read(cid, hoid, 0, 5, in); | |
878 | ASSERT_EQ(-ENOENT, r); | |
879 | } | |
880 | { | |
881 | ObjectStore::Transaction t; | |
882 | t.create_collection(cid, 0); | |
883 | cerr << "Creating collection " << cid << std::endl; | |
884 | r = apply_transaction(store, &osr, std::move(t)); | |
885 | ASSERT_EQ(r, 0); | |
886 | } | |
887 | { | |
888 | bool exists = store->exists(cid, hoid); | |
889 | ASSERT_TRUE(!exists); | |
890 | ||
891 | ObjectStore::Transaction t; | |
892 | t.touch(cid, hoid); | |
893 | cerr << "Creating object " << hoid << std::endl; | |
894 | r = apply_transaction(store, &osr, std::move(t)); | |
895 | ASSERT_EQ(r, 0); | |
896 | ||
897 | exists = store->exists(cid, hoid); | |
898 | ASSERT_EQ(true, exists); | |
899 | } | |
900 | std::string data; | |
901 | data.resize(0x10000 * 4); | |
902 | for(size_t i = 0;i < data.size(); i++) | |
903 | data[i] = i / 256; | |
904 | { | |
905 | ObjectStore::Transaction t; | |
906 | bufferlist bl, newdata; | |
907 | bl.append(data); | |
908 | t.write(cid, hoid, 0, bl.length(), bl); | |
909 | cerr << "CompressibleData (4xAU) Write" << std::endl; | |
910 | r = apply_transaction(store, &osr, std::move(t)); | |
911 | ASSERT_EQ(r, 0); | |
912 | ||
913 | r = store->read(cid, hoid, 0, data.size() , newdata); | |
914 | ||
915 | ASSERT_EQ(r, (int)data.size()); | |
916 | { | |
917 | bufferlist expected; | |
918 | expected.append(data); | |
919 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
920 | } | |
921 | newdata.clear(); | |
922 | r = store->read(cid, hoid, 0, 711 , newdata); | |
923 | ASSERT_EQ(r, 711); | |
924 | { | |
925 | bufferlist expected; | |
926 | expected.append(data.substr(0,711)); | |
927 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
928 | } | |
929 | newdata.clear(); | |
930 | r = store->read(cid, hoid, 0xf00f, data.size(), newdata); | |
931 | ASSERT_EQ(r, int(data.size() - 0xf00f) ); | |
932 | { | |
933 | bufferlist expected; | |
934 | expected.append(data.substr(0xf00f)); | |
935 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
936 | } | |
937 | { | |
938 | struct store_statfs_t statfs; | |
939 | int r = store->statfs(&statfs); | |
940 | ASSERT_EQ(r, 0); | |
941 | ASSERT_EQ(statfs.stored, (unsigned)data.size()); | |
942 | ASSERT_LE(statfs.compressed, (unsigned)data.size()); | |
943 | ASSERT_EQ(statfs.compressed_original, (unsigned)data.size()); | |
944 | ASSERT_LE(statfs.compressed_allocated, (unsigned)data.size()); | |
945 | } | |
946 | } | |
947 | std::string data2; | |
948 | data2.resize(0x10000 * 4 - 0x9000); | |
949 | for(size_t i = 0;i < data2.size(); i++) | |
950 | data2[i] = (i+1) / 256; | |
951 | { | |
952 | ObjectStore::Transaction t; | |
953 | bufferlist bl, newdata; | |
954 | bl.append(data2); | |
955 | t.write(cid, hoid, 0x8000, bl.length(), bl); | |
956 | cerr << "CompressibleData partial overwrite" << std::endl; | |
957 | r = apply_transaction(store, &osr, std::move(t)); | |
958 | ASSERT_EQ(r, 0); | |
959 | ||
960 | r = store->read(cid, hoid, 0, 0x10000, newdata); | |
961 | ASSERT_EQ(r, (int)0x10000); | |
962 | { | |
963 | bufferlist expected; | |
964 | expected.append(data.substr(0, 0x8000)); | |
965 | expected.append(data2.substr(0, 0x8000)); | |
966 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
967 | } | |
968 | newdata.clear(); | |
969 | r = store->read(cid, hoid, 0x9000, 711 , newdata); | |
970 | ASSERT_EQ(r, 711); | |
971 | { | |
972 | bufferlist expected; | |
973 | expected.append(data2.substr(0x1000,711)); | |
974 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
975 | } | |
976 | newdata.clear(); | |
977 | r = store->read(cid, hoid, 0x0, 0x40000, newdata); | |
978 | ASSERT_EQ(r, int(0x40000) ); | |
979 | { | |
980 | bufferlist expected; | |
981 | expected.append(data.substr(0, 0x8000)); | |
982 | expected.append(data2.substr(0, 0x37000)); | |
983 | expected.append(data.substr(0x3f000, 0x1000)); | |
984 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
985 | } | |
986 | } | |
987 | data2.resize(0x3f000); | |
988 | for(size_t i = 0;i < data2.size(); i++) | |
989 | data2[i] = (i+2) / 256; | |
990 | { | |
991 | ObjectStore::Transaction t; | |
992 | bufferlist bl, newdata; | |
993 | bl.append(data2); | |
994 | t.write(cid, hoid, 0, bl.length(), bl); | |
995 | cerr << "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
996 | r = apply_transaction(store, &osr, std::move(t)); | |
997 | ASSERT_EQ(r, 0); | |
998 | ||
999 | r = store->read(cid, hoid, 0, 0x3e000 - 1, newdata); | |
1000 | ASSERT_EQ(r, (int)0x3e000 - 1); | |
1001 | { | |
1002 | bufferlist expected; | |
1003 | expected.append(data2.substr(0, 0x3e000 - 1)); | |
1004 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1005 | } | |
1006 | newdata.clear(); | |
1007 | r = store->read(cid, hoid, 0x3e000-1, 0x2001, newdata); | |
1008 | ASSERT_EQ(r, 0x2001); | |
1009 | { | |
1010 | bufferlist expected; | |
1011 | expected.append(data2.substr(0x3e000-1, 0x1001)); | |
1012 | expected.append(data.substr(0x3f000, 0x1000)); | |
1013 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1014 | } | |
1015 | newdata.clear(); | |
1016 | r = store->read(cid, hoid, 0x0, 0x40000, newdata); | |
1017 | ASSERT_EQ(r, int(0x40000) ); | |
1018 | { | |
1019 | bufferlist expected; | |
1020 | expected.append(data2.substr(0, 0x3f000)); | |
1021 | expected.append(data.substr(0x3f000, 0x1000)); | |
1022 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1023 | } | |
1024 | } | |
1025 | data.resize(0x1001); | |
1026 | for(size_t i = 0;i < data.size(); i++) | |
1027 | data[i] = (i+3) / 256; | |
1028 | { | |
1029 | ObjectStore::Transaction t; | |
1030 | bufferlist bl, newdata; | |
1031 | bl.append(data); | |
1032 | t.write(cid, hoid, 0x3f000-1, bl.length(), bl); | |
1033 | cerr << "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
1034 | r = apply_transaction(store, &osr, std::move(t)); | |
1035 | ASSERT_EQ(r, 0); | |
1036 | ||
1037 | r = store->read(cid, hoid, 0x3e000, 0x2000, newdata); | |
1038 | ASSERT_EQ(r, (int)0x2000); | |
1039 | { | |
1040 | bufferlist expected; | |
1041 | expected.append(data2.substr(0x3e000, 0x1000 - 1)); | |
1042 | expected.append(data.substr(0, 0x1001)); | |
1043 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1044 | } | |
1045 | } | |
1046 | { | |
1047 | ObjectStore::Transaction t; | |
1048 | t.remove(cid, hoid); | |
1049 | cerr << "Cleaning object" << std::endl; | |
1050 | r = apply_transaction(store, &osr, std::move(t)); | |
1051 | ASSERT_EQ(r, 0); | |
1052 | } | |
1053 | //force fsck | |
1054 | EXPECT_EQ(store->umount(), 0); | |
1055 | EXPECT_EQ(store->mount(), 0); | |
1056 | auto orig_min_blob_size = g_conf->bluestore_compression_min_blob_size; | |
1057 | { | |
1058 | g_conf->set_val("bluestore_compression_min_blob_size", "262144"); | |
1059 | g_ceph_context->_conf->apply_changes(NULL); | |
1060 | data.resize(0x10000*6); | |
1061 | ||
1062 | for(size_t i = 0;i < data.size(); i++) | |
1063 | data[i] = i / 256; | |
1064 | ObjectStore::Transaction t; | |
1065 | bufferlist bl, newdata; | |
1066 | bl.append(data); | |
1067 | t.write(cid, hoid, 0, bl.length(), bl); | |
1068 | cerr << "CompressibleData large blob" << std::endl; | |
1069 | r = apply_transaction(store, &osr, std::move(t)); | |
1070 | ASSERT_EQ(r, 0); | |
1071 | } | |
1072 | //force fsck | |
1073 | EXPECT_EQ(store->umount(), 0); | |
1074 | EXPECT_EQ(store->mount(), 0); | |
1075 | ||
1076 | { | |
1077 | ObjectStore::Transaction t; | |
1078 | t.remove(cid, hoid); | |
1079 | t.remove_collection(cid); | |
1080 | cerr << "Cleaning" << std::endl; | |
1081 | r = apply_transaction(store, &osr, std::move(t)); | |
1082 | ASSERT_EQ(r, 0); | |
1083 | } | |
1084 | g_conf->set_val("bluestore_compression_min_blob_size", stringify(orig_min_blob_size)); | |
1085 | g_ceph_context->_conf->apply_changes(NULL); | |
1086 | } | |
1087 | ||
1088 | TEST_P(StoreTest, CompressionTest) { | |
1089 | if (string(GetParam()) != "bluestore") | |
1090 | return; | |
1091 | ||
1092 | g_conf->set_val("bluestore_compression_algorithm", "snappy"); | |
1093 | g_conf->set_val("bluestore_compression_mode", "force"); | |
1094 | ||
1095 | g_ceph_context->_conf->apply_changes(NULL); | |
1096 | ||
1097 | doCompressionTest(store); | |
1098 | ||
1099 | g_conf->set_val("bluestore_compression_algorithm", "zlib"); | |
1100 | g_conf->set_val("bluestore_compression_mode", "force"); | |
1101 | g_ceph_context->_conf->apply_changes(NULL); | |
1102 | ||
1103 | doCompressionTest(store); | |
1104 | ||
1105 | g_conf->set_val("bluestore_compression_algorithm", "snappy"); | |
1106 | g_conf->set_val("bluestore_compression_mode", "none"); | |
1107 | g_ceph_context->_conf->apply_changes(NULL); | |
1108 | } | |
1109 | ||
1110 | TEST_P(StoreTest, SimpleObjectTest) { | |
1111 | ObjectStore::Sequencer osr("test"); | |
1112 | int r; | |
1113 | coll_t cid; | |
1114 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1115 | { | |
1116 | bufferlist in; | |
1117 | r = store->read(cid, hoid, 0, 5, in); | |
1118 | ASSERT_EQ(-ENOENT, r); | |
1119 | } | |
1120 | { | |
1121 | ObjectStore::Transaction t; | |
1122 | t.create_collection(cid, 0); | |
1123 | cerr << "Creating collection " << cid << std::endl; | |
1124 | r = apply_transaction(store, &osr, std::move(t)); | |
1125 | ASSERT_EQ(r, 0); | |
1126 | } | |
1127 | { | |
1128 | bool exists = store->exists(cid, hoid); | |
1129 | ASSERT_TRUE(!exists); | |
1130 | ||
1131 | ObjectStore::Transaction t; | |
1132 | t.touch(cid, hoid); | |
1133 | cerr << "Creating object " << hoid << std::endl; | |
1134 | r = apply_transaction(store, &osr, std::move(t)); | |
1135 | ASSERT_EQ(r, 0); | |
1136 | ||
1137 | exists = store->exists(cid, hoid); | |
1138 | ASSERT_EQ(true, exists); | |
1139 | } | |
1140 | { | |
1141 | ObjectStore::Transaction t; | |
1142 | t.remove(cid, hoid); | |
1143 | t.touch(cid, hoid); | |
1144 | cerr << "Remove then create" << std::endl; | |
1145 | r = apply_transaction(store, &osr, std::move(t)); | |
1146 | ASSERT_EQ(r, 0); | |
1147 | } | |
1148 | { | |
1149 | ObjectStore::Transaction t; | |
1150 | bufferlist bl, orig; | |
1151 | bl.append("abcde"); | |
1152 | orig = bl; | |
1153 | t.remove(cid, hoid); | |
1154 | t.write(cid, hoid, 0, 5, bl); | |
1155 | cerr << "Remove then create" << std::endl; | |
1156 | r = apply_transaction(store, &osr, std::move(t)); | |
1157 | ASSERT_EQ(r, 0); | |
1158 | ||
1159 | bufferlist in; | |
1160 | r = store->read(cid, hoid, 0, 5, in); | |
1161 | ASSERT_EQ(5, r); | |
1162 | ASSERT_TRUE(bl_eq(orig, in)); | |
1163 | } | |
1164 | { | |
1165 | ObjectStore::Transaction t; | |
1166 | bufferlist bl, exp; | |
1167 | bl.append("abcde"); | |
1168 | exp = bl; | |
1169 | exp.append(bl); | |
1170 | t.write(cid, hoid, 5, 5, bl); | |
1171 | cerr << "Append" << std::endl; | |
1172 | r = apply_transaction(store, &osr, std::move(t)); | |
1173 | ASSERT_EQ(r, 0); | |
1174 | ||
1175 | bufferlist in; | |
1176 | r = store->read(cid, hoid, 0, 10, in); | |
1177 | ASSERT_EQ(10, r); | |
1178 | ASSERT_TRUE(bl_eq(exp, in)); | |
1179 | } | |
1180 | { | |
1181 | ObjectStore::Transaction t; | |
1182 | bufferlist bl, exp; | |
1183 | bl.append("abcdeabcde"); | |
1184 | exp = bl; | |
1185 | t.write(cid, hoid, 0, 10, bl); | |
1186 | cerr << "Full overwrite" << std::endl; | |
1187 | r = apply_transaction(store, &osr, std::move(t)); | |
1188 | ASSERT_EQ(r, 0); | |
1189 | ||
1190 | bufferlist in; | |
1191 | r = store->read(cid, hoid, 0, 10, in); | |
1192 | ASSERT_EQ(10, r); | |
1193 | ASSERT_TRUE(bl_eq(exp, in)); | |
1194 | } | |
1195 | { | |
1196 | ObjectStore::Transaction t; | |
1197 | bufferlist bl; | |
1198 | bl.append("abcde"); | |
1199 | t.write(cid, hoid, 3, 5, bl); | |
1200 | cerr << "Partial overwrite" << std::endl; | |
1201 | r = apply_transaction(store, &osr, std::move(t)); | |
1202 | ASSERT_EQ(r, 0); | |
1203 | ||
1204 | bufferlist in, exp; | |
1205 | exp.append("abcabcdede"); | |
1206 | r = store->read(cid, hoid, 0, 10, in); | |
1207 | ASSERT_EQ(10, r); | |
1208 | in.hexdump(cout); | |
1209 | ASSERT_TRUE(bl_eq(exp, in)); | |
1210 | } | |
1211 | { | |
1212 | { | |
1213 | ObjectStore::Transaction t; | |
1214 | bufferlist bl; | |
1215 | bl.append("fghij"); | |
1216 | t.truncate(cid, hoid, 0); | |
1217 | t.write(cid, hoid, 5, 5, bl); | |
1218 | cerr << "Truncate + hole" << std::endl; | |
1219 | r = apply_transaction(store, &osr, std::move(t)); | |
1220 | ASSERT_EQ(r, 0); | |
1221 | } | |
1222 | { | |
1223 | ObjectStore::Transaction t; | |
1224 | bufferlist bl; | |
1225 | bl.append("abcde"); | |
1226 | t.write(cid, hoid, 0, 5, bl); | |
1227 | cerr << "Reverse fill-in" << std::endl; | |
1228 | r = apply_transaction(store, &osr, std::move(t)); | |
1229 | ASSERT_EQ(r, 0); | |
1230 | } | |
1231 | ||
1232 | bufferlist in, exp; | |
1233 | exp.append("abcdefghij"); | |
1234 | r = store->read(cid, hoid, 0, 10, in); | |
1235 | ASSERT_EQ(10, r); | |
1236 | in.hexdump(cout); | |
1237 | ASSERT_TRUE(bl_eq(exp, in)); | |
1238 | } | |
1239 | { | |
1240 | ObjectStore::Transaction t; | |
1241 | bufferlist bl; | |
1242 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1243 | t.write(cid, hoid, 0, bl.length(), bl); | |
1244 | cerr << "larger overwrite" << std::endl; | |
1245 | r = apply_transaction(store, &osr, std::move(t)); | |
1246 | ASSERT_EQ(r, 0); | |
1247 | ||
1248 | bufferlist in; | |
1249 | r = store->read(cid, hoid, 0, bl.length(), in); | |
1250 | ASSERT_EQ((int)bl.length(), r); | |
1251 | in.hexdump(cout); | |
1252 | ASSERT_TRUE(bl_eq(bl, in)); | |
1253 | } | |
1254 | { | |
1255 | bufferlist bl; | |
1256 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1257 | ||
1258 | //test: offset=len=0 mean read all data | |
1259 | bufferlist in; | |
1260 | r = store->read(cid, hoid, 0, 0, in); | |
1261 | ASSERT_EQ((int)bl.length(), r); | |
1262 | in.hexdump(cout); | |
1263 | ASSERT_TRUE(bl_eq(bl, in)); | |
1264 | } | |
1265 | { | |
1266 | //verifying unaligned csums | |
1267 | std::string s1("1"), s2(0x1000, '2'), s3("00"); | |
1268 | { | |
1269 | ObjectStore::Transaction t; | |
1270 | bufferlist bl; | |
1271 | bl.append(s1); | |
1272 | bl.append(s2); | |
1273 | t.truncate(cid, hoid, 0); | |
1274 | t.write(cid, hoid, 0x1000-1, bl.length(), bl); | |
1275 | cerr << "Write unaligned csum, stage 1" << std::endl; | |
1276 | r = apply_transaction(store, &osr, std::move(t)); | |
1277 | ASSERT_EQ(r, 0); | |
1278 | } | |
1279 | ||
1280 | bufferlist in, exp1, exp2, exp3; | |
1281 | exp1.append(s1); | |
1282 | exp2.append(s2); | |
1283 | exp3.append(s3); | |
1284 | r = store->read(cid, hoid, 0x1000-1, 1, in); | |
1285 | ASSERT_EQ(1, r); | |
1286 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1287 | in.clear(); | |
1288 | r = store->read(cid, hoid, 0x1000, 0x1000, in); | |
1289 | ASSERT_EQ(0x1000, r); | |
1290 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1291 | ||
1292 | { | |
1293 | ObjectStore::Transaction t; | |
1294 | bufferlist bl; | |
1295 | bl.append(s3); | |
1296 | t.write(cid, hoid, 1, bl.length(), bl); | |
1297 | cerr << "Write unaligned csum, stage 2" << std::endl; | |
1298 | r = apply_transaction(store, &osr, std::move(t)); | |
1299 | ASSERT_EQ(r, 0); | |
1300 | } | |
1301 | in.clear(); | |
1302 | r = store->read(cid, hoid, 1, 2, in); | |
1303 | ASSERT_EQ(2, r); | |
1304 | ASSERT_TRUE(bl_eq(exp3, in)); | |
1305 | in.clear(); | |
1306 | r = store->read(cid, hoid, 0x1000-1, 1, in); | |
1307 | ASSERT_EQ(1, r); | |
1308 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1309 | in.clear(); | |
1310 | r = store->read(cid, hoid, 0x1000, 0x1000, in); | |
1311 | ASSERT_EQ(0x1000, r); | |
1312 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1313 | ||
1314 | } | |
1315 | ||
1316 | { | |
1317 | ObjectStore::Transaction t; | |
1318 | t.remove(cid, hoid); | |
1319 | t.remove_collection(cid); | |
1320 | cerr << "Cleaning" << std::endl; | |
1321 | r = apply_transaction(store, &osr, std::move(t)); | |
1322 | ASSERT_EQ(r, 0); | |
1323 | } | |
1324 | } | |
1325 | ||
1326 | #if defined(HAVE_LIBAIO) | |
1327 | TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { | |
1328 | if(string(GetParam()) != "bluestore") | |
1329 | return; | |
1330 | StartDeferred(65536); | |
1331 | g_conf->set_val("bluestore_compression_mode", "force"); | |
1332 | ||
1333 | // just a big number to disble gc | |
1334 | g_conf->set_val("bluestore_gc_enable_total_threshold", "100000"); | |
1335 | g_conf->apply_changes(NULL); | |
1336 | int r; | |
1337 | ||
1338 | ObjectStore::Sequencer osr("test"); | |
1339 | coll_t cid; | |
1340 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1341 | ghobject_t hoid2 = hoid; | |
1342 | hoid2.hobj.snap = 1; | |
1343 | { | |
1344 | bufferlist in; | |
1345 | r = store->read(cid, hoid, 0, 5, in); | |
1346 | ASSERT_EQ(-ENOENT, r); | |
1347 | } | |
1348 | { | |
1349 | ObjectStore::Transaction t; | |
1350 | t.create_collection(cid, 0); | |
1351 | cerr << "Creating collection " << cid << std::endl; | |
1352 | r = apply_transaction(store, &osr, std::move(t)); | |
1353 | ASSERT_EQ(r, 0); | |
1354 | } | |
1355 | { | |
1356 | bool exists = store->exists(cid, hoid); | |
1357 | ASSERT_TRUE(!exists); | |
1358 | ||
1359 | ObjectStore::Transaction t; | |
1360 | t.touch(cid, hoid); | |
1361 | cerr << "Creating object " << hoid << std::endl; | |
1362 | r = apply_transaction(store, &osr, std::move(t)); | |
1363 | ASSERT_EQ(r, 0); | |
1364 | ||
1365 | exists = store->exists(cid, hoid); | |
1366 | ASSERT_EQ(true, exists); | |
1367 | } | |
1368 | { | |
1369 | struct store_statfs_t statfs; | |
1370 | int r = store->statfs(&statfs); | |
1371 | ASSERT_EQ(r, 0); | |
1372 | ASSERT_EQ( 0u, statfs.allocated); | |
1373 | ASSERT_EQ( 0u, statfs.stored); | |
1374 | ASSERT_EQ(g_conf->bluestore_block_size, statfs.total); | |
1375 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf->bluestore_block_size); | |
1376 | //force fsck | |
1377 | EXPECT_EQ(store->umount(), 0); | |
1378 | EXPECT_EQ(store->mount(), 0); | |
1379 | } | |
1380 | { | |
1381 | ObjectStore::Transaction t; | |
1382 | bufferlist bl; | |
1383 | bl.append("abcde"); | |
1384 | t.write(cid, hoid, 0, 5, bl); | |
1385 | cerr << "Append 5 bytes" << std::endl; | |
1386 | r = apply_transaction(store, &osr, std::move(t)); | |
1387 | ASSERT_EQ(r, 0); | |
1388 | ||
1389 | struct store_statfs_t statfs; | |
1390 | int r = store->statfs(&statfs); | |
1391 | ASSERT_EQ(r, 0); | |
1392 | ASSERT_EQ(5, statfs.stored); | |
1393 | ASSERT_EQ(0x10000, statfs.allocated); | |
1394 | ASSERT_EQ(0, statfs.compressed); | |
1395 | ASSERT_EQ(0, statfs.compressed_original); | |
1396 | ASSERT_EQ(0, statfs.compressed_allocated); | |
1397 | //force fsck | |
1398 | EXPECT_EQ(store->umount(), 0); | |
1399 | EXPECT_EQ(store->mount(), 0); | |
1400 | } | |
1401 | { | |
1402 | ObjectStore::Transaction t; | |
1403 | std::string s(0x30000, 'a'); | |
1404 | bufferlist bl; | |
1405 | bl.append(s); | |
1406 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1407 | cerr << "Append 0x30000 compressible bytes" << std::endl; | |
1408 | r = apply_transaction(store, &osr, std::move(t)); | |
1409 | ASSERT_EQ(r, 0); | |
1410 | ||
1411 | struct store_statfs_t statfs; | |
1412 | int r = store->statfs(&statfs); | |
1413 | ASSERT_EQ(r, 0); | |
1414 | ASSERT_EQ(0x30005, statfs.stored); | |
1415 | ASSERT_EQ(0x30000, statfs.allocated); | |
1416 | ASSERT_LE(statfs.compressed, 0x10000); | |
1417 | ASSERT_EQ(0x20000, statfs.compressed_original); | |
1418 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
1419 | //force fsck | |
1420 | EXPECT_EQ(store->umount(), 0); | |
1421 | EXPECT_EQ(store->mount(), 0); | |
1422 | } | |
1423 | { | |
1424 | ObjectStore::Transaction t; | |
1425 | t.zero(cid, hoid, 1, 3); | |
1426 | t.zero(cid, hoid, 0x20000, 9); | |
1427 | cerr << "Punch hole at 1~3, 0x20000~9" << std::endl; | |
1428 | r = apply_transaction(store, &osr, std::move(t)); | |
1429 | ASSERT_EQ(r, 0); | |
1430 | ||
1431 | struct store_statfs_t statfs; | |
1432 | int r = store->statfs(&statfs); | |
1433 | ASSERT_EQ(r, 0); | |
1434 | ASSERT_EQ(0x30005 - 3 - 9, statfs.stored); | |
1435 | ASSERT_EQ(0x30000, statfs.allocated); | |
1436 | ASSERT_LE(statfs.compressed, 0x10000); | |
1437 | ASSERT_EQ(0x20000 - 9, statfs.compressed_original); | |
1438 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
1439 | //force fsck | |
1440 | EXPECT_EQ(store->umount(), 0); | |
1441 | EXPECT_EQ(store->mount(), 0); | |
1442 | } | |
1443 | { | |
1444 | ObjectStore::Transaction t; | |
1445 | std::string s(0x1000, 'b'); | |
1446 | bufferlist bl; | |
1447 | bl.append(s); | |
1448 | t.write(cid, hoid, 1, bl.length(), bl); | |
1449 | t.write(cid, hoid, 0x10001, bl.length(), bl); | |
1450 | cerr << "Overwrite first and second(compressible) extents" << std::endl; | |
1451 | r = apply_transaction(store, &osr, std::move(t)); | |
1452 | ASSERT_EQ(r, 0); | |
1453 | ||
1454 | struct store_statfs_t statfs; | |
1455 | int r = store->statfs(&statfs); | |
1456 | ASSERT_EQ(r, 0); | |
1457 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs.stored); | |
1458 | ASSERT_EQ(0x40000, statfs.allocated); | |
1459 | ASSERT_LE(statfs.compressed, 0x10000); | |
1460 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs.compressed_original); | |
1461 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
1462 | //force fsck | |
1463 | EXPECT_EQ(store->umount(), 0); | |
1464 | EXPECT_EQ(store->mount(), 0); | |
1465 | } | |
1466 | { | |
1467 | ObjectStore::Transaction t; | |
1468 | std::string s(0x10000, 'c'); | |
1469 | bufferlist bl; | |
1470 | bl.append(s); | |
1471 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1472 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
1473 | t.write(cid, hoid, 0x30000, bl.length(), bl); | |
1474 | cerr << "Overwrite compressed extent with 3 uncompressible ones" << std::endl; | |
1475 | r = apply_transaction(store, &osr, std::move(t)); | |
1476 | ASSERT_EQ(r, 0); | |
1477 | ||
1478 | struct store_statfs_t statfs; | |
1479 | int r = store->statfs(&statfs); | |
1480 | ASSERT_EQ(r, 0); | |
1481 | ASSERT_EQ(0x30000 + 0x1001, statfs.stored); | |
1482 | ASSERT_EQ(0x40000, statfs.allocated); | |
1483 | ASSERT_LE(statfs.compressed, 0); | |
1484 | ASSERT_EQ(0, statfs.compressed_original); | |
1485 | ASSERT_EQ(0, statfs.compressed_allocated); | |
1486 | //force fsck | |
1487 | EXPECT_EQ(store->umount(), 0); | |
1488 | EXPECT_EQ(store->mount(), 0); | |
1489 | } | |
1490 | { | |
1491 | ObjectStore::Transaction t; | |
1492 | t.zero(cid, hoid, 0, 0x40000); | |
1493 | cerr << "Zero object" << std::endl; | |
1494 | r = apply_transaction(store, &osr, std::move(t)); | |
1495 | ASSERT_EQ(r, 0); | |
1496 | struct store_statfs_t statfs; | |
1497 | int r = store->statfs(&statfs); | |
1498 | ASSERT_EQ(r, 0); | |
1499 | ASSERT_EQ(0u, statfs.allocated); | |
1500 | ASSERT_EQ(0u, statfs.stored); | |
1501 | ASSERT_EQ(0u, statfs.compressed_original); | |
1502 | ASSERT_EQ(0u, statfs.compressed); | |
1503 | ASSERT_EQ(0u, statfs.compressed_allocated); | |
1504 | //force fsck | |
1505 | EXPECT_EQ(store->umount(), 0); | |
1506 | EXPECT_EQ(store->mount(), 0); | |
1507 | } | |
1508 | { | |
1509 | ObjectStore::Transaction t; | |
1510 | std::string s(0x10000, 'c'); | |
1511 | bufferlist bl; | |
1512 | bl.append(s); | |
1513 | bl.append(s); | |
1514 | bl.append(s); | |
1515 | bl.append(s.substr(0, 0x10000-2)); | |
1516 | t.write(cid, hoid, 0, bl.length(), bl); | |
1517 | cerr << "Yet another compressible write" << std::endl; | |
1518 | r = apply_transaction(store, &osr, std::move(t)); | |
1519 | ASSERT_EQ(r, 0); | |
1520 | struct store_statfs_t statfs; | |
1521 | r = store->statfs(&statfs); | |
1522 | ASSERT_EQ(r, 0); | |
1523 | ASSERT_EQ(0x40000 - 2, statfs.stored); | |
1524 | ASSERT_EQ(0x30000, statfs.allocated); | |
1525 | ASSERT_LE(statfs.compressed, 0x10000); | |
1526 | ASSERT_EQ(0x20000, statfs.compressed_original); | |
1527 | ASSERT_EQ(0x10000, statfs.compressed_allocated); | |
1528 | //force fsck | |
1529 | EXPECT_EQ(store->umount(), 0); | |
1530 | EXPECT_EQ(store->mount(), 0); | |
1531 | } | |
1532 | { | |
1533 | struct store_statfs_t statfs; | |
1534 | r = store->statfs(&statfs); | |
1535 | ASSERT_EQ(r, 0); | |
1536 | ||
1537 | ObjectStore::Transaction t; | |
1538 | t.clone(cid, hoid, hoid2); | |
1539 | cerr << "Clone compressed objecte" << std::endl; | |
1540 | r = apply_transaction(store, &osr, std::move(t)); | |
1541 | ASSERT_EQ(r, 0); | |
1542 | struct store_statfs_t statfs2; | |
1543 | r = store->statfs(&statfs2); | |
1544 | ASSERT_EQ(r, 0); | |
1545 | ASSERT_GT(statfs2.stored, statfs.stored); | |
1546 | ASSERT_EQ(statfs2.allocated, statfs.allocated); | |
1547 | ASSERT_GT(statfs2.compressed, statfs.compressed); | |
1548 | ASSERT_GT(statfs2.compressed_original, statfs.compressed_original); | |
1549 | ASSERT_EQ(statfs2.compressed_allocated, statfs.compressed_allocated); | |
1550 | } | |
1551 | ||
1552 | { | |
1553 | ObjectStore::Transaction t; | |
1554 | t.remove(cid, hoid); | |
1555 | t.remove(cid, hoid2); | |
1556 | t.remove_collection(cid); | |
1557 | cerr << "Cleaning" << std::endl; | |
1558 | r = apply_transaction(store, &osr, std::move(t)); | |
1559 | ASSERT_EQ(r, 0); | |
1560 | ||
1561 | struct store_statfs_t statfs; | |
1562 | r = store->statfs(&statfs); | |
1563 | ASSERT_EQ(r, 0); | |
1564 | ASSERT_EQ( 0u, statfs.allocated); | |
1565 | ASSERT_EQ( 0u, statfs.stored); | |
1566 | ASSERT_EQ( 0u, statfs.compressed_original); | |
1567 | ASSERT_EQ( 0u, statfs.compressed); | |
1568 | ASSERT_EQ( 0u, statfs.compressed_allocated); | |
1569 | } | |
1570 | g_conf->set_val("bluestore_gc_enable_total_threshold", "0"); | |
1571 | g_conf->set_val("bluestore_compression_mode", "none"); | |
1572 | g_ceph_context->_conf->apply_changes(NULL); | |
1573 | } | |
1574 | ||
1575 | TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) { | |
1576 | if(string(GetParam()) != "bluestore") | |
1577 | return; | |
1578 | StartDeferred(0x10000); | |
1579 | ||
1580 | ObjectStore::Sequencer osr("test"); | |
1581 | int r; | |
1582 | coll_t cid; | |
1583 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1584 | { | |
1585 | ObjectStore::Transaction t; | |
1586 | t.create_collection(cid, 0); | |
1587 | cerr << "Creating collection " << cid << std::endl; | |
1588 | r = apply_transaction(store, &osr, std::move(t)); | |
1589 | ASSERT_EQ(r, 0); | |
1590 | } | |
1591 | { | |
1592 | bool exists = store->exists(cid, hoid); | |
1593 | ASSERT_TRUE(!exists); | |
1594 | ||
1595 | ObjectStore::Transaction t; | |
1596 | t.touch(cid, hoid); | |
1597 | cerr << "Creating object " << hoid << std::endl; | |
1598 | r = apply_transaction(store, &osr, std::move(t)); | |
1599 | ASSERT_EQ(r, 0); | |
1600 | ||
1601 | exists = store->exists(cid, hoid); | |
1602 | ASSERT_EQ(true, exists); | |
1603 | } | |
1604 | { | |
1605 | struct store_statfs_t statfs; | |
1606 | int r = store->statfs(&statfs); | |
1607 | ASSERT_EQ(r, 0); | |
1608 | ASSERT_EQ(g_conf->bluestore_block_size, statfs.total); | |
1609 | ASSERT_EQ(0u, statfs.allocated); | |
1610 | ASSERT_EQ(0u, statfs.stored); | |
1611 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf->bluestore_block_size); | |
1612 | } | |
1613 | std::string data; | |
1614 | data.resize(0x10000 * 3); | |
1615 | { | |
1616 | ObjectStore::Transaction t; | |
1617 | for(size_t i = 0;i < data.size(); i++) | |
1618 | data[i] = i / 256 + 1; | |
1619 | bufferlist bl, newdata; | |
1620 | bl.append(data); | |
1621 | t.write(cid, hoid, 0, bl.length(), bl); | |
1622 | t.zero(cid, hoid, 0x10000, 0x10000); | |
1623 | cerr << "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl; | |
1624 | r = apply_transaction(store, &osr, std::move(t)); | |
1625 | ASSERT_EQ(r, 0); | |
1626 | ||
1627 | struct store_statfs_t statfs; | |
1628 | int r = store->statfs(&statfs); | |
1629 | ASSERT_EQ(r, 0); | |
1630 | ASSERT_EQ(0x20000, statfs.stored); | |
1631 | ASSERT_EQ(0x20000, statfs.allocated); | |
1632 | ||
1633 | r = store->read(cid, hoid, 0, data.size(), newdata); | |
1634 | ASSERT_EQ(r, (int)data.size()); | |
1635 | { | |
1636 | bufferlist expected; | |
1637 | expected.append(data.substr(0, 0x10000)); | |
1638 | expected.append(string(0x10000, 0)); | |
1639 | expected.append(data.substr(0x20000, 0x10000)); | |
1640 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1641 | } | |
1642 | newdata.clear(); | |
1643 | ||
1644 | r = store->read(cid, hoid, 1, data.size()-2, newdata); | |
1645 | ASSERT_EQ(r, (int)data.size()-2); | |
1646 | { | |
1647 | bufferlist expected; | |
1648 | expected.append(data.substr(1, 0x10000-1)); | |
1649 | expected.append(string(0x10000, 0)); | |
1650 | expected.append(data.substr(0x20000, 0x10000 - 1)); | |
1651 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1652 | } | |
1653 | newdata.clear(); | |
1654 | } | |
1655 | //force fsck | |
1656 | EXPECT_EQ(store->umount(), 0); | |
1657 | EXPECT_EQ(store->mount(), 0); | |
1658 | ||
1659 | { | |
1660 | ObjectStore::Transaction t; | |
1661 | std::string data2(3, 'b'); | |
1662 | bufferlist bl, newdata; | |
1663 | bl.append(data2); | |
1664 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
1665 | cerr << "Write 3 bytes after the hole" << std::endl; | |
1666 | r = apply_transaction(store, &osr, std::move(t)); | |
1667 | ASSERT_EQ(r, 0); | |
1668 | ||
1669 | struct store_statfs_t statfs; | |
1670 | int r = store->statfs(&statfs); | |
1671 | ASSERT_EQ(r, 0); | |
1672 | ASSERT_EQ(0x20000, statfs.allocated); | |
1673 | ASSERT_EQ(0x20000, statfs.stored); | |
1674 | ||
1675 | r = store->read(cid, hoid, 0x20000-1, 21, newdata); | |
1676 | ASSERT_EQ(r, (int)21); | |
1677 | { | |
1678 | bufferlist expected; | |
1679 | expected.append(string(0x1, 0)); | |
1680 | expected.append(string(data2)); | |
1681 | expected.append(data.substr(0x20003, 21-4)); | |
1682 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1683 | } | |
1684 | newdata.clear(); | |
1685 | } | |
1686 | //force fsck | |
1687 | EXPECT_EQ(store->umount(), 0); | |
1688 | EXPECT_EQ(store->mount(), 0); | |
1689 | ||
1690 | { | |
1691 | ObjectStore::Transaction t; | |
1692 | std::string data2(3, 'a'); | |
1693 | bufferlist bl, newdata; | |
1694 | bl.append(data2); | |
1695 | t.write(cid, hoid, 0x10000+1, bl.length(), bl); | |
1696 | cerr << "Write 3 bytes to the hole" << std::endl; | |
1697 | r = apply_transaction(store, &osr, std::move(t)); | |
1698 | ASSERT_EQ(r, 0); | |
1699 | ||
1700 | struct store_statfs_t statfs; | |
1701 | int r = store->statfs(&statfs); | |
1702 | ASSERT_EQ(r, 0); | |
1703 | ASSERT_EQ(0x30000, statfs.allocated); | |
1704 | ASSERT_EQ(0x20003, statfs.stored); | |
1705 | ||
1706 | r = store->read(cid, hoid, 0x10000-1, 0x10000+22, newdata); | |
1707 | ASSERT_EQ(r, (int)0x10000+22); | |
1708 | { | |
1709 | bufferlist expected; | |
1710 | expected.append(data.substr(0x10000-1, 1)); | |
1711 | expected.append(string(0x1, 0)); | |
1712 | expected.append(data2); | |
1713 | expected.append(string(0x10000-4, 0)); | |
1714 | expected.append(string(0x3, 'b')); | |
1715 | expected.append(data.substr(0x20004, 21-3)); | |
1716 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1717 | } | |
1718 | newdata.clear(); | |
1719 | } | |
1720 | { | |
1721 | ObjectStore::Transaction t; | |
1722 | bufferlist bl, newdata; | |
1723 | bl.append(string(0x30000, 'c')); | |
1724 | t.write(cid, hoid, 0, 0x30000, bl); | |
1725 | t.zero(cid, hoid, 0, 0x10000); | |
1726 | t.zero(cid, hoid, 0x20000, 0x10000); | |
1727 | cerr << "Rewrite an object and create two holes at the begining and the end" << std::endl; | |
1728 | r = apply_transaction(store, &osr, std::move(t)); | |
1729 | ASSERT_EQ(r, 0); | |
1730 | ||
1731 | struct store_statfs_t statfs; | |
1732 | int r = store->statfs(&statfs); | |
1733 | ASSERT_EQ(r, 0); | |
1734 | ASSERT_EQ(0x10000, statfs.allocated); | |
1735 | ASSERT_EQ(0x10000, statfs.stored); | |
1736 | ||
1737 | r = store->read(cid, hoid, 0, 0x30000, newdata); | |
1738 | ASSERT_EQ(r, (int)0x30000); | |
1739 | { | |
1740 | bufferlist expected; | |
1741 | expected.append(string(0x10000, 0)); | |
1742 | expected.append(string(0x10000, 'c')); | |
1743 | expected.append(string(0x10000, 0)); | |
1744 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1745 | } | |
1746 | newdata.clear(); | |
1747 | } | |
1748 | ||
1749 | //force fsck | |
1750 | EXPECT_EQ(store->umount(), 0); | |
1751 | EXPECT_EQ(store->mount(), 0); | |
1752 | ||
1753 | { | |
1754 | ObjectStore::Transaction t; | |
1755 | t.remove(cid, hoid); | |
1756 | t.remove_collection(cid); | |
1757 | cerr << "Cleaning" << std::endl; | |
1758 | r = apply_transaction(store, &osr, std::move(t)); | |
1759 | ASSERT_EQ(r, 0); | |
1760 | ||
1761 | struct store_statfs_t statfs; | |
1762 | r = store->statfs(&statfs); | |
1763 | ASSERT_EQ(r, 0); | |
1764 | ASSERT_EQ( 0u, statfs.allocated); | |
1765 | ASSERT_EQ( 0u, statfs.stored); | |
1766 | ASSERT_EQ( 0u, statfs.compressed_original); | |
1767 | ASSERT_EQ( 0u, statfs.compressed); | |
1768 | ASSERT_EQ( 0u, statfs.compressed_allocated); | |
1769 | } | |
1770 | } | |
1771 | #endif | |
1772 | ||
1773 | TEST_P(StoreTest, ManySmallWrite) { | |
1774 | ObjectStore::Sequencer osr("test"); | |
1775 | int r; | |
1776 | coll_t cid; | |
1777 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1778 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
1779 | { | |
1780 | ObjectStore::Transaction t; | |
1781 | t.create_collection(cid, 0); | |
1782 | cerr << "Creating collection " << cid << std::endl; | |
1783 | r = apply_transaction(store, &osr, std::move(t)); | |
1784 | ASSERT_EQ(r, 0); | |
1785 | } | |
1786 | bufferlist bl; | |
1787 | bufferptr bp(4096); | |
1788 | bp.zero(); | |
1789 | bl.append(bp); | |
1790 | for (int i=0; i<100; ++i) { | |
1791 | ObjectStore::Transaction t; | |
1792 | t.write(cid, a, i*4096, 4096, bl, 0); | |
1793 | r = apply_transaction(store, &osr, std::move(t)); | |
1794 | ASSERT_EQ(r, 0); | |
1795 | } | |
1796 | for (int i=0; i<100; ++i) { | |
1797 | ObjectStore::Transaction t; | |
1798 | t.write(cid, b, (rand() % 1024)*4096, 4096, bl, 0); | |
1799 | r = apply_transaction(store, &osr, std::move(t)); | |
1800 | ASSERT_EQ(r, 0); | |
1801 | } | |
1802 | { | |
1803 | ObjectStore::Transaction t; | |
1804 | t.remove(cid, a); | |
1805 | t.remove(cid, b); | |
1806 | t.remove_collection(cid); | |
1807 | cerr << "Cleaning" << std::endl; | |
1808 | r = apply_transaction(store, &osr, std::move(t)); | |
1809 | ASSERT_EQ(r, 0); | |
1810 | } | |
1811 | } | |
1812 | ||
1813 | TEST_P(StoreTest, MultiSmallWriteSameBlock) { | |
1814 | ObjectStore::Sequencer osr("test"); | |
1815 | int r; | |
1816 | coll_t cid; | |
1817 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1818 | { | |
1819 | ObjectStore::Transaction t; | |
1820 | t.create_collection(cid, 0); | |
1821 | cerr << "Creating collection " << cid << std::endl; | |
1822 | r = apply_transaction(store, &osr, std::move(t)); | |
1823 | ASSERT_EQ(r, 0); | |
1824 | } | |
1825 | bufferlist bl; | |
1826 | bl.append("short"); | |
1827 | C_SaferCond c, d; | |
1828 | // touch same block in both same transaction, tls, and pipelined txns | |
1829 | { | |
1830 | ObjectStore::Transaction t, u; | |
1831 | t.write(cid, a, 0, 5, bl, 0); | |
1832 | t.write(cid, a, 5, 5, bl, 0); | |
1833 | t.write(cid, a, 4094, 5, bl, 0); | |
1834 | t.write(cid, a, 9000, 5, bl, 0); | |
1835 | u.write(cid, a, 10, 5, bl, 0); | |
1836 | u.write(cid, a, 7000, 5, bl, 0); | |
1837 | vector<ObjectStore::Transaction> v = {t, u}; | |
1838 | store->queue_transactions(&osr, v, nullptr, &c); | |
1839 | } | |
1840 | { | |
1841 | ObjectStore::Transaction t, u; | |
1842 | t.write(cid, a, 40, 5, bl, 0); | |
1843 | t.write(cid, a, 45, 5, bl, 0); | |
1844 | t.write(cid, a, 4094, 5, bl, 0); | |
1845 | t.write(cid, a, 6000, 5, bl, 0); | |
1846 | u.write(cid, a, 610, 5, bl, 0); | |
1847 | u.write(cid, a, 11000, 5, bl, 0); | |
1848 | vector<ObjectStore::Transaction> v = {t, u}; | |
1849 | store->queue_transactions(&osr, v, nullptr, &d); | |
1850 | } | |
1851 | c.wait(); | |
1852 | d.wait(); | |
1853 | { | |
1854 | bufferlist bl2; | |
1855 | r = store->read(cid, a, 0, 16000, bl2); | |
1856 | ASSERT_GE(r, 0); | |
1857 | } | |
1858 | { | |
1859 | ObjectStore::Transaction t; | |
1860 | t.remove(cid, a); | |
1861 | t.remove_collection(cid); | |
1862 | cerr << "Cleaning" << std::endl; | |
1863 | r = apply_transaction(store, &osr, std::move(t)); | |
1864 | ASSERT_EQ(r, 0); | |
1865 | } | |
1866 | } | |
1867 | ||
1868 | TEST_P(StoreTest, SmallSkipFront) { | |
1869 | ObjectStore::Sequencer osr("test"); | |
1870 | int r; | |
1871 | coll_t cid; | |
1872 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1873 | { | |
1874 | ObjectStore::Transaction t; | |
1875 | t.create_collection(cid, 0); | |
1876 | cerr << "Creating collection " << cid << std::endl; | |
1877 | r = apply_transaction(store, &osr, std::move(t)); | |
1878 | ASSERT_EQ(r, 0); | |
1879 | } | |
1880 | { | |
1881 | ObjectStore::Transaction t; | |
1882 | t.touch(cid, a); | |
1883 | t.truncate(cid, a, 3000); | |
1884 | r = apply_transaction(store, &osr, std::move(t)); | |
1885 | ASSERT_EQ(r, 0); | |
1886 | } | |
1887 | { | |
1888 | bufferlist bl; | |
1889 | bufferptr bp(4096); | |
1890 | memset(bp.c_str(), 1, 4096); | |
1891 | bl.append(bp); | |
1892 | ObjectStore::Transaction t; | |
1893 | t.write(cid, a, 4096, 4096, bl); | |
1894 | r = apply_transaction(store, &osr, std::move(t)); | |
1895 | ASSERT_EQ(r, 0); | |
1896 | } | |
1897 | { | |
1898 | bufferlist bl; | |
1899 | ASSERT_EQ(8192, store->read(cid, a, 0, 8192, bl)); | |
1900 | for (unsigned i=0; i<4096; ++i) | |
1901 | ASSERT_EQ(0, bl[i]); | |
1902 | for (unsigned i=4096; i<8192; ++i) | |
1903 | ASSERT_EQ(1, bl[i]); | |
1904 | } | |
1905 | { | |
1906 | ObjectStore::Transaction t; | |
1907 | t.remove(cid, a); | |
1908 | t.remove_collection(cid); | |
1909 | cerr << "Cleaning" << std::endl; | |
1910 | r = apply_transaction(store, &osr, std::move(t)); | |
1911 | ASSERT_EQ(r, 0); | |
1912 | } | |
1913 | } | |
1914 | ||
1915 | TEST_P(StoreTest, AppendDeferredVsTailCache) { | |
1916 | ObjectStore::Sequencer osr("test"); | |
1917 | int r; | |
1918 | coll_t cid; | |
1919 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
1920 | { | |
1921 | ObjectStore::Transaction t; | |
1922 | t.create_collection(cid, 0); | |
1923 | cerr << "Creating collection " << cid << std::endl; | |
1924 | r = store->apply_transaction(&osr, std::move(t)); | |
1925 | ASSERT_EQ(r, 0); | |
1926 | } | |
1927 | unsigned min_alloc = g_conf->bluestore_min_alloc_size; | |
1928 | g_conf->set_val("bluestore_inject_deferred_apply_delay", "1.0"); | |
1929 | g_ceph_context->_conf->apply_changes(NULL); | |
1930 | unsigned size = min_alloc / 3; | |
1931 | bufferptr bpa(size); | |
1932 | memset(bpa.c_str(), 1, bpa.length()); | |
1933 | bufferlist bla; | |
1934 | bla.append(bpa); | |
1935 | { | |
1936 | ObjectStore::Transaction t; | |
1937 | t.write(cid, a, 0, bla.length(), bla, 0); | |
1938 | r = store->apply_transaction(&osr, std::move(t)); | |
1939 | ASSERT_EQ(r, 0); | |
1940 | } | |
1941 | ||
1942 | // force cached tail to clear ... | |
1943 | { | |
1944 | int r = store->umount(); | |
1945 | ASSERT_EQ(0, r); | |
1946 | r = store->mount(); | |
1947 | ASSERT_EQ(0, r); | |
1948 | } | |
1949 | ||
1950 | bufferptr bpb(size); | |
1951 | memset(bpb.c_str(), 2, bpb.length()); | |
1952 | bufferlist blb; | |
1953 | blb.append(bpb); | |
1954 | { | |
1955 | ObjectStore::Transaction t; | |
1956 | t.write(cid, a, bla.length(), blb.length(), blb, 0); | |
1957 | r = store->apply_transaction(&osr, std::move(t)); | |
1958 | ASSERT_EQ(r, 0); | |
1959 | } | |
1960 | bufferptr bpc(size); | |
1961 | memset(bpc.c_str(), 3, bpc.length()); | |
1962 | bufferlist blc; | |
1963 | blc.append(bpc); | |
1964 | { | |
1965 | ObjectStore::Transaction t; | |
1966 | t.write(cid, a, bla.length() + blb.length(), blc.length(), blc, 0); | |
1967 | r = store->apply_transaction(&osr, std::move(t)); | |
1968 | ASSERT_EQ(r, 0); | |
1969 | } | |
1970 | bufferlist final; | |
1971 | final.append(bla); | |
1972 | final.append(blb); | |
1973 | final.append(blc); | |
1974 | bufferlist actual; | |
1975 | { | |
1976 | ASSERT_EQ((int)final.length(), | |
1977 | store->read(cid, a, 0, final.length(), actual)); | |
1978 | ASSERT_TRUE(bl_eq(final, actual)); | |
1979 | } | |
1980 | { | |
1981 | ObjectStore::Transaction t; | |
1982 | t.remove(cid, a); | |
1983 | t.remove_collection(cid); | |
1984 | cerr << "Cleaning" << std::endl; | |
1985 | r = store->apply_transaction(&osr, std::move(t)); | |
1986 | ASSERT_EQ(r, 0); | |
1987 | } | |
1988 | g_conf->set_val("bluestore_inject_deferred_apply_delay", "0"); | |
1989 | g_ceph_context->_conf->apply_changes(NULL); | |
1990 | } | |
1991 | ||
1992 | TEST_P(StoreTest, AppendZeroTrailingSharedBlock) { | |
1993 | ObjectStore::Sequencer osr("test"); | |
1994 | int r; | |
1995 | coll_t cid; | |
1996 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
1997 | ghobject_t b = a; | |
1998 | b.hobj.snap = 1; | |
1999 | { | |
2000 | ObjectStore::Transaction t; | |
2001 | t.create_collection(cid, 0); | |
2002 | cerr << "Creating collection " << cid << std::endl; | |
2003 | r = store->apply_transaction(&osr, std::move(t)); | |
2004 | ASSERT_EQ(r, 0); | |
2005 | } | |
2006 | unsigned min_alloc = g_conf->bluestore_min_alloc_size; | |
2007 | unsigned size = min_alloc / 3; | |
2008 | bufferptr bpa(size); | |
2009 | memset(bpa.c_str(), 1, bpa.length()); | |
2010 | bufferlist bla; | |
2011 | bla.append(bpa); | |
2012 | // make sure there is some trailing gunk in the last block | |
2013 | { | |
2014 | bufferlist bt; | |
2015 | bt.append(bla); | |
2016 | bt.append("BADBADBADBAD"); | |
2017 | ObjectStore::Transaction t; | |
2018 | t.write(cid, a, 0, bt.length(), bt, 0); | |
2019 | r = store->apply_transaction(&osr, std::move(t)); | |
2020 | ASSERT_EQ(r, 0); | |
2021 | } | |
2022 | { | |
2023 | ObjectStore::Transaction t; | |
2024 | t.truncate(cid, a, size); | |
2025 | r = store->apply_transaction(&osr, std::move(t)); | |
2026 | ASSERT_EQ(r, 0); | |
2027 | } | |
2028 | ||
2029 | // clone | |
2030 | { | |
2031 | ObjectStore::Transaction t; | |
2032 | t.clone(cid, a, b); | |
2033 | r = store->apply_transaction(&osr, std::move(t)); | |
2034 | ASSERT_EQ(r, 0); | |
2035 | } | |
2036 | ||
2037 | // append with implicit zeroing | |
2038 | bufferptr bpb(size); | |
2039 | memset(bpb.c_str(), 2, bpb.length()); | |
2040 | bufferlist blb; | |
2041 | blb.append(bpb); | |
2042 | { | |
2043 | ObjectStore::Transaction t; | |
2044 | t.write(cid, a, min_alloc * 3, blb.length(), blb, 0); | |
2045 | r = store->apply_transaction(&osr, std::move(t)); | |
2046 | ASSERT_EQ(r, 0); | |
2047 | } | |
2048 | bufferlist final; | |
2049 | final.append(bla); | |
2050 | bufferlist zeros; | |
2051 | zeros.append_zero(min_alloc * 3 - size); | |
2052 | final.append(zeros); | |
2053 | final.append(blb); | |
2054 | bufferlist actual; | |
2055 | { | |
2056 | ASSERT_EQ((int)final.length(), | |
2057 | store->read(cid, a, 0, final.length(), actual)); | |
2058 | final.hexdump(cout); | |
2059 | actual.hexdump(cout); | |
2060 | ASSERT_TRUE(bl_eq(final, actual)); | |
2061 | } | |
2062 | { | |
2063 | ObjectStore::Transaction t; | |
2064 | t.remove(cid, a); | |
2065 | t.remove(cid, b); | |
2066 | t.remove_collection(cid); | |
2067 | cerr << "Cleaning" << std::endl; | |
2068 | r = store->apply_transaction(&osr, std::move(t)); | |
2069 | ASSERT_EQ(r, 0); | |
2070 | } | |
2071 | } | |
2072 | ||
2073 | TEST_P(StoreTest, SmallSequentialUnaligned) { | |
2074 | ObjectStore::Sequencer osr("test"); | |
2075 | int r; | |
2076 | coll_t cid; | |
2077 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2078 | { | |
2079 | ObjectStore::Transaction t; | |
2080 | t.create_collection(cid, 0); | |
2081 | cerr << "Creating collection " << cid << std::endl; | |
2082 | r = apply_transaction(store, &osr, std::move(t)); | |
2083 | ASSERT_EQ(r, 0); | |
2084 | } | |
2085 | bufferlist bl; | |
2086 | int len = 1000; | |
2087 | bufferptr bp(len); | |
2088 | bp.zero(); | |
2089 | bl.append(bp); | |
2090 | for (int i=0; i<1000; ++i) { | |
2091 | ObjectStore::Transaction t; | |
2092 | t.write(cid, a, i*len, len, bl, 0); | |
2093 | r = apply_transaction(store, &osr, std::move(t)); | |
2094 | ASSERT_EQ(r, 0); | |
2095 | } | |
2096 | { | |
2097 | ObjectStore::Transaction t; | |
2098 | t.remove(cid, a); | |
2099 | t.remove_collection(cid); | |
2100 | cerr << "Cleaning" << std::endl; | |
2101 | r = apply_transaction(store, &osr, std::move(t)); | |
2102 | ASSERT_EQ(r, 0); | |
2103 | } | |
2104 | } | |
2105 | ||
2106 | TEST_P(StoreTest, ManyBigWrite) { | |
2107 | ObjectStore::Sequencer osr("test"); | |
2108 | int r; | |
2109 | coll_t cid; | |
2110 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2111 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
2112 | { | |
2113 | ObjectStore::Transaction t; | |
2114 | t.create_collection(cid, 0); | |
2115 | cerr << "Creating collection " << cid << std::endl; | |
2116 | r = apply_transaction(store, &osr, std::move(t)); | |
2117 | ASSERT_EQ(r, 0); | |
2118 | } | |
2119 | bufferlist bl; | |
2120 | bufferptr bp(4 * 1048576); | |
2121 | bp.zero(); | |
2122 | bl.append(bp); | |
2123 | for (int i=0; i<10; ++i) { | |
2124 | ObjectStore::Transaction t; | |
2125 | t.write(cid, a, i*4*1048586, 4*1048576, bl, 0); | |
2126 | r = apply_transaction(store, &osr, std::move(t)); | |
2127 | ASSERT_EQ(r, 0); | |
2128 | } | |
2129 | // aligned | |
2130 | for (int i=0; i<10; ++i) { | |
2131 | ObjectStore::Transaction t; | |
2132 | t.write(cid, b, (rand() % 256)*4*1048576, 4*1048576, bl, 0); | |
2133 | r = apply_transaction(store, &osr, std::move(t)); | |
2134 | ASSERT_EQ(r, 0); | |
2135 | } | |
2136 | // unaligned | |
2137 | for (int i=0; i<10; ++i) { | |
2138 | ObjectStore::Transaction t; | |
2139 | t.write(cid, b, (rand() % (256*4096))*1024, 4*1048576, bl, 0); | |
2140 | r = apply_transaction(store, &osr, std::move(t)); | |
2141 | ASSERT_EQ(r, 0); | |
2142 | } | |
2143 | // do some zeros | |
2144 | for (int i=0; i<10; ++i) { | |
2145 | ObjectStore::Transaction t; | |
2146 | t.zero(cid, b, (rand() % (256*4096))*1024, 16*1048576); | |
2147 | r = apply_transaction(store, &osr, std::move(t)); | |
2148 | ASSERT_EQ(r, 0); | |
2149 | } | |
2150 | { | |
2151 | ObjectStore::Transaction t; | |
2152 | t.remove(cid, a); | |
2153 | t.remove(cid, b); | |
2154 | t.remove_collection(cid); | |
2155 | cerr << "Cleaning" << std::endl; | |
2156 | r = apply_transaction(store, &osr, std::move(t)); | |
2157 | ASSERT_EQ(r, 0); | |
2158 | } | |
2159 | } | |
2160 | ||
2161 | TEST_P(StoreTest, BigWriteBigZero) { | |
2162 | ObjectStore::Sequencer osr("test"); | |
2163 | int r; | |
2164 | coll_t cid; | |
2165 | ghobject_t a(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
2166 | { | |
2167 | ObjectStore::Transaction t; | |
2168 | t.create_collection(cid, 0); | |
2169 | r = apply_transaction(store, &osr, std::move(t)); | |
2170 | ASSERT_EQ(r, 0); | |
2171 | } | |
2172 | bufferlist bl; | |
2173 | bufferptr bp(1048576); | |
2174 | memset(bp.c_str(), 'b', bp.length()); | |
2175 | bl.append(bp); | |
2176 | bufferlist s; | |
2177 | bufferptr sp(4096); | |
2178 | memset(sp.c_str(), 's', sp.length()); | |
2179 | s.append(sp); | |
2180 | { | |
2181 | ObjectStore::Transaction t; | |
2182 | t.write(cid, a, 0, bl.length(), bl); | |
2183 | r = apply_transaction(store, &osr, std::move(t)); | |
2184 | ASSERT_EQ(r, 0); | |
2185 | } | |
2186 | { | |
2187 | ObjectStore::Transaction t; | |
2188 | t.zero(cid, a, bl.length() / 4, bl.length() / 2); | |
2189 | r = apply_transaction(store, &osr, std::move(t)); | |
2190 | ASSERT_EQ(r, 0); | |
2191 | } | |
2192 | { | |
2193 | ObjectStore::Transaction t; | |
2194 | t.write(cid, a, bl.length() / 2, s.length(), s); | |
2195 | r = apply_transaction(store, &osr, std::move(t)); | |
2196 | ASSERT_EQ(r, 0); | |
2197 | } | |
2198 | { | |
2199 | ObjectStore::Transaction t; | |
2200 | t.remove(cid, a); | |
2201 | t.remove_collection(cid); | |
2202 | r = apply_transaction(store, &osr, std::move(t)); | |
2203 | ASSERT_EQ(r, 0); | |
2204 | } | |
2205 | } | |
2206 | ||
2207 | TEST_P(StoreTest, MiscFragmentTests) { | |
2208 | ObjectStore::Sequencer osr("test"); | |
2209 | int r; | |
2210 | coll_t cid; | |
2211 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2212 | { | |
2213 | ObjectStore::Transaction t; | |
2214 | t.create_collection(cid, 0); | |
2215 | cerr << "Creating collection " << cid << std::endl; | |
2216 | r = apply_transaction(store, &osr, std::move(t)); | |
2217 | ASSERT_EQ(r, 0); | |
2218 | } | |
2219 | bufferlist bl; | |
2220 | bufferptr bp(524288); | |
2221 | bp.zero(); | |
2222 | bl.append(bp); | |
2223 | { | |
2224 | ObjectStore::Transaction t; | |
2225 | t.write(cid, a, 0, 524288, bl, 0); | |
2226 | r = apply_transaction(store, &osr, std::move(t)); | |
2227 | ASSERT_EQ(r, 0); | |
2228 | } | |
2229 | { | |
2230 | ObjectStore::Transaction t; | |
2231 | t.write(cid, a, 1048576, 524288, bl, 0); | |
2232 | r = apply_transaction(store, &osr, std::move(t)); | |
2233 | ASSERT_EQ(r, 0); | |
2234 | } | |
2235 | { | |
2236 | bufferlist inbl; | |
2237 | int r = store->read(cid, a, 524288 + 131072, 1024, inbl); | |
2238 | ASSERT_EQ(r, 1024); | |
2239 | ASSERT_EQ(inbl.length(), 1024u); | |
2240 | ASSERT_TRUE(inbl.is_zero()); | |
2241 | } | |
2242 | { | |
2243 | ObjectStore::Transaction t; | |
2244 | t.write(cid, a, 1048576 - 4096, 524288, bl, 0); | |
2245 | r = apply_transaction(store, &osr, std::move(t)); | |
2246 | ASSERT_EQ(r, 0); | |
2247 | } | |
2248 | { | |
2249 | ObjectStore::Transaction t; | |
2250 | t.remove(cid, a); | |
2251 | t.remove_collection(cid); | |
2252 | cerr << "Cleaning" << std::endl; | |
2253 | r = apply_transaction(store, &osr, std::move(t)); | |
2254 | ASSERT_EQ(r, 0); | |
2255 | } | |
2256 | ||
2257 | } | |
2258 | ||
2259 | TEST_P(StoreTest, ZeroVsObjectSize) { | |
2260 | ObjectStore::Sequencer osr("test"); | |
2261 | int r; | |
2262 | coll_t cid; | |
2263 | struct stat stat; | |
2264 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
2265 | { | |
2266 | ObjectStore::Transaction t; | |
2267 | t.create_collection(cid, 0); | |
2268 | cerr << "Creating collection " << cid << std::endl; | |
2269 | r = apply_transaction(store, &osr, std::move(t)); | |
2270 | ASSERT_EQ(r, 0); | |
2271 | } | |
2272 | bufferlist a; | |
2273 | a.append("stuff"); | |
2274 | { | |
2275 | ObjectStore::Transaction t; | |
2276 | t.write(cid, hoid, 0, 5, a); | |
2277 | r = apply_transaction(store, &osr, std::move(t)); | |
2278 | ASSERT_EQ(r, 0); | |
2279 | } | |
2280 | ASSERT_EQ(0, store->stat(cid, hoid, &stat)); | |
2281 | ASSERT_EQ(5, stat.st_size); | |
2282 | { | |
2283 | ObjectStore::Transaction t; | |
2284 | t.zero(cid, hoid, 1, 2); | |
2285 | r = apply_transaction(store, &osr, std::move(t)); | |
2286 | ASSERT_EQ(r, 0); | |
2287 | } | |
2288 | ASSERT_EQ(0, store->stat(cid, hoid, &stat)); | |
2289 | ASSERT_EQ(5, stat.st_size); | |
2290 | { | |
2291 | ObjectStore::Transaction t; | |
2292 | t.zero(cid, hoid, 3, 200); | |
2293 | r = apply_transaction(store, &osr, std::move(t)); | |
2294 | ASSERT_EQ(r, 0); | |
2295 | } | |
2296 | ASSERT_EQ(0, store->stat(cid, hoid, &stat)); | |
2297 | ASSERT_EQ(203, stat.st_size); | |
2298 | { | |
2299 | ObjectStore::Transaction t; | |
2300 | t.zero(cid, hoid, 100000, 200); | |
2301 | r = apply_transaction(store, &osr, std::move(t)); | |
2302 | ASSERT_EQ(r, 0); | |
2303 | } | |
2304 | ASSERT_EQ(0, store->stat(cid, hoid, &stat)); | |
2305 | ASSERT_EQ(100200, stat.st_size); | |
2306 | } | |
2307 | ||
2308 | TEST_P(StoreTest, ZeroLengthWrite) { | |
2309 | ObjectStore::Sequencer osr("test"); | |
2310 | int r; | |
2311 | coll_t cid; | |
2312 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
2313 | { | |
2314 | ObjectStore::Transaction t; | |
2315 | t.create_collection(cid, 0); | |
2316 | t.touch(cid, hoid); | |
2317 | r = apply_transaction(store, &osr, std::move(t)); | |
2318 | ASSERT_EQ(r, 0); | |
2319 | } | |
2320 | { | |
2321 | ObjectStore::Transaction t; | |
2322 | bufferlist empty; | |
2323 | t.write(cid, hoid, 1048576, 0, empty); | |
2324 | r = apply_transaction(store, &osr, std::move(t)); | |
2325 | ASSERT_EQ(r, 0); | |
2326 | } | |
2327 | struct stat stat; | |
2328 | r = store->stat(cid, hoid, &stat); | |
2329 | ASSERT_EQ(0, r); | |
2330 | ASSERT_EQ(0, stat.st_size); | |
2331 | ||
2332 | bufferlist newdata; | |
2333 | r = store->read(cid, hoid, 0, 1048576, newdata); | |
2334 | ASSERT_EQ(0, r); | |
2335 | } | |
2336 | ||
2337 | TEST_P(StoreTest, ZeroLengthZero) { | |
2338 | ObjectStore::Sequencer osr("test"); | |
2339 | int r; | |
2340 | coll_t cid; | |
2341 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
2342 | { | |
2343 | ObjectStore::Transaction t; | |
2344 | t.create_collection(cid, 0); | |
2345 | t.touch(cid, hoid); | |
2346 | r = apply_transaction(store, &osr, std::move(t)); | |
2347 | ASSERT_EQ(0, r); | |
2348 | } | |
2349 | { | |
2350 | ObjectStore::Transaction t; | |
2351 | t.zero(cid, hoid, 1048576, 0); | |
2352 | r = apply_transaction(store, &osr, std::move(t)); | |
2353 | ASSERT_EQ(0, r); | |
2354 | } | |
2355 | struct stat stat; | |
2356 | r = store->stat(cid, hoid, &stat); | |
2357 | ASSERT_EQ(0, r); | |
2358 | ASSERT_EQ(0, stat.st_size); | |
2359 | ||
2360 | bufferlist newdata; | |
2361 | r = store->read(cid, hoid, 0, 1048576, newdata); | |
2362 | ASSERT_EQ(0, r); | |
2363 | } | |
2364 | ||
2365 | TEST_P(StoreTest, SimpleAttrTest) { | |
2366 | ObjectStore::Sequencer osr("test"); | |
2367 | int r; | |
2368 | coll_t cid; | |
2369 | ghobject_t hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP))); | |
2370 | bufferlist val, val2; | |
2371 | val.append("value"); | |
2372 | val.append("value2"); | |
2373 | { | |
2374 | bufferptr bp; | |
2375 | map<string,bufferptr> aset; | |
2376 | r = store->getattr(cid, hoid, "nofoo", bp); | |
2377 | ASSERT_EQ(-ENOENT, r); | |
2378 | r = store->getattrs(cid, hoid, aset); | |
2379 | ASSERT_EQ(-ENOENT, r); | |
2380 | } | |
2381 | { | |
2382 | ObjectStore::Transaction t; | |
2383 | t.create_collection(cid, 0); | |
2384 | r = apply_transaction(store, &osr, std::move(t)); | |
2385 | ASSERT_EQ(r, 0); | |
2386 | } | |
2387 | osr.flush(); | |
2388 | { | |
2389 | bool empty; | |
2390 | int r = store->collection_empty(cid, &empty); | |
2391 | ASSERT_EQ(0, r); | |
2392 | ASSERT_TRUE(empty); | |
2393 | } | |
2394 | { | |
2395 | bufferptr bp; | |
2396 | r = store->getattr(cid, hoid, "nofoo", bp); | |
2397 | ASSERT_EQ(-ENOENT, r); | |
2398 | } | |
2399 | { | |
2400 | ObjectStore::Transaction t; | |
2401 | t.touch(cid, hoid); | |
2402 | t.setattr(cid, hoid, "foo", val); | |
2403 | t.setattr(cid, hoid, "bar", val2); | |
2404 | r = apply_transaction(store, &osr, std::move(t)); | |
2405 | ASSERT_EQ(r, 0); | |
2406 | } | |
2407 | osr.flush(); | |
2408 | { | |
2409 | bool empty; | |
2410 | int r = store->collection_empty(cid, &empty); | |
2411 | ASSERT_EQ(0, r); | |
2412 | ASSERT_TRUE(!empty); | |
2413 | } | |
2414 | { | |
2415 | bufferptr bp; | |
2416 | r = store->getattr(cid, hoid, "nofoo", bp); | |
2417 | ASSERT_EQ(-ENODATA, r); | |
2418 | ||
2419 | r = store->getattr(cid, hoid, "foo", bp); | |
2420 | ASSERT_EQ(0, r); | |
2421 | bufferlist bl; | |
2422 | bl.append(bp); | |
2423 | ASSERT_TRUE(bl_eq(val, bl)); | |
2424 | ||
2425 | map<string,bufferptr> bm; | |
2426 | r = store->getattrs(cid, hoid, bm); | |
2427 | ASSERT_EQ(0, r); | |
2428 | ||
2429 | } | |
2430 | { | |
2431 | ObjectStore::Transaction t; | |
2432 | t.remove(cid, hoid); | |
2433 | t.remove_collection(cid); | |
2434 | r = apply_transaction(store, &osr, std::move(t)); | |
2435 | ASSERT_EQ(r, 0); | |
2436 | } | |
2437 | } | |
2438 | ||
2439 | TEST_P(StoreTest, SimpleListTest) { | |
2440 | ObjectStore::Sequencer osr("test"); | |
2441 | int r; | |
2442 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
2443 | { | |
2444 | ObjectStore::Transaction t; | |
2445 | t.create_collection(cid, 0); | |
2446 | cerr << "Creating collection " << cid << std::endl; | |
2447 | r = apply_transaction(store, &osr, std::move(t)); | |
2448 | ASSERT_EQ(r, 0); | |
2449 | } | |
2450 | set<ghobject_t> all; | |
2451 | { | |
2452 | ObjectStore::Transaction t; | |
2453 | for (int i=0; i<200; ++i) { | |
2454 | string name("object_"); | |
2455 | name += stringify(i); | |
2456 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2457 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2458 | hoid.hobj.pool = 1; | |
2459 | all.insert(hoid); | |
2460 | t.touch(cid, hoid); | |
2461 | cerr << "Creating object " << hoid << std::endl; | |
2462 | } | |
2463 | r = apply_transaction(store, &osr, std::move(t)); | |
2464 | ASSERT_EQ(r, 0); | |
2465 | } | |
2466 | { | |
2467 | set<ghobject_t> saw; | |
2468 | vector<ghobject_t> objects; | |
2469 | ghobject_t next, current; | |
2470 | while (!next.is_max()) { | |
2471 | int r = store->collection_list(cid, current, ghobject_t::get_max(), | |
2472 | 50, | |
2473 | &objects, &next); | |
2474 | ASSERT_EQ(r, 0); | |
2475 | ASSERT_TRUE(sorted(objects)); | |
2476 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2477 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2478 | ++p) { | |
2479 | if (saw.count(*p)) { | |
2480 | cout << "got DUP " << *p << std::endl; | |
2481 | } else { | |
2482 | //cout << "got new " << *p << std::endl; | |
2483 | } | |
2484 | saw.insert(*p); | |
2485 | } | |
2486 | objects.clear(); | |
2487 | current = next; | |
2488 | } | |
2489 | ASSERT_EQ(saw.size(), all.size()); | |
2490 | ASSERT_EQ(saw, all); | |
2491 | } | |
2492 | { | |
2493 | ObjectStore::Transaction t; | |
2494 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2495 | t.remove(cid, *p); | |
2496 | t.remove_collection(cid); | |
2497 | cerr << "Cleaning" << std::endl; | |
2498 | r = apply_transaction(store, &osr, std::move(t)); | |
2499 | ASSERT_EQ(r, 0); | |
2500 | } | |
2501 | } | |
2502 | ||
2503 | TEST_P(StoreTest, ListEndTest) { | |
2504 | ObjectStore::Sequencer osr("test"); | |
2505 | int r; | |
2506 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
2507 | { | |
2508 | ObjectStore::Transaction t; | |
2509 | t.create_collection(cid, 0); | |
2510 | cerr << "Creating collection " << cid << std::endl; | |
2511 | r = apply_transaction(store, &osr, std::move(t)); | |
2512 | ASSERT_EQ(r, 0); | |
2513 | } | |
2514 | set<ghobject_t> all; | |
2515 | { | |
2516 | ObjectStore::Transaction t; | |
2517 | for (int i=0; i<200; ++i) { | |
2518 | string name("object_"); | |
2519 | name += stringify(i); | |
2520 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2521 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2522 | hoid.hobj.pool = 1; | |
2523 | all.insert(hoid); | |
2524 | t.touch(cid, hoid); | |
2525 | cerr << "Creating object " << hoid << std::endl; | |
2526 | } | |
2527 | r = apply_transaction(store, &osr, std::move(t)); | |
2528 | ASSERT_EQ(r, 0); | |
2529 | } | |
2530 | { | |
2531 | ghobject_t end(hobject_t(sobject_t("object_100", CEPH_NOSNAP)), | |
2532 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2533 | end.hobj.pool = 1; | |
2534 | vector<ghobject_t> objects; | |
2535 | ghobject_t next; | |
2536 | int r = store->collection_list(cid, ghobject_t(), end, 500, | |
2537 | &objects, &next); | |
2538 | ASSERT_EQ(r, 0); | |
2539 | for (auto &p : objects) { | |
2540 | ASSERT_NE(p, end); | |
2541 | } | |
2542 | } | |
2543 | { | |
2544 | ObjectStore::Transaction t; | |
2545 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2546 | t.remove(cid, *p); | |
2547 | t.remove_collection(cid); | |
2548 | cerr << "Cleaning" << std::endl; | |
2549 | r = apply_transaction(store, &osr, std::move(t)); | |
2550 | ASSERT_EQ(r, 0); | |
2551 | } | |
2552 | } | |
2553 | ||
2554 | TEST_P(StoreTest, Sort) { | |
2555 | { | |
2556 | hobject_t a(sobject_t("a", CEPH_NOSNAP)); | |
2557 | hobject_t b = a; | |
2558 | ASSERT_EQ(a, b); | |
2559 | b.oid.name = "b"; | |
2560 | ASSERT_NE(a, b); | |
2561 | ASSERT_TRUE(a < b); | |
2562 | a.pool = 1; | |
2563 | b.pool = 2; | |
2564 | ASSERT_TRUE(a < b); | |
2565 | a.pool = 3; | |
2566 | ASSERT_TRUE(a > b); | |
2567 | } | |
2568 | { | |
2569 | ghobject_t a(hobject_t(sobject_t("a", CEPH_NOSNAP))); | |
2570 | ghobject_t b(hobject_t(sobject_t("b", CEPH_NOSNAP))); | |
2571 | a.hobj.pool = 1; | |
2572 | b.hobj.pool = 1; | |
2573 | ASSERT_TRUE(a < b); | |
2574 | a.hobj.pool = -3; | |
2575 | ASSERT_TRUE(a < b); | |
2576 | a.hobj.pool = 1; | |
2577 | b.hobj.pool = -3; | |
2578 | ASSERT_TRUE(a > b); | |
2579 | } | |
2580 | } | |
2581 | ||
2582 | TEST_P(StoreTest, MultipoolListTest) { | |
2583 | ObjectStore::Sequencer osr("test"); | |
2584 | int r; | |
2585 | int poolid = 4373; | |
2586 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
2587 | { | |
2588 | ObjectStore::Transaction t; | |
2589 | t.create_collection(cid, 0); | |
2590 | cerr << "Creating collection " << cid << std::endl; | |
2591 | r = apply_transaction(store, &osr, std::move(t)); | |
2592 | ASSERT_EQ(r, 0); | |
2593 | } | |
2594 | set<ghobject_t> all, saw; | |
2595 | { | |
2596 | ObjectStore::Transaction t; | |
2597 | for (int i=0; i<200; ++i) { | |
2598 | string name("object_"); | |
2599 | name += stringify(i); | |
2600 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
2601 | if (rand() & 1) | |
2602 | hoid.hobj.pool = -2 - poolid; | |
2603 | else | |
2604 | hoid.hobj.pool = poolid; | |
2605 | all.insert(hoid); | |
2606 | t.touch(cid, hoid); | |
2607 | cerr << "Creating object " << hoid << std::endl; | |
2608 | } | |
2609 | r = apply_transaction(store, &osr, std::move(t)); | |
2610 | ASSERT_EQ(r, 0); | |
2611 | } | |
2612 | { | |
2613 | vector<ghobject_t> objects; | |
2614 | ghobject_t next, current; | |
2615 | while (!next.is_max()) { | |
2616 | int r = store->collection_list(cid, current, ghobject_t::get_max(), 50, | |
2617 | &objects, &next); | |
2618 | ASSERT_EQ(r, 0); | |
2619 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2620 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2621 | ++p) { | |
2622 | saw.insert(*p); | |
2623 | } | |
2624 | objects.clear(); | |
2625 | current = next; | |
2626 | } | |
2627 | ASSERT_EQ(saw, all); | |
2628 | } | |
2629 | { | |
2630 | ObjectStore::Transaction t; | |
2631 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2632 | t.remove(cid, *p); | |
2633 | t.remove_collection(cid); | |
2634 | cerr << "Cleaning" << std::endl; | |
2635 | r = apply_transaction(store, &osr, std::move(t)); | |
2636 | ASSERT_EQ(r, 0); | |
2637 | } | |
2638 | } | |
2639 | ||
2640 | TEST_P(StoreTest, SimpleCloneTest) { | |
2641 | ObjectStore::Sequencer osr("test"); | |
2642 | int r; | |
2643 | coll_t cid; | |
2644 | { | |
2645 | ObjectStore::Transaction t; | |
2646 | t.create_collection(cid, 0); | |
2647 | cerr << "Creating collection " << cid << std::endl; | |
2648 | r = apply_transaction(store, &osr, std::move(t)); | |
2649 | ASSERT_EQ(r, 0); | |
2650 | } | |
2651 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
2652 | "key", 123, -1, "")); | |
2653 | bufferlist small, large, xlarge, newdata, attr; | |
2654 | small.append("small"); | |
2655 | large.append("large"); | |
2656 | xlarge.append("xlarge"); | |
2657 | { | |
2658 | ObjectStore::Transaction t; | |
2659 | t.touch(cid, hoid); | |
2660 | t.setattr(cid, hoid, "attr1", small); | |
2661 | t.setattr(cid, hoid, "attr2", large); | |
2662 | t.setattr(cid, hoid, "attr3", xlarge); | |
2663 | t.write(cid, hoid, 0, small.length(), small); | |
2664 | t.write(cid, hoid, 10, small.length(), small); | |
2665 | cerr << "Creating object and set attr " << hoid << std::endl; | |
2666 | r = apply_transaction(store, &osr, std::move(t)); | |
2667 | ASSERT_EQ(r, 0); | |
2668 | } | |
2669 | ||
2670 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
2671 | "key", 123, -1, "")); | |
2672 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP))); | |
2673 | { | |
2674 | ObjectStore::Transaction t; | |
2675 | t.clone(cid, hoid, hoid2); | |
2676 | t.setattr(cid, hoid2, "attr2", small); | |
2677 | t.rmattr(cid, hoid2, "attr1"); | |
2678 | t.write(cid, hoid, 10, large.length(), large); | |
2679 | t.setattr(cid, hoid, "attr1", large); | |
2680 | t.setattr(cid, hoid, "attr2", small); | |
2681 | cerr << "Clone object and rm attr" << std::endl; | |
2682 | r = apply_transaction(store, &osr, std::move(t)); | |
2683 | ASSERT_EQ(r, 0); | |
2684 | ||
2685 | r = store->read(cid, hoid, 10, 5, newdata); | |
2686 | ASSERT_EQ(r, 5); | |
2687 | ASSERT_TRUE(bl_eq(large, newdata)); | |
2688 | ||
2689 | newdata.clear(); | |
2690 | r = store->read(cid, hoid, 0, 5, newdata); | |
2691 | ASSERT_EQ(r, 5); | |
2692 | ASSERT_TRUE(bl_eq(small, newdata)); | |
2693 | ||
2694 | newdata.clear(); | |
2695 | r = store->read(cid, hoid2, 10, 5, newdata); | |
2696 | ASSERT_EQ(r, 5); | |
2697 | ASSERT_TRUE(bl_eq(small, newdata)); | |
2698 | ||
2699 | r = store->getattr(cid, hoid2, "attr2", attr); | |
2700 | ASSERT_EQ(r, 0); | |
2701 | ASSERT_TRUE(bl_eq(small, attr)); | |
2702 | ||
2703 | attr.clear(); | |
2704 | r = store->getattr(cid, hoid2, "attr3", attr); | |
2705 | ASSERT_EQ(r, 0); | |
2706 | ASSERT_TRUE(bl_eq(xlarge, attr)); | |
2707 | ||
2708 | attr.clear(); | |
2709 | r = store->getattr(cid, hoid, "attr1", attr); | |
2710 | ASSERT_EQ(r, 0); | |
2711 | ASSERT_TRUE(bl_eq(large, attr)); | |
2712 | } | |
2713 | { | |
2714 | ObjectStore::Transaction t; | |
2715 | t.remove(cid, hoid); | |
2716 | t.remove(cid, hoid2); | |
2717 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2718 | } | |
2719 | { | |
2720 | bufferlist final; | |
2721 | bufferptr p(16384); | |
2722 | memset(p.c_str(), 1, p.length()); | |
2723 | bufferlist pl; | |
2724 | pl.append(p); | |
2725 | final.append(p); | |
2726 | ObjectStore::Transaction t; | |
2727 | t.write(cid, hoid, 0, pl.length(), pl); | |
2728 | t.clone(cid, hoid, hoid2); | |
2729 | bufferptr a(4096); | |
2730 | memset(a.c_str(), 2, a.length()); | |
2731 | bufferlist al; | |
2732 | al.append(a); | |
2733 | final.append(a); | |
2734 | t.write(cid, hoid, pl.length(), a.length(), al); | |
2735 | r = apply_transaction(store, &osr, std::move(t)); | |
2736 | ASSERT_EQ(r, 0); | |
2737 | bufferlist rl; | |
2738 | ASSERT_EQ((int)final.length(), | |
2739 | store->read(cid, hoid, 0, final.length(), rl)); | |
2740 | ASSERT_TRUE(bl_eq(rl, final)); | |
2741 | } | |
2742 | { | |
2743 | ObjectStore::Transaction t; | |
2744 | t.remove(cid, hoid); | |
2745 | t.remove(cid, hoid2); | |
2746 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2747 | } | |
2748 | { | |
2749 | bufferlist final; | |
2750 | bufferptr p(16384); | |
2751 | memset(p.c_str(), 111, p.length()); | |
2752 | bufferlist pl; | |
2753 | pl.append(p); | |
2754 | final.append(p); | |
2755 | ObjectStore::Transaction t; | |
2756 | t.write(cid, hoid, 0, pl.length(), pl); | |
2757 | t.clone(cid, hoid, hoid2); | |
2758 | bufferptr z(4096); | |
2759 | z.zero(); | |
2760 | final.append(z); | |
2761 | bufferptr a(4096); | |
2762 | memset(a.c_str(), 112, a.length()); | |
2763 | bufferlist al; | |
2764 | al.append(a); | |
2765 | final.append(a); | |
2766 | t.write(cid, hoid, pl.length() + z.length(), a.length(), al); | |
2767 | r = apply_transaction(store, &osr, std::move(t)); | |
2768 | ASSERT_EQ(r, 0); | |
2769 | bufferlist rl; | |
2770 | ASSERT_EQ((int)final.length(), | |
2771 | store->read(cid, hoid, 0, final.length(), rl)); | |
2772 | ASSERT_TRUE(bl_eq(rl, final)); | |
2773 | } | |
2774 | { | |
2775 | ObjectStore::Transaction t; | |
2776 | t.remove(cid, hoid); | |
2777 | t.remove(cid, hoid2); | |
2778 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2779 | } | |
2780 | { | |
2781 | bufferlist final; | |
2782 | bufferptr p(16000); | |
2783 | memset(p.c_str(), 5, p.length()); | |
2784 | bufferlist pl; | |
2785 | pl.append(p); | |
2786 | final.append(p); | |
2787 | ObjectStore::Transaction t; | |
2788 | t.write(cid, hoid, 0, pl.length(), pl); | |
2789 | t.clone(cid, hoid, hoid2); | |
2790 | bufferptr z(1000); | |
2791 | z.zero(); | |
2792 | final.append(z); | |
2793 | bufferptr a(8000); | |
2794 | memset(a.c_str(), 6, a.length()); | |
2795 | bufferlist al; | |
2796 | al.append(a); | |
2797 | final.append(a); | |
2798 | t.write(cid, hoid, 17000, a.length(), al); | |
2799 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2800 | bufferlist rl; | |
2801 | ASSERT_EQ((int)final.length(), | |
2802 | store->read(cid, hoid, 0, final.length(), rl)); | |
2803 | /*cout << "expected:\n"; | |
2804 | final.hexdump(cout); | |
2805 | cout << "got:\n"; | |
2806 | rl.hexdump(cout);*/ | |
2807 | ASSERT_TRUE(bl_eq(rl, final)); | |
2808 | } | |
2809 | { | |
2810 | ObjectStore::Transaction t; | |
2811 | t.remove(cid, hoid); | |
2812 | t.remove(cid, hoid2); | |
2813 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2814 | } | |
2815 | { | |
2816 | bufferptr p(1048576); | |
2817 | memset(p.c_str(), 3, p.length()); | |
2818 | bufferlist pl; | |
2819 | pl.append(p); | |
2820 | ObjectStore::Transaction t; | |
2821 | t.write(cid, hoid, 0, pl.length(), pl); | |
2822 | t.clone(cid, hoid, hoid2); | |
2823 | bufferptr a(65536); | |
2824 | memset(a.c_str(), 4, a.length()); | |
2825 | bufferlist al; | |
2826 | al.append(a); | |
2827 | t.write(cid, hoid, a.length(), a.length(), al); | |
2828 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2829 | bufferlist rl; | |
2830 | bufferlist final; | |
2831 | final.substr_of(pl, 0, al.length()); | |
2832 | final.append(al); | |
2833 | bufferlist end; | |
2834 | end.substr_of(pl, al.length()*2, pl.length() - al.length()*2); | |
2835 | final.append(end); | |
2836 | ASSERT_EQ((int)final.length(), | |
2837 | store->read(cid, hoid, 0, final.length(), rl)); | |
2838 | /*cout << "expected:\n"; | |
2839 | final.hexdump(cout); | |
2840 | cout << "got:\n"; | |
2841 | rl.hexdump(cout);*/ | |
2842 | ASSERT_TRUE(bl_eq(rl, final)); | |
2843 | } | |
2844 | { | |
2845 | ObjectStore::Transaction t; | |
2846 | t.remove(cid, hoid); | |
2847 | t.remove(cid, hoid2); | |
2848 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2849 | } | |
2850 | { | |
2851 | bufferptr p(65536); | |
2852 | memset(p.c_str(), 7, p.length()); | |
2853 | bufferlist pl; | |
2854 | pl.append(p); | |
2855 | ObjectStore::Transaction t; | |
2856 | t.write(cid, hoid, 0, pl.length(), pl); | |
2857 | t.clone(cid, hoid, hoid2); | |
2858 | bufferptr a(4096); | |
2859 | memset(a.c_str(), 8, a.length()); | |
2860 | bufferlist al; | |
2861 | al.append(a); | |
2862 | t.write(cid, hoid, 32768, a.length(), al); | |
2863 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2864 | bufferlist rl; | |
2865 | bufferlist final; | |
2866 | final.substr_of(pl, 0, 32768); | |
2867 | final.append(al); | |
2868 | bufferlist end; | |
2869 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
2870 | final.append(end); | |
2871 | ASSERT_EQ((int)final.length(), | |
2872 | store->read(cid, hoid, 0, final.length(), rl)); | |
2873 | /*cout << "expected:\n"; | |
2874 | final.hexdump(cout); | |
2875 | cout << "got:\n"; | |
2876 | rl.hexdump(cout);*/ | |
2877 | ASSERT_TRUE(bl_eq(rl, final)); | |
2878 | } | |
2879 | { | |
2880 | ObjectStore::Transaction t; | |
2881 | t.remove(cid, hoid); | |
2882 | t.remove(cid, hoid2); | |
2883 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2884 | } | |
2885 | { | |
2886 | bufferptr p(65536); | |
2887 | memset(p.c_str(), 9, p.length()); | |
2888 | bufferlist pl; | |
2889 | pl.append(p); | |
2890 | ObjectStore::Transaction t; | |
2891 | t.write(cid, hoid, 0, pl.length(), pl); | |
2892 | t.clone(cid, hoid, hoid2); | |
2893 | bufferptr a(4096); | |
2894 | memset(a.c_str(), 10, a.length()); | |
2895 | bufferlist al; | |
2896 | al.append(a); | |
2897 | t.write(cid, hoid, 33768, a.length(), al); | |
2898 | ASSERT_EQ(0, apply_transaction(store, &osr, std::move(t))); | |
2899 | bufferlist rl; | |
2900 | bufferlist final; | |
2901 | final.substr_of(pl, 0, 33768); | |
2902 | final.append(al); | |
2903 | bufferlist end; | |
2904 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
2905 | final.append(end); | |
2906 | ASSERT_EQ((int)final.length(), | |
2907 | store->read(cid, hoid, 0, final.length(), rl)); | |
2908 | /*cout << "expected:\n"; | |
2909 | final.hexdump(cout); | |
2910 | cout << "got:\n"; | |
2911 | rl.hexdump(cout);*/ | |
2912 | ASSERT_TRUE(bl_eq(rl, final)); | |
2913 | } | |
2914 | ||
2915 | //Unfortunately we need a workaround for filestore since EXPECT_DEATH | |
2916 | // macro has potential issues when using /in multithread environments. | |
2917 | //It works well for all stores but filestore for now. | |
2918 | //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well - | |
2919 | // test app clone asserts on store folder presence. | |
2920 | // | |
2921 | if (string(GetParam()) != "filestore") { | |
2922 | //verify if non-empty collection is properly handled after store reload | |
2923 | r = store->umount(); | |
2924 | ASSERT_EQ(r, 0); | |
2925 | r = store->mount(); | |
2926 | ASSERT_EQ(r, 0); | |
2927 | ||
2928 | ObjectStore::Transaction t; | |
2929 | t.remove_collection(cid); | |
2930 | cerr << "Invalid rm coll" << std::endl; | |
2931 | PrCtl unset_dumpable; | |
2932 | EXPECT_DEATH(apply_transaction(store, &osr, std::move(t)), ""); | |
2933 | } | |
2934 | { | |
2935 | ObjectStore::Transaction t; | |
2936 | t.touch(cid, hoid3); //new record in db | |
2937 | r = apply_transaction(store, &osr, std::move(t)); | |
2938 | ASSERT_EQ(r, 0); | |
2939 | } | |
2940 | //See comment above for "filestore" check explanation. | |
2941 | if (string(GetParam()) != "filestore") { | |
2942 | ObjectStore::Transaction t; | |
2943 | //verify if non-empty collection is properly handled when there are some pending removes and live records in db | |
2944 | cerr << "Invalid rm coll again" << std::endl; | |
2945 | r = store->umount(); | |
2946 | ASSERT_EQ(r, 0); | |
2947 | r = store->mount(); | |
2948 | ASSERT_EQ(r, 0); | |
2949 | ||
2950 | t.remove(cid, hoid); | |
2951 | t.remove(cid, hoid2); | |
2952 | t.remove_collection(cid); | |
2953 | PrCtl unset_dumpable; | |
2954 | EXPECT_DEATH(apply_transaction(store, &osr, std::move(t)), ""); | |
2955 | } | |
2956 | { | |
2957 | ObjectStore::Transaction t; | |
2958 | t.remove(cid, hoid); | |
2959 | t.remove(cid, hoid2); | |
2960 | t.remove(cid, hoid3); | |
2961 | t.remove_collection(cid); | |
2962 | cerr << "Cleaning" << std::endl; | |
2963 | r = apply_transaction(store, &osr, std::move(t)); | |
2964 | ASSERT_EQ(r, 0); | |
2965 | } | |
2966 | } | |
2967 | ||
2968 | TEST_P(StoreTest, OmapSimple) { | |
2969 | ObjectStore::Sequencer osr("test"); | |
2970 | int r; | |
2971 | coll_t cid; | |
2972 | { | |
2973 | ObjectStore::Transaction t; | |
2974 | t.create_collection(cid, 0); | |
2975 | cerr << "Creating collection " << cid << std::endl; | |
2976 | r = apply_transaction(store, &osr, std::move(t)); | |
2977 | ASSERT_EQ(r, 0); | |
2978 | } | |
2979 | ghobject_t hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP), | |
2980 | "key", 123, -1, "")); | |
2981 | bufferlist small; | |
2982 | small.append("small"); | |
2983 | map<string,bufferlist> km; | |
2984 | km["foo"] = small; | |
2985 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
2986 | bufferlist header; | |
2987 | header.append("this is a header"); | |
2988 | { | |
2989 | ObjectStore::Transaction t; | |
2990 | t.touch(cid, hoid); | |
2991 | t.omap_setkeys(cid, hoid, km); | |
2992 | t.omap_setheader(cid, hoid, header); | |
2993 | cerr << "Creating object and set omap " << hoid << std::endl; | |
2994 | r = apply_transaction(store, &osr, std::move(t)); | |
2995 | ASSERT_EQ(r, 0); | |
2996 | } | |
2997 | // get header, keys | |
2998 | { | |
2999 | bufferlist h; | |
3000 | map<string,bufferlist> r; | |
3001 | store->omap_get(cid, hoid, &h, &r); | |
3002 | ASSERT_TRUE(bl_eq(header, h)); | |
3003 | ASSERT_EQ(r.size(), km.size()); | |
3004 | cout << "r: " << r << std::endl; | |
3005 | } | |
3006 | // test iterator with seek_to_first | |
3007 | { | |
3008 | map<string,bufferlist> r; | |
3009 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(cid, hoid); | |
3010 | for (iter->seek_to_first(); iter->valid(); iter->next(false)) { | |
3011 | r[iter->key()] = iter->value(); | |
3012 | } | |
3013 | cout << "r: " << r << std::endl; | |
3014 | ASSERT_EQ(r.size(), km.size()); | |
3015 | } | |
3016 | // test iterator with initial lower_bound | |
3017 | { | |
3018 | map<string,bufferlist> r; | |
3019 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(cid, hoid); | |
3020 | for (iter->lower_bound(string()); iter->valid(); iter->next(false)) { | |
3021 | r[iter->key()] = iter->value(); | |
3022 | } | |
3023 | cout << "r: " << r << std::endl; | |
3024 | ASSERT_EQ(r.size(), km.size()); | |
3025 | } | |
3026 | { | |
3027 | ObjectStore::Transaction t; | |
3028 | t.remove(cid, hoid); | |
3029 | t.remove_collection(cid); | |
3030 | cerr << "Cleaning" << std::endl; | |
3031 | r = apply_transaction(store, &osr, std::move(t)); | |
3032 | ASSERT_EQ(r, 0); | |
3033 | } | |
3034 | } | |
3035 | ||
3036 | TEST_P(StoreTest, OmapCloneTest) { | |
3037 | ObjectStore::Sequencer osr("test"); | |
3038 | int r; | |
3039 | coll_t cid; | |
3040 | { | |
3041 | ObjectStore::Transaction t; | |
3042 | t.create_collection(cid, 0); | |
3043 | cerr << "Creating collection " << cid << std::endl; | |
3044 | r = apply_transaction(store, &osr, std::move(t)); | |
3045 | ASSERT_EQ(r, 0); | |
3046 | } | |
3047 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3048 | "key", 123, -1, "")); | |
3049 | bufferlist small; | |
3050 | small.append("small"); | |
3051 | map<string,bufferlist> km; | |
3052 | km["foo"] = small; | |
3053 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3054 | bufferlist header; | |
3055 | header.append("this is a header"); | |
3056 | { | |
3057 | ObjectStore::Transaction t; | |
3058 | t.touch(cid, hoid); | |
3059 | t.omap_setkeys(cid, hoid, km); | |
3060 | t.omap_setheader(cid, hoid, header); | |
3061 | cerr << "Creating object and set omap " << hoid << std::endl; | |
3062 | r = apply_transaction(store, &osr, std::move(t)); | |
3063 | ASSERT_EQ(r, 0); | |
3064 | } | |
3065 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3066 | "key", 123, -1, "")); | |
3067 | { | |
3068 | ObjectStore::Transaction t; | |
3069 | t.clone(cid, hoid, hoid2); | |
3070 | cerr << "Clone object" << std::endl; | |
3071 | r = apply_transaction(store, &osr, std::move(t)); | |
3072 | ASSERT_EQ(r, 0); | |
3073 | } | |
3074 | { | |
3075 | map<string,bufferlist> r; | |
3076 | bufferlist h; | |
3077 | store->omap_get(cid, hoid2, &h, &r); | |
3078 | ASSERT_TRUE(bl_eq(header, h)); | |
3079 | ASSERT_EQ(r.size(), km.size()); | |
3080 | } | |
3081 | { | |
3082 | ObjectStore::Transaction t; | |
3083 | t.remove(cid, hoid); | |
3084 | t.remove(cid, hoid2); | |
3085 | t.remove_collection(cid); | |
3086 | cerr << "Cleaning" << std::endl; | |
3087 | r = apply_transaction(store, &osr, std::move(t)); | |
3088 | ASSERT_EQ(r, 0); | |
3089 | } | |
3090 | } | |
3091 | ||
3092 | TEST_P(StoreTest, SimpleCloneRangeTest) { | |
3093 | ObjectStore::Sequencer osr("test"); | |
3094 | int r; | |
3095 | coll_t cid; | |
3096 | { | |
3097 | ObjectStore::Transaction t; | |
3098 | t.create_collection(cid, 0); | |
3099 | cerr << "Creating collection " << cid << std::endl; | |
3100 | r = apply_transaction(store, &osr, std::move(t)); | |
3101 | ASSERT_EQ(r, 0); | |
3102 | } | |
3103 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3104 | hoid.hobj.pool = -1; | |
3105 | bufferlist small, newdata; | |
3106 | small.append("small"); | |
3107 | { | |
3108 | ObjectStore::Transaction t; | |
3109 | t.write(cid, hoid, 10, 5, small); | |
3110 | cerr << "Creating object and write bl " << hoid << std::endl; | |
3111 | r = apply_transaction(store, &osr, std::move(t)); | |
3112 | ASSERT_EQ(r, 0); | |
3113 | } | |
3114 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
3115 | hoid2.hobj.pool = -1; | |
3116 | { | |
3117 | ObjectStore::Transaction t; | |
3118 | t.clone_range(cid, hoid, hoid2, 10, 5, 10); | |
3119 | cerr << "Clone range object" << std::endl; | |
3120 | r = apply_transaction(store, &osr, std::move(t)); | |
3121 | ASSERT_EQ(r, 0); | |
3122 | r = store->read(cid, hoid2, 10, 5, newdata); | |
3123 | ASSERT_EQ(r, 5); | |
3124 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3125 | } | |
3126 | { | |
3127 | ObjectStore::Transaction t; | |
3128 | t.truncate(cid, hoid, 1024*1024); | |
3129 | t.clone_range(cid, hoid, hoid2, 0, 1024*1024, 0); | |
3130 | cerr << "Clone range object" << std::endl; | |
3131 | r = apply_transaction(store, &osr, std::move(t)); | |
3132 | ASSERT_EQ(r, 0); | |
3133 | struct stat stat, stat2; | |
3134 | r = store->stat(cid, hoid, &stat); | |
3135 | r = store->stat(cid, hoid2, &stat2); | |
3136 | ASSERT_EQ(stat.st_size, stat2.st_size); | |
3137 | ASSERT_EQ(1024*1024, stat2.st_size); | |
3138 | } | |
3139 | { | |
3140 | ObjectStore::Transaction t; | |
3141 | t.remove(cid, hoid); | |
3142 | t.remove(cid, hoid2); | |
3143 | t.remove_collection(cid); | |
3144 | cerr << "Cleaning" << std::endl; | |
3145 | r = apply_transaction(store, &osr, std::move(t)); | |
3146 | ASSERT_EQ(r, 0); | |
3147 | } | |
3148 | } | |
3149 | ||
3150 | ||
3151 | TEST_P(StoreTest, SimpleObjectLongnameTest) { | |
3152 | ObjectStore::Sequencer osr("test"); | |
3153 | int r; | |
3154 | coll_t cid; | |
3155 | { | |
3156 | ObjectStore::Transaction t; | |
3157 | t.create_collection(cid, 0); | |
3158 | cerr << "Creating collection " << cid << std::endl; | |
3159 | r = apply_transaction(store, &osr, std::move(t)); | |
3160 | ASSERT_EQ(r, 0); | |
3161 | } | |
3162 | ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP))); | |
3163 | { | |
3164 | ObjectStore::Transaction t; | |
3165 | t.touch(cid, hoid); | |
3166 | cerr << "Creating object " << hoid << std::endl; | |
3167 | r = apply_transaction(store, &osr, std::move(t)); | |
3168 | ASSERT_EQ(r, 0); | |
3169 | } | |
3170 | { | |
3171 | ObjectStore::Transaction t; | |
3172 | t.remove(cid, hoid); | |
3173 | t.remove_collection(cid); | |
3174 | cerr << "Cleaning" << std::endl; | |
3175 | r = apply_transaction(store, &osr, std::move(t)); | |
3176 | ASSERT_EQ(r, 0); | |
3177 | } | |
3178 | } | |
3179 | ||
3180 | ghobject_t generate_long_name(unsigned i) | |
3181 | { | |
3182 | stringstream name; | |
3183 | name << "object id " << i << " "; | |
3184 | for (unsigned j = 0; j < 500; ++j) name << 'a'; | |
3185 | ghobject_t hoid(hobject_t(sobject_t(name.str(), CEPH_NOSNAP))); | |
3186 | hoid.hobj.set_hash(i % 2); | |
3187 | return hoid; | |
3188 | } | |
3189 | ||
3190 | TEST_P(StoreTest, LongnameSplitTest) { | |
3191 | ObjectStore::Sequencer osr("test"); | |
3192 | int r; | |
3193 | coll_t cid; | |
3194 | { | |
3195 | ObjectStore::Transaction t; | |
3196 | t.create_collection(cid, 0); | |
3197 | cerr << "Creating collection " << cid << std::endl; | |
3198 | r = apply_transaction(store, &osr, std::move(t)); | |
3199 | ASSERT_EQ(0, r); | |
3200 | } | |
3201 | for (unsigned i = 0; i < 320; ++i) { | |
3202 | ObjectStore::Transaction t; | |
3203 | ghobject_t hoid = generate_long_name(i); | |
3204 | t.touch(cid, hoid); | |
3205 | cerr << "Creating object " << hoid << std::endl; | |
3206 | r = apply_transaction(store, &osr, std::move(t)); | |
3207 | ASSERT_EQ(0, r); | |
3208 | } | |
3209 | ||
3210 | ghobject_t test_obj = generate_long_name(319); | |
3211 | ghobject_t test_obj_2 = test_obj; | |
3212 | test_obj_2.generation = 0; | |
3213 | { | |
3214 | ObjectStore::Transaction t; | |
3215 | // should cause a split | |
3216 | t.collection_move_rename( | |
3217 | cid, test_obj, | |
3218 | cid, test_obj_2); | |
3219 | r = apply_transaction(store, &osr, std::move(t)); | |
3220 | ASSERT_EQ(0, r); | |
3221 | } | |
3222 | ||
3223 | for (unsigned i = 0; i < 319; ++i) { | |
3224 | ObjectStore::Transaction t; | |
3225 | ghobject_t hoid = generate_long_name(i); | |
3226 | t.remove(cid, hoid); | |
3227 | cerr << "Removing object " << hoid << std::endl; | |
3228 | r = apply_transaction(store, &osr, std::move(t)); | |
3229 | ASSERT_EQ(0, r); | |
3230 | } | |
3231 | { | |
3232 | ObjectStore::Transaction t; | |
3233 | t.remove(cid, test_obj_2); | |
3234 | t.remove_collection(cid); | |
3235 | cerr << "Cleaning" << std::endl; | |
3236 | r = apply_transaction(store, &osr, std::move(t)); | |
3237 | ASSERT_EQ(0, r); | |
3238 | } | |
3239 | ||
3240 | } | |
3241 | ||
3242 | TEST_P(StoreTest, ManyObjectTest) { | |
3243 | ObjectStore::Sequencer osr("test"); | |
3244 | int NUM_OBJS = 2000; | |
3245 | int r = 0; | |
3246 | coll_t cid; | |
3247 | string base = ""; | |
3248 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
3249 | set<ghobject_t> created; | |
3250 | { | |
3251 | ObjectStore::Transaction t; | |
3252 | t.create_collection(cid, 0); | |
3253 | r = apply_transaction(store, &osr, std::move(t)); | |
3254 | ASSERT_EQ(r, 0); | |
3255 | } | |
3256 | for (int i = 0; i < NUM_OBJS; ++i) { | |
3257 | if (!(i % 5)) { | |
3258 | cerr << "Object " << i << std::endl; | |
3259 | } | |
3260 | ObjectStore::Transaction t; | |
3261 | char buf[100]; | |
3262 | snprintf(buf, sizeof(buf), "%d", i); | |
3263 | ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP))); | |
3264 | t.touch(cid, hoid); | |
3265 | created.insert(hoid); | |
3266 | r = apply_transaction(store, &osr, std::move(t)); | |
3267 | ASSERT_EQ(r, 0); | |
3268 | } | |
3269 | ||
3270 | for (set<ghobject_t>::iterator i = created.begin(); | |
3271 | i != created.end(); | |
3272 | ++i) { | |
3273 | struct stat buf; | |
3274 | ASSERT_TRUE(!store->stat(cid, *i, &buf)); | |
3275 | } | |
3276 | ||
3277 | set<ghobject_t> listed, listed2; | |
3278 | vector<ghobject_t> objects; | |
3279 | r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0); | |
3280 | ASSERT_EQ(r, 0); | |
3281 | ||
3282 | cerr << "objects.size() is " << objects.size() << std::endl; | |
3283 | for (vector<ghobject_t> ::iterator i = objects.begin(); | |
3284 | i != objects.end(); | |
3285 | ++i) { | |
3286 | listed.insert(*i); | |
3287 | ASSERT_TRUE(created.count(*i)); | |
3288 | } | |
3289 | ASSERT_TRUE(listed.size() == created.size()); | |
3290 | ||
3291 | ghobject_t start, next; | |
3292 | objects.clear(); | |
3293 | r = store->collection_list( | |
3294 | cid, | |
3295 | ghobject_t::get_max(), | |
3296 | ghobject_t::get_max(), | |
3297 | 50, | |
3298 | &objects, | |
3299 | &next | |
3300 | ); | |
3301 | ASSERT_EQ(r, 0); | |
3302 | ASSERT_TRUE(objects.empty()); | |
3303 | ||
3304 | objects.clear(); | |
3305 | listed.clear(); | |
3306 | ghobject_t start2, next2; | |
3307 | while (1) { | |
3308 | r = store->collection_list(cid, start, ghobject_t::get_max(), | |
3309 | 50, | |
3310 | &objects, | |
3311 | &next); | |
3312 | ASSERT_TRUE(sorted(objects)); | |
3313 | ASSERT_EQ(r, 0); | |
3314 | listed.insert(objects.begin(), objects.end()); | |
3315 | if (objects.size() < 50) { | |
3316 | ASSERT_TRUE(next.is_max()); | |
3317 | break; | |
3318 | } | |
3319 | objects.clear(); | |
3320 | ||
3321 | start = next; | |
3322 | } | |
3323 | cerr << "listed.size() is " << listed.size() << std::endl; | |
3324 | ASSERT_TRUE(listed.size() == created.size()); | |
3325 | if (listed2.size()) { | |
3326 | ASSERT_EQ(listed.size(), listed2.size()); | |
3327 | } | |
3328 | for (set<ghobject_t>::iterator i = listed.begin(); | |
3329 | i != listed.end(); | |
3330 | ++i) { | |
3331 | ASSERT_TRUE(created.count(*i)); | |
3332 | } | |
3333 | ||
3334 | for (set<ghobject_t>::iterator i = created.begin(); | |
3335 | i != created.end(); | |
3336 | ++i) { | |
3337 | ObjectStore::Transaction t; | |
3338 | t.remove(cid, *i); | |
3339 | r = apply_transaction(store, &osr, std::move(t)); | |
3340 | ASSERT_EQ(r, 0); | |
3341 | } | |
3342 | cerr << "cleaning up" << std::endl; | |
3343 | { | |
3344 | ObjectStore::Transaction t; | |
3345 | t.remove_collection(cid); | |
3346 | r = apply_transaction(store, &osr, std::move(t)); | |
3347 | ASSERT_EQ(r, 0); | |
3348 | } | |
3349 | } | |
3350 | ||
3351 | ||
3352 | class ObjectGenerator { | |
3353 | public: | |
3354 | virtual ghobject_t create_object(gen_type *gen) = 0; | |
3355 | virtual ~ObjectGenerator() {} | |
3356 | }; | |
3357 | ||
3358 | class MixedGenerator : public ObjectGenerator { | |
3359 | public: | |
3360 | unsigned seq; | |
3361 | int64_t poolid; | |
3362 | explicit MixedGenerator(int64_t p) : seq(0), poolid(p) {} | |
3363 | ghobject_t create_object(gen_type *gen) override { | |
3364 | char buf[100]; | |
3365 | snprintf(buf, sizeof(buf), "OBJ_%u", seq); | |
3366 | string name(buf); | |
3367 | if (seq % 2) { | |
3368 | for (unsigned i = 0; i < 300; ++i) { | |
3369 | name.push_back('a'); | |
3370 | } | |
3371 | } | |
3372 | ++seq; | |
3373 | return ghobject_t( | |
3374 | hobject_t( | |
3375 | name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), | |
3376 | (((seq / 1024) % 2) * 0xF00 ) + | |
3377 | (seq & 0xFF), | |
3378 | poolid, "")); | |
3379 | } | |
3380 | }; | |
3381 | ||
3382 | class SyntheticWorkloadState { | |
3383 | struct Object { | |
3384 | bufferlist data; | |
3385 | map<string, bufferlist> attrs; | |
3386 | }; | |
3387 | public: | |
3388 | static const unsigned max_in_flight = 16; | |
3389 | static const unsigned max_objects = 3000; | |
3390 | static const unsigned max_attr_size = 5; | |
3391 | static const unsigned max_attr_name_len = 100; | |
3392 | static const unsigned max_attr_value_len = 1024 * 64; | |
3393 | coll_t cid; | |
3394 | unsigned write_alignment; | |
3395 | unsigned max_object_len, max_write_len; | |
3396 | unsigned in_flight; | |
3397 | map<ghobject_t, Object> contents; | |
3398 | set<ghobject_t> available_objects; | |
3399 | set<ghobject_t> in_flight_objects; | |
3400 | ObjectGenerator *object_gen; | |
3401 | gen_type *rng; | |
3402 | ObjectStore *store; | |
3403 | ObjectStore::Sequencer *osr; | |
3404 | ||
3405 | Mutex lock; | |
3406 | Cond cond; | |
3407 | ||
3408 | struct EnterExit { | |
3409 | const char *msg; | |
3410 | explicit EnterExit(const char *m) : msg(m) { | |
3411 | //cout << pthread_self() << " enter " << msg << std::endl; | |
3412 | } | |
3413 | ~EnterExit() { | |
3414 | //cout << pthread_self() << " exit " << msg << std::endl; | |
3415 | } | |
3416 | }; | |
3417 | ||
3418 | class C_SyntheticOnReadable : public Context { | |
3419 | public: | |
3420 | SyntheticWorkloadState *state; | |
3421 | ghobject_t hoid; | |
3422 | C_SyntheticOnReadable(SyntheticWorkloadState *state, ghobject_t hoid) | |
3423 | : state(state), hoid(hoid) {} | |
3424 | ||
3425 | void finish(int r) override { | |
3426 | Mutex::Locker locker(state->lock); | |
3427 | EnterExit ee("onreadable finish"); | |
3428 | ASSERT_TRUE(state->in_flight_objects.count(hoid)); | |
3429 | ASSERT_EQ(r, 0); | |
3430 | state->in_flight_objects.erase(hoid); | |
3431 | if (state->contents.count(hoid)) | |
3432 | state->available_objects.insert(hoid); | |
3433 | --(state->in_flight); | |
3434 | state->cond.Signal(); | |
3435 | ||
3436 | bufferlist r2; | |
3437 | r = state->store->read(state->cid, hoid, 0, state->contents[hoid].data.length(), r2); | |
3438 | assert(bl_eq(state->contents[hoid].data, r2)); | |
3439 | state->cond.Signal(); | |
3440 | } | |
3441 | }; | |
3442 | ||
3443 | class C_SyntheticOnStash : public Context { | |
3444 | public: | |
3445 | SyntheticWorkloadState *state; | |
3446 | ghobject_t oid, noid; | |
3447 | ||
3448 | C_SyntheticOnStash(SyntheticWorkloadState *state, | |
3449 | ghobject_t oid, ghobject_t noid) | |
3450 | : state(state), oid(oid), noid(noid) {} | |
3451 | ||
3452 | void finish(int r) override { | |
3453 | Mutex::Locker locker(state->lock); | |
3454 | EnterExit ee("stash finish"); | |
3455 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3456 | ASSERT_EQ(r, 0); | |
3457 | state->in_flight_objects.erase(oid); | |
3458 | if (state->contents.count(noid)) | |
3459 | state->available_objects.insert(noid); | |
3460 | --(state->in_flight); | |
3461 | bufferlist r2; | |
3462 | r = state->store->read( | |
3463 | state->cid, noid, 0, | |
3464 | state->contents[noid].data.length(), r2); | |
3465 | assert(bl_eq(state->contents[noid].data, r2)); | |
3466 | state->cond.Signal(); | |
3467 | } | |
3468 | }; | |
3469 | ||
3470 | class C_SyntheticOnClone : public Context { | |
3471 | public: | |
3472 | SyntheticWorkloadState *state; | |
3473 | ghobject_t oid, noid; | |
3474 | ||
3475 | C_SyntheticOnClone(SyntheticWorkloadState *state, | |
3476 | ghobject_t oid, ghobject_t noid) | |
3477 | : state(state), oid(oid), noid(noid) {} | |
3478 | ||
3479 | void finish(int r) override { | |
3480 | Mutex::Locker locker(state->lock); | |
3481 | EnterExit ee("clone finish"); | |
3482 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3483 | ASSERT_EQ(r, 0); | |
3484 | state->in_flight_objects.erase(oid); | |
3485 | if (state->contents.count(oid)) | |
3486 | state->available_objects.insert(oid); | |
3487 | if (state->contents.count(noid)) | |
3488 | state->available_objects.insert(noid); | |
3489 | --(state->in_flight); | |
3490 | bufferlist r2; | |
3491 | r = state->store->read(state->cid, noid, 0, state->contents[noid].data.length(), r2); | |
3492 | assert(bl_eq(state->contents[noid].data, r2)); | |
3493 | state->cond.Signal(); | |
3494 | } | |
3495 | }; | |
3496 | ||
3497 | static void filled_byte_array(bufferlist& bl, size_t size) | |
3498 | { | |
3499 | static const char alphanum[] = "0123456789" | |
3500 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
3501 | "abcdefghijklmnopqrstuvwxyz"; | |
3502 | if (!size) { | |
3503 | return; | |
3504 | } | |
3505 | bufferptr bp(size); | |
3506 | for (unsigned int i = 0; i < size - 1; i++) { | |
3507 | // severely limit entropy so we can compress... | |
3508 | bp[i] = alphanum[rand() % 10]; //(sizeof(alphanum) - 1)]; | |
3509 | } | |
3510 | bp[size - 1] = '\0'; | |
3511 | ||
3512 | bl.append(bp); | |
3513 | } | |
3514 | ||
3515 | SyntheticWorkloadState(ObjectStore *store, | |
3516 | ObjectGenerator *gen, | |
3517 | gen_type *rng, | |
3518 | ObjectStore::Sequencer *osr, | |
3519 | coll_t cid, | |
3520 | unsigned max_size, | |
3521 | unsigned max_write, | |
3522 | unsigned alignment) | |
3523 | : cid(cid), write_alignment(alignment), max_object_len(max_size), | |
3524 | max_write_len(max_write), in_flight(0), object_gen(gen), | |
3525 | rng(rng), store(store), osr(osr), lock("State lock") {} | |
3526 | ||
3527 | int init() { | |
3528 | ObjectStore::Transaction t; | |
3529 | t.create_collection(cid, 0); | |
3530 | return apply_transaction(store, osr, std::move(t)); | |
3531 | } | |
3532 | void shutdown() { | |
3533 | while (1) { | |
3534 | vector<ghobject_t> objects; | |
3535 | int r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), | |
3536 | 10, &objects, 0); | |
3537 | assert(r >= 0); | |
3538 | if (objects.empty()) | |
3539 | break; | |
3540 | ObjectStore::Transaction t; | |
3541 | for (vector<ghobject_t>::iterator p = objects.begin(); | |
3542 | p != objects.end(); ++p) { | |
3543 | t.remove(cid, *p); | |
3544 | } | |
3545 | apply_transaction(store, osr, std::move(t)); | |
3546 | } | |
3547 | ObjectStore::Transaction t; | |
3548 | t.remove_collection(cid); | |
3549 | apply_transaction(store, osr, std::move(t)); | |
3550 | } | |
3551 | void statfs(store_statfs_t& stat) { | |
3552 | store->statfs(&stat); | |
3553 | } | |
3554 | ||
3555 | ghobject_t get_uniform_random_object() { | |
3556 | while (in_flight >= max_in_flight || available_objects.empty()) | |
3557 | cond.Wait(lock); | |
3558 | boost::uniform_int<> choose(0, available_objects.size() - 1); | |
3559 | int index = choose(*rng); | |
3560 | set<ghobject_t>::iterator i = available_objects.begin(); | |
3561 | for ( ; index > 0; --index, ++i) ; | |
3562 | ghobject_t ret = *i; | |
3563 | return ret; | |
3564 | } | |
3565 | ||
3566 | void wait_for_ready() { | |
3567 | while (in_flight >= max_in_flight) | |
3568 | cond.Wait(lock); | |
3569 | } | |
3570 | ||
3571 | void wait_for_done() { | |
3572 | osr->flush(); | |
3573 | Mutex::Locker locker(lock); | |
3574 | while (in_flight) | |
3575 | cond.Wait(lock); | |
3576 | } | |
3577 | ||
3578 | bool can_create() { | |
3579 | return (available_objects.size() + in_flight_objects.size()) < max_objects; | |
3580 | } | |
3581 | ||
3582 | bool can_unlink() { | |
3583 | return (available_objects.size() + in_flight_objects.size()) > 0; | |
3584 | } | |
3585 | ||
3586 | unsigned get_random_alloc_hints() { | |
3587 | unsigned f = 0; | |
3588 | { | |
3589 | boost::uniform_int<> u(0, 3); | |
3590 | switch (u(*rng)) { | |
3591 | case 1: | |
3592 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE; | |
3593 | break; | |
3594 | case 2: | |
3595 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE; | |
3596 | break; | |
3597 | } | |
3598 | } | |
3599 | { | |
3600 | boost::uniform_int<> u(0, 3); | |
3601 | switch (u(*rng)) { | |
3602 | case 1: | |
3603 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ; | |
3604 | break; | |
3605 | case 2: | |
3606 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ; | |
3607 | break; | |
3608 | } | |
3609 | } | |
3610 | { | |
3611 | // append_only, immutable | |
3612 | boost::uniform_int<> u(0, 4); | |
3613 | f |= u(*rng) << 4; | |
3614 | } | |
3615 | { | |
3616 | boost::uniform_int<> u(0, 3); | |
3617 | switch (u(*rng)) { | |
3618 | case 1: | |
3619 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED; | |
3620 | break; | |
3621 | case 2: | |
3622 | f |= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED; | |
3623 | break; | |
3624 | } | |
3625 | } | |
3626 | { | |
3627 | boost::uniform_int<> u(0, 3); | |
3628 | switch (u(*rng)) { | |
3629 | case 1: | |
3630 | f |= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE; | |
3631 | break; | |
3632 | case 2: | |
3633 | f |= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE; | |
3634 | break; | |
3635 | } | |
3636 | } | |
3637 | return f; | |
3638 | } | |
3639 | ||
3640 | int touch() { | |
3641 | Mutex::Locker locker(lock); | |
3642 | EnterExit ee("touch"); | |
3643 | if (!can_create()) | |
3644 | return -ENOSPC; | |
3645 | wait_for_ready(); | |
3646 | ghobject_t new_obj = object_gen->create_object(rng); | |
3647 | available_objects.erase(new_obj); | |
3648 | ObjectStore::Transaction t; | |
3649 | t.touch(cid, new_obj); | |
3650 | boost::uniform_int<> u(17, 22); | |
3651 | boost::uniform_int<> v(12, 17); | |
3652 | t.set_alloc_hint(cid, new_obj, | |
3653 | 1ull << u(*rng), | |
3654 | 1ull << v(*rng), | |
3655 | get_random_alloc_hints()); | |
3656 | ++in_flight; | |
3657 | in_flight_objects.insert(new_obj); | |
3658 | if (!contents.count(new_obj)) | |
3659 | contents[new_obj] = Object(); | |
3660 | int status = store->queue_transaction(osr, std::move(t), new C_SyntheticOnReadable(this, new_obj)); | |
3661 | return status; | |
3662 | } | |
3663 | ||
3664 | int stash() { | |
3665 | Mutex::Locker locker(lock); | |
3666 | EnterExit ee("stash"); | |
3667 | if (!can_unlink()) | |
3668 | return -ENOENT; | |
3669 | if (!can_create()) | |
3670 | return -ENOSPC; | |
3671 | wait_for_ready(); | |
3672 | ||
3673 | ghobject_t old_obj; | |
3674 | int max = 20; | |
3675 | do { | |
3676 | old_obj = get_uniform_random_object(); | |
3677 | } while (--max && !contents[old_obj].data.length()); | |
3678 | available_objects.erase(old_obj); | |
3679 | ghobject_t new_obj = old_obj; | |
3680 | new_obj.generation++; | |
3681 | available_objects.erase(new_obj); | |
3682 | ||
3683 | ObjectStore::Transaction t; | |
3684 | t.collection_move_rename(cid, old_obj, cid, new_obj); | |
3685 | ++in_flight; | |
3686 | in_flight_objects.insert(old_obj); | |
3687 | ||
3688 | contents[new_obj].attrs = contents[old_obj].attrs; | |
3689 | contents[new_obj].data = contents[old_obj].data; | |
3690 | contents.erase(old_obj); | |
3691 | int status = store->queue_transaction( | |
3692 | osr, std::move(t), | |
3693 | new C_SyntheticOnStash(this, old_obj, new_obj)); | |
3694 | return status; | |
3695 | } | |
3696 | ||
3697 | int clone() { | |
3698 | Mutex::Locker locker(lock); | |
3699 | EnterExit ee("clone"); | |
3700 | if (!can_unlink()) | |
3701 | return -ENOENT; | |
3702 | if (!can_create()) | |
3703 | return -ENOSPC; | |
3704 | wait_for_ready(); | |
3705 | ||
3706 | ghobject_t old_obj; | |
3707 | int max = 20; | |
3708 | do { | |
3709 | old_obj = get_uniform_random_object(); | |
3710 | } while (--max && !contents[old_obj].data.length()); | |
3711 | available_objects.erase(old_obj); | |
3712 | ghobject_t new_obj = object_gen->create_object(rng); | |
3713 | // make the hash match | |
3714 | new_obj.hobj.set_hash(old_obj.hobj.get_hash()); | |
3715 | available_objects.erase(new_obj); | |
3716 | ||
3717 | ObjectStore::Transaction t; | |
3718 | t.clone(cid, old_obj, new_obj); | |
3719 | ++in_flight; | |
3720 | in_flight_objects.insert(old_obj); | |
3721 | ||
3722 | contents[new_obj].attrs = contents[old_obj].attrs; | |
3723 | contents[new_obj].data = contents[old_obj].data; | |
3724 | ||
3725 | int status = store->queue_transaction( | |
3726 | osr, std::move(t), | |
3727 | new C_SyntheticOnClone(this, old_obj, new_obj)); | |
3728 | return status; | |
3729 | } | |
3730 | ||
3731 | int clone_range() { | |
3732 | Mutex::Locker locker(lock); | |
3733 | EnterExit ee("clone_range"); | |
3734 | if (!can_unlink()) | |
3735 | return -ENOENT; | |
3736 | if (!can_create()) | |
3737 | return -ENOSPC; | |
3738 | wait_for_ready(); | |
3739 | ||
3740 | ghobject_t old_obj; | |
3741 | int max = 20; | |
3742 | do { | |
3743 | old_obj = get_uniform_random_object(); | |
3744 | } while (--max && !contents[old_obj].data.length()); | |
3745 | bufferlist &srcdata = contents[old_obj].data; | |
3746 | if (srcdata.length() == 0) { | |
3747 | return 0; | |
3748 | } | |
3749 | available_objects.erase(old_obj); | |
3750 | ghobject_t new_obj = get_uniform_random_object(); | |
3751 | available_objects.erase(new_obj); | |
3752 | ||
3753 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
3754 | boost::uniform_int<> u2(0, max_write_len); | |
3755 | uint64_t srcoff = u1(*rng); | |
3756 | // make src and dst offsets match, since that's what the osd does | |
3757 | uint64_t dstoff = srcoff; //u1(*rng); | |
3758 | uint64_t len = u2(*rng); | |
3759 | if (write_alignment) { | |
3760 | srcoff = ROUND_UP_TO(srcoff, write_alignment); | |
3761 | dstoff = ROUND_UP_TO(dstoff, write_alignment); | |
3762 | len = ROUND_UP_TO(len, write_alignment); | |
3763 | } | |
3764 | ||
3765 | if (srcoff > srcdata.length() - 1) { | |
3766 | srcoff = srcdata.length() - 1; | |
3767 | } | |
3768 | if (srcoff + len > srcdata.length()) { | |
3769 | len = srcdata.length() - srcoff; | |
3770 | } | |
3771 | if (0) | |
3772 | cout << __func__ << " from " << srcoff << "~" << len | |
3773 | << " (size " << srcdata.length() << ") to " | |
3774 | << dstoff << "~" << len << std::endl; | |
3775 | ||
3776 | ObjectStore::Transaction t; | |
3777 | t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff); | |
3778 | ++in_flight; | |
3779 | in_flight_objects.insert(old_obj); | |
3780 | ||
3781 | bufferlist bl; | |
3782 | if (srcoff < srcdata.length()) { | |
3783 | if (srcoff + len > srcdata.length()) { | |
3784 | bl.substr_of(srcdata, srcoff, srcdata.length() - srcoff); | |
3785 | } else { | |
3786 | bl.substr_of(srcdata, srcoff, len); | |
3787 | } | |
3788 | } | |
3789 | ||
3790 | bufferlist& dstdata = contents[new_obj].data; | |
3791 | if (dstdata.length() <= dstoff) { | |
3792 | if (bl.length() > 0) { | |
3793 | dstdata.append_zero(dstoff - dstdata.length()); | |
3794 | dstdata.append(bl); | |
3795 | } | |
3796 | } else { | |
3797 | bufferlist value; | |
3798 | assert(dstdata.length() > dstoff); | |
3799 | dstdata.copy(0, dstoff, value); | |
3800 | value.append(bl); | |
3801 | if (value.length() < dstdata.length()) | |
3802 | dstdata.copy(value.length(), | |
3803 | dstdata.length() - value.length(), value); | |
3804 | value.swap(dstdata); | |
3805 | } | |
3806 | ||
3807 | int status = store->queue_transaction( | |
3808 | osr, std::move(t), new C_SyntheticOnClone(this, old_obj, new_obj)); | |
3809 | return status; | |
3810 | } | |
3811 | ||
3812 | ||
3813 | int write() { | |
3814 | Mutex::Locker locker(lock); | |
3815 | EnterExit ee("write"); | |
3816 | if (!can_unlink()) | |
3817 | return -ENOENT; | |
3818 | wait_for_ready(); | |
3819 | ||
3820 | ghobject_t new_obj = get_uniform_random_object(); | |
3821 | available_objects.erase(new_obj); | |
3822 | ObjectStore::Transaction t; | |
3823 | ||
3824 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
3825 | boost::uniform_int<> u2(0, max_write_len); | |
3826 | uint64_t offset = u1(*rng); | |
3827 | uint64_t len = u2(*rng); | |
3828 | bufferlist bl; | |
3829 | if (write_alignment) { | |
3830 | offset = ROUND_UP_TO(offset, write_alignment); | |
3831 | len = ROUND_UP_TO(len, write_alignment); | |
3832 | } | |
3833 | ||
3834 | filled_byte_array(bl, len); | |
3835 | ||
3836 | bufferlist& data = contents[new_obj].data; | |
3837 | if (data.length() <= offset) { | |
3838 | if (len > 0) { | |
3839 | data.append_zero(offset-data.length()); | |
3840 | data.append(bl); | |
3841 | } | |
3842 | } else { | |
3843 | bufferlist value; | |
3844 | assert(data.length() > offset); | |
3845 | data.copy(0, offset, value); | |
3846 | value.append(bl); | |
3847 | if (value.length() < data.length()) | |
3848 | data.copy(value.length(), | |
3849 | data.length()-value.length(), value); | |
3850 | value.swap(data); | |
3851 | } | |
3852 | ||
3853 | t.write(cid, new_obj, offset, len, bl); | |
3854 | ++in_flight; | |
3855 | in_flight_objects.insert(new_obj); | |
3856 | int status = store->queue_transaction( | |
3857 | osr, std::move(t), new C_SyntheticOnReadable(this, new_obj)); | |
3858 | return status; | |
3859 | } | |
3860 | ||
3861 | int truncate() { | |
3862 | Mutex::Locker locker(lock); | |
3863 | EnterExit ee("truncate"); | |
3864 | if (!can_unlink()) | |
3865 | return -ENOENT; | |
3866 | wait_for_ready(); | |
3867 | ||
3868 | ghobject_t obj = get_uniform_random_object(); | |
3869 | available_objects.erase(obj); | |
3870 | ObjectStore::Transaction t; | |
3871 | ||
3872 | boost::uniform_int<> choose(0, max_object_len); | |
3873 | size_t len = choose(*rng); | |
3874 | if (write_alignment) { | |
3875 | len = ROUND_UP_TO(len, write_alignment); | |
3876 | } | |
3877 | ||
3878 | t.truncate(cid, obj, len); | |
3879 | ++in_flight; | |
3880 | in_flight_objects.insert(obj); | |
3881 | bufferlist& data = contents[obj].data; | |
3882 | if (data.length() <= len) { | |
3883 | data.append_zero(len - data.length()); | |
3884 | } else { | |
3885 | bufferlist bl; | |
3886 | data.copy(0, len, bl); | |
3887 | bl.swap(data); | |
3888 | } | |
3889 | ||
3890 | int status = store->queue_transaction( | |
3891 | osr, std::move(t), new C_SyntheticOnReadable(this, obj)); | |
3892 | return status; | |
3893 | } | |
3894 | ||
3895 | int zero() { | |
3896 | Mutex::Locker locker(lock); | |
3897 | EnterExit ee("zero"); | |
3898 | if (!can_unlink()) | |
3899 | return -ENOENT; | |
3900 | wait_for_ready(); | |
3901 | ||
3902 | ghobject_t new_obj = get_uniform_random_object(); | |
3903 | available_objects.erase(new_obj); | |
3904 | ObjectStore::Transaction t; | |
3905 | ||
3906 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
3907 | boost::uniform_int<> u2(0, max_write_len); | |
3908 | uint64_t offset = u1(*rng); | |
3909 | uint64_t len = u2(*rng); | |
3910 | if (write_alignment) { | |
3911 | offset = ROUND_UP_TO(offset, write_alignment); | |
3912 | len = ROUND_UP_TO(len, write_alignment); | |
3913 | } | |
3914 | ||
3915 | if (len > 0) { | |
3916 | auto& data = contents[new_obj].data; | |
3917 | if (data.length() < offset + len) { | |
3918 | data.append_zero(offset+len-data.length()); | |
3919 | } | |
3920 | bufferlist n; | |
3921 | n.substr_of(data, 0, offset); | |
3922 | n.append_zero(len); | |
3923 | if (data.length() > offset + len) | |
3924 | data.copy(offset + len, data.length() - offset - len, n); | |
3925 | data.swap(n); | |
3926 | } | |
3927 | ||
3928 | t.zero(cid, new_obj, offset, len); | |
3929 | ++in_flight; | |
3930 | in_flight_objects.insert(new_obj); | |
3931 | int status = store->queue_transaction( | |
3932 | osr, std::move(t), new C_SyntheticOnReadable(this, new_obj)); | |
3933 | return status; | |
3934 | } | |
3935 | ||
3936 | void read() { | |
3937 | EnterExit ee("read"); | |
3938 | boost::uniform_int<> u1(0, max_object_len/2); | |
3939 | boost::uniform_int<> u2(0, max_object_len); | |
3940 | uint64_t offset = u1(*rng); | |
3941 | uint64_t len = u2(*rng); | |
3942 | if (offset > len) | |
3943 | swap(offset, len); | |
3944 | ||
3945 | ghobject_t obj; | |
3946 | bufferlist expected; | |
3947 | int r; | |
3948 | { | |
3949 | Mutex::Locker locker(lock); | |
3950 | EnterExit ee("read locked"); | |
3951 | if (!can_unlink()) | |
3952 | return ; | |
3953 | wait_for_ready(); | |
3954 | ||
3955 | obj = get_uniform_random_object(); | |
3956 | expected = contents[obj].data; | |
3957 | } | |
3958 | bufferlist bl, result; | |
3959 | if (0) cout << " obj " << obj | |
3960 | << " size " << expected.length() | |
3961 | << " offset " << offset | |
3962 | << " len " << len << std::endl; | |
3963 | r = store->read(cid, obj, offset, len, result); | |
3964 | if (offset >= expected.length()) { | |
3965 | ASSERT_EQ(r, 0); | |
3966 | } else { | |
3967 | size_t max_len = expected.length() - offset; | |
3968 | if (len > max_len) | |
3969 | len = max_len; | |
3970 | assert(len == result.length()); | |
3971 | ASSERT_EQ(len, result.length()); | |
3972 | expected.copy(offset, len, bl); | |
3973 | ASSERT_EQ(r, (int)len); | |
3974 | ASSERT_TRUE(bl_eq(bl, result)); | |
3975 | } | |
3976 | } | |
3977 | ||
3978 | int setattrs() { | |
3979 | Mutex::Locker locker(lock); | |
3980 | EnterExit ee("setattrs"); | |
3981 | if (!can_unlink()) | |
3982 | return -ENOENT; | |
3983 | wait_for_ready(); | |
3984 | ||
3985 | ghobject_t obj = get_uniform_random_object(); | |
3986 | available_objects.erase(obj); | |
3987 | ObjectStore::Transaction t; | |
3988 | ||
3989 | boost::uniform_int<> u0(1, max_attr_size); | |
3990 | boost::uniform_int<> u1(4, max_attr_name_len); | |
3991 | boost::uniform_int<> u2(4, max_attr_value_len); | |
3992 | boost::uniform_int<> u3(0, 100); | |
3993 | uint64_t size = u0(*rng); | |
3994 | uint64_t name_len; | |
3995 | map<string, bufferlist> attrs; | |
3996 | set<string> keys; | |
3997 | for (map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
3998 | it != contents[obj].attrs.end(); ++it) | |
3999 | keys.insert(it->first); | |
4000 | ||
4001 | while (size--) { | |
4002 | bufferlist name, value; | |
4003 | uint64_t get_exist = u3(*rng); | |
4004 | uint64_t value_len = u2(*rng); | |
4005 | filled_byte_array(value, value_len); | |
4006 | if (get_exist < 50 && keys.size()) { | |
4007 | set<string>::iterator k = keys.begin(); | |
4008 | attrs[*k] = value; | |
4009 | contents[obj].attrs[*k] = value; | |
4010 | keys.erase(k); | |
4011 | } else { | |
4012 | name_len = u1(*rng); | |
4013 | filled_byte_array(name, name_len); | |
4014 | attrs[name.c_str()] = value; | |
4015 | contents[obj].attrs[name.c_str()] = value; | |
4016 | } | |
4017 | } | |
4018 | t.setattrs(cid, obj, attrs); | |
4019 | ++in_flight; | |
4020 | in_flight_objects.insert(obj); | |
4021 | int status = store->queue_transaction( | |
4022 | osr, std::move(t), new C_SyntheticOnReadable(this, obj)); | |
4023 | return status; | |
4024 | } | |
4025 | ||
4026 | void getattrs() { | |
4027 | EnterExit ee("getattrs"); | |
4028 | ghobject_t obj; | |
4029 | map<string, bufferlist> expected; | |
4030 | { | |
4031 | Mutex::Locker locker(lock); | |
4032 | EnterExit ee("getattrs locked"); | |
4033 | if (!can_unlink()) | |
4034 | return ; | |
4035 | wait_for_ready(); | |
4036 | ||
4037 | int retry = 10; | |
4038 | do { | |
4039 | obj = get_uniform_random_object(); | |
4040 | if (!--retry) | |
4041 | return ; | |
4042 | } while (contents[obj].attrs.empty()); | |
4043 | expected = contents[obj].attrs; | |
4044 | } | |
4045 | map<string, bufferlist> attrs; | |
4046 | int r = store->getattrs(cid, obj, attrs); | |
4047 | ASSERT_TRUE(r == 0); | |
4048 | ASSERT_TRUE(attrs.size() == expected.size()); | |
4049 | for (map<string, bufferlist>::iterator it = expected.begin(); | |
4050 | it != expected.end(); ++it) { | |
4051 | ASSERT_TRUE(bl_eq(attrs[it->first], it->second)); | |
4052 | } | |
4053 | } | |
4054 | ||
4055 | void getattr() { | |
4056 | EnterExit ee("getattr"); | |
4057 | ghobject_t obj; | |
4058 | int r; | |
4059 | int retry; | |
4060 | map<string, bufferlist> expected; | |
4061 | { | |
4062 | Mutex::Locker locker(lock); | |
4063 | EnterExit ee("getattr locked"); | |
4064 | if (!can_unlink()) | |
4065 | return ; | |
4066 | wait_for_ready(); | |
4067 | ||
4068 | retry = 10; | |
4069 | do { | |
4070 | obj = get_uniform_random_object(); | |
4071 | if (!--retry) | |
4072 | return ; | |
4073 | } while (contents[obj].attrs.empty()); | |
4074 | expected = contents[obj].attrs; | |
4075 | } | |
4076 | boost::uniform_int<> u(0, expected.size()-1); | |
4077 | retry = u(*rng); | |
4078 | map<string, bufferlist>::iterator it = expected.begin(); | |
4079 | while (retry) { | |
4080 | retry--; | |
4081 | ++it; | |
4082 | } | |
4083 | ||
4084 | bufferlist bl; | |
4085 | r = store->getattr(cid, obj, it->first, bl); | |
4086 | ASSERT_EQ(r, 0); | |
4087 | ASSERT_TRUE(bl_eq(it->second, bl)); | |
4088 | } | |
4089 | ||
4090 | int rmattr() { | |
4091 | Mutex::Locker locker(lock); | |
4092 | EnterExit ee("rmattr"); | |
4093 | if (!can_unlink()) | |
4094 | return -ENOENT; | |
4095 | wait_for_ready(); | |
4096 | ||
4097 | ghobject_t obj; | |
4098 | int retry = 10; | |
4099 | do { | |
4100 | obj = get_uniform_random_object(); | |
4101 | if (!--retry) | |
4102 | return 0; | |
4103 | } while (contents[obj].attrs.empty()); | |
4104 | ||
4105 | boost::uniform_int<> u(0, contents[obj].attrs.size()-1); | |
4106 | retry = u(*rng); | |
4107 | map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4108 | while (retry) { | |
4109 | retry--; | |
4110 | ++it; | |
4111 | } | |
4112 | ||
4113 | available_objects.erase(obj); | |
4114 | ObjectStore::Transaction t; | |
4115 | t.rmattr(cid, obj, it->first); | |
4116 | ||
4117 | contents[obj].attrs.erase(it->first); | |
4118 | ++in_flight; | |
4119 | in_flight_objects.insert(obj); | |
4120 | int status = store->queue_transaction( | |
4121 | osr, std::move(t), new C_SyntheticOnReadable(this, obj)); | |
4122 | return status; | |
4123 | } | |
4124 | ||
4125 | void fsck(bool deep) { | |
4126 | Mutex::Locker locker(lock); | |
4127 | EnterExit ee("fsck"); | |
4128 | while (in_flight) | |
4129 | cond.Wait(lock); | |
4130 | store->umount(); | |
4131 | int r = store->fsck(deep); | |
4132 | assert(r == 0 || r == -EOPNOTSUPP); | |
4133 | store->mount(); | |
4134 | } | |
4135 | ||
4136 | void scan() { | |
4137 | Mutex::Locker locker(lock); | |
4138 | EnterExit ee("scan"); | |
4139 | while (in_flight) | |
4140 | cond.Wait(lock); | |
4141 | vector<ghobject_t> objects; | |
4142 | set<ghobject_t> objects_set, objects_set2; | |
4143 | ghobject_t next, current; | |
4144 | while (1) { | |
4145 | //cerr << "scanning..." << std::endl; | |
4146 | int r = store->collection_list(cid, current, ghobject_t::get_max(), 100, | |
4147 | &objects, &next); | |
4148 | ASSERT_EQ(r, 0); | |
4149 | ASSERT_TRUE(sorted(objects)); | |
4150 | objects_set.insert(objects.begin(), objects.end()); | |
4151 | objects.clear(); | |
4152 | if (next.is_max()) break; | |
4153 | current = next; | |
4154 | } | |
4155 | if (objects_set.size() != available_objects.size()) { | |
4156 | for (set<ghobject_t>::iterator p = objects_set.begin(); | |
4157 | p != objects_set.end(); | |
4158 | ++p) | |
4159 | if (available_objects.count(*p) == 0) { | |
4160 | cerr << "+ " << *p << std::endl; | |
4161 | ceph_abort(); | |
4162 | } | |
4163 | for (set<ghobject_t>::iterator p = available_objects.begin(); | |
4164 | p != available_objects.end(); | |
4165 | ++p) | |
4166 | if (objects_set.count(*p) == 0) | |
4167 | cerr << "- " << *p << std::endl; | |
4168 | //cerr << " objects_set: " << objects_set << std::endl; | |
4169 | //cerr << " available_set: " << available_objects << std::endl; | |
4170 | assert(0 == "badness"); | |
4171 | } | |
4172 | ||
4173 | ASSERT_EQ(objects_set.size(), available_objects.size()); | |
4174 | for (set<ghobject_t>::iterator i = objects_set.begin(); | |
4175 | i != objects_set.end(); | |
4176 | ++i) { | |
4177 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4178 | } | |
4179 | ||
4180 | int r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), | |
4181 | INT_MAX, &objects, 0); | |
4182 | ASSERT_EQ(r, 0); | |
4183 | objects_set2.insert(objects.begin(), objects.end()); | |
4184 | ASSERT_EQ(objects_set2.size(), available_objects.size()); | |
4185 | for (set<ghobject_t>::iterator i = objects_set2.begin(); | |
4186 | i != objects_set2.end(); | |
4187 | ++i) { | |
4188 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4189 | if (available_objects.count(*i) == 0) { | |
4190 | cerr << "+ " << *i << std::endl; | |
4191 | } | |
4192 | } | |
4193 | } | |
4194 | ||
4195 | void stat() { | |
4196 | EnterExit ee("stat"); | |
4197 | ghobject_t hoid; | |
4198 | uint64_t expected; | |
4199 | { | |
4200 | Mutex::Locker locker(lock); | |
4201 | EnterExit ee("stat lock1"); | |
4202 | if (!can_unlink()) | |
4203 | return ; | |
4204 | hoid = get_uniform_random_object(); | |
4205 | in_flight_objects.insert(hoid); | |
4206 | available_objects.erase(hoid); | |
4207 | ++in_flight; | |
4208 | expected = contents[hoid].data.length(); | |
4209 | } | |
4210 | struct stat buf; | |
4211 | int r = store->stat(cid, hoid, &buf); | |
4212 | ASSERT_EQ(0, r); | |
4213 | assert((uint64_t)buf.st_size == expected); | |
4214 | ASSERT_TRUE((uint64_t)buf.st_size == expected); | |
4215 | { | |
4216 | Mutex::Locker locker(lock); | |
4217 | EnterExit ee("stat lock2"); | |
4218 | --in_flight; | |
4219 | cond.Signal(); | |
4220 | in_flight_objects.erase(hoid); | |
4221 | available_objects.insert(hoid); | |
4222 | } | |
4223 | } | |
4224 | ||
4225 | int unlink() { | |
4226 | Mutex::Locker locker(lock); | |
4227 | EnterExit ee("unlink"); | |
4228 | if (!can_unlink()) | |
4229 | return -ENOENT; | |
4230 | ghobject_t to_remove = get_uniform_random_object(); | |
4231 | ObjectStore::Transaction t; | |
4232 | t.remove(cid, to_remove); | |
4233 | ++in_flight; | |
4234 | available_objects.erase(to_remove); | |
4235 | in_flight_objects.insert(to_remove); | |
4236 | contents.erase(to_remove); | |
4237 | int status = store->queue_transaction(osr, std::move(t), new C_SyntheticOnReadable(this, to_remove)); | |
4238 | return status; | |
4239 | } | |
4240 | ||
4241 | void print_internal_state() { | |
4242 | Mutex::Locker locker(lock); | |
4243 | cerr << "available_objects: " << available_objects.size() | |
4244 | << " in_flight_objects: " << in_flight_objects.size() | |
4245 | << " total objects: " << in_flight_objects.size() + available_objects.size() | |
4246 | << " in_flight " << in_flight << std::endl; | |
4247 | } | |
4248 | }; | |
4249 | ||
4250 | ||
4251 | void doSyntheticTest(boost::scoped_ptr<ObjectStore>& store, | |
4252 | int num_ops, | |
4253 | uint64_t max_obj, uint64_t max_wr, uint64_t align) | |
4254 | { | |
4255 | ObjectStore::Sequencer osr("test"); | |
4256 | MixedGenerator gen(555); | |
4257 | gen_type rng(time(NULL)); | |
4258 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
4259 | ||
4260 | g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "false"); | |
4261 | g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "false"); | |
4262 | g_ceph_context->_conf->apply_changes(NULL); | |
4263 | ||
4264 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid, | |
4265 | max_obj, max_wr, align); | |
4266 | test_obj.init(); | |
4267 | for (int i = 0; i < num_ops/10; ++i) { | |
4268 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
4269 | test_obj.touch(); | |
4270 | } | |
4271 | for (int i = 0; i < num_ops; ++i) { | |
4272 | if (!(i % 1000)) { | |
4273 | cerr << "Op " << i << std::endl; | |
4274 | test_obj.print_internal_state(); | |
4275 | } | |
4276 | boost::uniform_int<> true_false(0, 999); | |
4277 | int val = true_false(rng); | |
4278 | if (val > 998) { | |
4279 | test_obj.fsck(true); | |
4280 | } else if (val > 997) { | |
4281 | test_obj.fsck(false); | |
4282 | } else if (val > 970) { | |
4283 | test_obj.scan(); | |
4284 | } else if (val > 950) { | |
4285 | test_obj.stat(); | |
4286 | } else if (val > 850) { | |
4287 | test_obj.zero(); | |
4288 | } else if (val > 800) { | |
4289 | test_obj.unlink(); | |
4290 | } else if (val > 550) { | |
4291 | test_obj.write(); | |
4292 | } else if (val > 500) { | |
4293 | test_obj.clone(); | |
4294 | } else if (val > 450) { | |
4295 | test_obj.clone_range(); | |
4296 | } else if (val > 300) { | |
4297 | test_obj.stash(); | |
4298 | } else if (val > 100) { | |
4299 | test_obj.read(); | |
4300 | } else { | |
4301 | test_obj.truncate(); | |
4302 | } | |
4303 | } | |
4304 | test_obj.wait_for_done(); | |
4305 | test_obj.shutdown(); | |
4306 | ||
4307 | g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "true"); | |
4308 | g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "true"); | |
4309 | g_ceph_context->_conf->apply_changes(NULL); | |
4310 | } | |
4311 | ||
4312 | TEST_P(StoreTest, Synthetic) { | |
4313 | doSyntheticTest(store, 10000, 400*1024, 40*1024, 0); | |
4314 | } | |
4315 | ||
4316 | ||
4317 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixSharding) { | |
4318 | if (string(GetParam()) != "bluestore") | |
4319 | return; | |
4320 | ||
4321 | const char *m[][10] = { | |
4322 | { "bluestore_min_alloc_size", "4096", 0 }, // must be the first! | |
4323 | { "num_ops", "50000", 0 }, | |
4324 | { "max_write", "65536", 0 }, | |
4325 | { "max_size", "262144", 0 }, | |
4326 | { "alignment", "4096", 0 }, | |
4327 | { "bluestore_max_blob_size", "65536", 0 }, | |
4328 | { "bluestore_extent_map_shard_min_size", "60", 0 }, | |
4329 | { "bluestore_extent_map_shard_max_size", "300", 0 }, | |
4330 | { "bluestore_extent_map_shard_target_size", "150", 0 }, | |
4331 | { "bluestore_default_buffered_read", "true", 0 }, | |
4332 | { "bluestore_default_buffered_write", "true", 0 }, | |
4333 | { 0 }, | |
4334 | }; | |
4335 | do_matrix(m, store, doSyntheticTest); | |
4336 | } | |
4337 | ||
4338 | TEST_P(StoreTestSpecificAUSize, ZipperPatternSharded) { | |
4339 | if(string(GetParam()) != "bluestore") | |
4340 | return; | |
4341 | StartDeferred(4096); | |
4342 | ||
4343 | int r; | |
4344 | ObjectStore::Sequencer osr("test"); | |
4345 | coll_t cid; | |
4346 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
4347 | { | |
4348 | ObjectStore::Transaction t; | |
4349 | t.create_collection(cid, 0); | |
4350 | cerr << "Creating collection " << cid << std::endl; | |
4351 | r = apply_transaction(store, &osr, std::move(t)); | |
4352 | ASSERT_EQ(r, 0); | |
4353 | } | |
4354 | bufferlist bl; | |
4355 | int len = 4096; | |
4356 | bufferptr bp(len); | |
4357 | bp.zero(); | |
4358 | bl.append(bp); | |
4359 | for (int i=0; i<1000; ++i) { | |
4360 | ObjectStore::Transaction t; | |
4361 | t.write(cid, a, i*2*len, len, bl, 0); | |
4362 | r = apply_transaction(store, &osr, std::move(t)); | |
4363 | ASSERT_EQ(r, 0); | |
4364 | } | |
4365 | for (int i=0; i<1000; ++i) { | |
4366 | ObjectStore::Transaction t; | |
4367 | t.write(cid, a, i*2*len + 1, len, bl, 0); | |
4368 | r = apply_transaction(store, &osr, std::move(t)); | |
4369 | ASSERT_EQ(r, 0); | |
4370 | } | |
4371 | { | |
4372 | ObjectStore::Transaction t; | |
4373 | t.remove(cid, a); | |
4374 | t.remove_collection(cid); | |
4375 | cerr << "Cleaning" << std::endl; | |
4376 | r = apply_transaction(store, &osr, std::move(t)); | |
4377 | ASSERT_EQ(r, 0); | |
4378 | } | |
4379 | } | |
4380 | ||
4381 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumAlgorithm) { | |
4382 | if (string(GetParam()) != "bluestore") | |
4383 | return; | |
4384 | ||
4385 | const char *m[][10] = { | |
4386 | { "bluestore_min_alloc_size", "65536", 0 }, // must be the first! | |
4387 | { "max_write", "65536", 0 }, | |
4388 | { "max_size", "1048576", 0 }, | |
4389 | { "alignment", "16", 0 }, | |
4390 | { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", | |
4391 | "xxhash64", "none", 0 }, | |
4392 | { "bluestore_default_buffered_write", "false", 0 }, | |
4393 | { 0 }, | |
4394 | }; | |
4395 | do_matrix(m, store, doSyntheticTest); | |
4396 | } | |
4397 | ||
4398 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumVsCompression) { | |
4399 | if (string(GetParam()) != "bluestore") | |
4400 | return; | |
4401 | ||
4402 | const char *m[][10] = { | |
4403 | { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first! | |
4404 | { "max_write", "131072", 0 }, | |
4405 | { "max_size", "262144", 0 }, | |
4406 | { "alignment", "512", 0 }, | |
4407 | { "bluestore_compression_mode", "force", 0}, | |
4408 | { "bluestore_compression_algorithm", "snappy", "zlib", 0 }, | |
4409 | { "bluestore_csum_type", "crc32c", 0 }, | |
4410 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4411 | { "bluestore_default_buffered_write", "true", "false", 0 }, | |
4412 | { "bluestore_sync_submit_transaction", "false", 0 }, | |
4413 | { 0 }, | |
4414 | }; | |
4415 | do_matrix(m, store, doSyntheticTest); | |
4416 | } | |
4417 | ||
4418 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompression) { | |
4419 | if (string(GetParam()) != "bluestore") | |
4420 | return; | |
4421 | ||
4422 | const char *m[][10] = { | |
4423 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4424 | { "max_write", "1048576", 0 }, | |
4425 | { "max_size", "4194304", 0 }, | |
4426 | { "alignment", "65536", 0 }, | |
4427 | { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0}, | |
4428 | { "bluestore_default_buffered_write", "false", 0 }, | |
4429 | { "bluestore_sync_submit_transaction", "true", 0 }, | |
4430 | { 0 }, | |
4431 | }; | |
4432 | do_matrix(m, store, doSyntheticTest); | |
4433 | } | |
4434 | ||
4435 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompressionAlgorithm) { | |
4436 | if (string(GetParam()) != "bluestore") | |
4437 | return; | |
4438 | ||
4439 | const char *m[][10] = { | |
4440 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4441 | { "max_write", "1048576", 0 }, | |
4442 | { "max_size", "4194304", 0 }, | |
4443 | { "alignment", "65536", 0 }, | |
4444 | { "bluestore_compression_algorithm", "zlib", "snappy", 0 }, | |
4445 | { "bluestore_compression_mode", "force", 0 }, | |
4446 | { "bluestore_default_buffered_write", "false", 0 }, | |
4447 | { 0 }, | |
4448 | }; | |
4449 | do_matrix(m, store, doSyntheticTest); | |
4450 | } | |
4451 | ||
4452 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixNoCsum) { | |
4453 | if (string(GetParam()) != "bluestore") | |
4454 | return; | |
4455 | ||
4456 | const char *m[][10] = { | |
4457 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4458 | { "max_write", "65536", 0 }, | |
4459 | { "max_size", "1048576", 0 }, | |
4460 | { "alignment", "512", 0 }, | |
4461 | { "bluestore_max_blob_size", "262144", 0 }, | |
4462 | { "bluestore_compression_mode", "force", "none", 0}, | |
4463 | { "bluestore_csum_type", "none", 0}, | |
4464 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4465 | { "bluestore_default_buffered_write", "true", 0 }, | |
4466 | { "bluestore_sync_submit_transaction", "true", "false", 0 }, | |
4467 | { 0 }, | |
4468 | }; | |
4469 | do_matrix(m, store, doSyntheticTest); | |
4470 | } | |
4471 | ||
4472 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixPreferDeferred) { | |
4473 | if (string(GetParam()) != "bluestore") | |
4474 | return; | |
4475 | ||
4476 | const char *m[][10] = { | |
4477 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4478 | { "max_write", "65536", 0 }, | |
4479 | { "max_size", "1048576", 0 }, | |
4480 | { "alignment", "512", 0 }, | |
4481 | { "bluestore_max_blob_size", "262144", 0 }, | |
4482 | { "bluestore_compression_mode", "force", "none", 0}, | |
4483 | { "bluestore_prefer_deferred_size", "32768", "0", 0}, | |
4484 | { 0 }, | |
4485 | }; | |
4486 | do_matrix(m, store, doSyntheticTest); | |
4487 | } | |
4488 | ||
4489 | TEST_P(StoreTest, AttrSynthetic) { | |
4490 | ObjectStore::Sequencer osr("test"); | |
4491 | MixedGenerator gen(447); | |
4492 | gen_type rng(time(NULL)); | |
4493 | coll_t cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD)); | |
4494 | ||
4495 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, &osr, cid, 40*1024, 4*1024, 0); | |
4496 | test_obj.init(); | |
4497 | for (int i = 0; i < 500; ++i) { | |
4498 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; | |
4499 | test_obj.touch(); | |
4500 | } | |
4501 | for (int i = 0; i < 1000; ++i) { | |
4502 | if (!(i % 100)) { | |
4503 | cerr << "Op " << i << std::endl; | |
4504 | test_obj.print_internal_state(); | |
4505 | } | |
4506 | boost::uniform_int<> true_false(0, 99); | |
4507 | int val = true_false(rng); | |
4508 | if (val > 97) { | |
4509 | test_obj.scan(); | |
4510 | } else if (val > 93) { | |
4511 | test_obj.stat(); | |
4512 | } else if (val > 75) { | |
4513 | test_obj.rmattr(); | |
4514 | } else if (val > 47) { | |
4515 | test_obj.setattrs(); | |
4516 | } else if (val > 45) { | |
4517 | test_obj.clone(); | |
4518 | } else if (val > 37) { | |
4519 | test_obj.stash(); | |
4520 | } else if (val > 30) { | |
4521 | test_obj.getattrs(); | |
4522 | } else { | |
4523 | test_obj.getattr(); | |
4524 | } | |
4525 | } | |
4526 | test_obj.wait_for_done(); | |
4527 | test_obj.shutdown(); | |
4528 | } | |
4529 | ||
4530 | TEST_P(StoreTest, HashCollisionTest) { | |
4531 | ObjectStore::Sequencer osr("test"); | |
4532 | int64_t poolid = 11; | |
4533 | coll_t cid(spg_t(pg_t(0,poolid),shard_id_t::NO_SHARD)); | |
4534 | int r; | |
4535 | { | |
4536 | ObjectStore::Transaction t; | |
4537 | t.create_collection(cid, 0); | |
4538 | r = apply_transaction(store, &osr, std::move(t)); | |
4539 | ASSERT_EQ(r, 0); | |
4540 | } | |
4541 | string base = ""; | |
4542 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
4543 | set<ghobject_t> created; | |
4544 | for (int n = 0; n < 10; ++n) { | |
4545 | char nbuf[100]; | |
4546 | sprintf(nbuf, "n%d", n); | |
4547 | for (int i = 0; i < 1000; ++i) { | |
4548 | char buf[100]; | |
4549 | sprintf(buf, "%d", i); | |
4550 | if (!(i % 100)) { | |
4551 | cerr << "Object n" << n << " "<< i << std::endl; | |
4552 | } | |
4553 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, poolid, string(nbuf))); | |
4554 | { | |
4555 | ObjectStore::Transaction t; | |
4556 | t.touch(cid, hoid); | |
4557 | r = apply_transaction(store, &osr, std::move(t)); | |
4558 | ASSERT_EQ(r, 0); | |
4559 | } | |
4560 | created.insert(hoid); | |
4561 | } | |
4562 | } | |
4563 | vector<ghobject_t> objects; | |
4564 | r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0); | |
4565 | ASSERT_EQ(r, 0); | |
4566 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
4567 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
4568 | ASSERT_TRUE(listed.size() == created.size()); | |
4569 | objects.clear(); | |
4570 | listed.clear(); | |
4571 | ghobject_t current, next; | |
4572 | while (1) { | |
4573 | r = store->collection_list(cid, current, ghobject_t::get_max(), 60, | |
4574 | &objects, &next); | |
4575 | ASSERT_EQ(r, 0); | |
4576 | ASSERT_TRUE(sorted(objects)); | |
4577 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
4578 | i != objects.end(); | |
4579 | ++i) { | |
4580 | if (listed.count(*i)) | |
4581 | cerr << *i << " repeated" << std::endl; | |
4582 | listed.insert(*i); | |
4583 | } | |
4584 | if (objects.size() < 50) { | |
4585 | ASSERT_TRUE(next.is_max()); | |
4586 | break; | |
4587 | } | |
4588 | objects.clear(); | |
4589 | current = next; | |
4590 | } | |
4591 | cerr << "listed.size() is " << listed.size() << std::endl; | |
4592 | ASSERT_TRUE(listed.size() == created.size()); | |
4593 | for (set<ghobject_t>::iterator i = listed.begin(); | |
4594 | i != listed.end(); | |
4595 | ++i) { | |
4596 | ASSERT_TRUE(created.count(*i)); | |
4597 | } | |
4598 | ||
4599 | for (set<ghobject_t>::iterator i = created.begin(); | |
4600 | i != created.end(); | |
4601 | ++i) { | |
4602 | ObjectStore::Transaction t; | |
4603 | t.remove(cid, *i); | |
4604 | r = apply_transaction(store, &osr, std::move(t)); | |
4605 | ASSERT_EQ(r, 0); | |
4606 | } | |
4607 | ObjectStore::Transaction t; | |
4608 | t.remove_collection(cid); | |
4609 | r = apply_transaction(store, &osr, std::move(t)); | |
4610 | ASSERT_EQ(r, 0); | |
4611 | } | |
4612 | ||
4613 | TEST_P(StoreTest, ScrubTest) { | |
4614 | ObjectStore::Sequencer osr("test"); | |
4615 | int64_t poolid = 111; | |
4616 | coll_t cid(spg_t(pg_t(0, poolid),shard_id_t(1))); | |
4617 | int r; | |
4618 | { | |
4619 | ObjectStore::Transaction t; | |
4620 | t.create_collection(cid, 0); | |
4621 | r = apply_transaction(store, &osr, std::move(t)); | |
4622 | ASSERT_EQ(r, 0); | |
4623 | } | |
4624 | string base = "aaaaa"; | |
4625 | set<ghobject_t> created; | |
4626 | for (int i = 0; i < 1000; ++i) { | |
4627 | char buf[100]; | |
4628 | sprintf(buf, "%d", i); | |
4629 | if (!(i % 5)) { | |
4630 | cerr << "Object " << i << std::endl; | |
4631 | } | |
4632 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, i, | |
4633 | poolid, ""), | |
4634 | ghobject_t::NO_GEN, shard_id_t(1)); | |
4635 | { | |
4636 | ObjectStore::Transaction t; | |
4637 | t.touch(cid, hoid); | |
4638 | r = apply_transaction(store, &osr, std::move(t)); | |
4639 | ASSERT_EQ(r, 0); | |
4640 | } | |
4641 | created.insert(hoid); | |
4642 | } | |
4643 | ||
4644 | // Add same hobject_t but different generation | |
4645 | { | |
4646 | ghobject_t hoid1(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), | |
4647 | ghobject_t::NO_GEN, shard_id_t(1)); | |
4648 | ghobject_t hoid2(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)1, shard_id_t(1)); | |
4649 | ghobject_t hoid3(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)2, shard_id_t(1)); | |
4650 | ObjectStore::Transaction t; | |
4651 | t.touch(cid, hoid1); | |
4652 | t.touch(cid, hoid2); | |
4653 | t.touch(cid, hoid3); | |
4654 | r = apply_transaction(store, &osr, std::move(t)); | |
4655 | created.insert(hoid1); | |
4656 | created.insert(hoid2); | |
4657 | created.insert(hoid3); | |
4658 | ASSERT_EQ(r, 0); | |
4659 | } | |
4660 | ||
4661 | vector<ghobject_t> objects; | |
4662 | r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), | |
4663 | INT_MAX, &objects, 0); | |
4664 | ASSERT_EQ(r, 0); | |
4665 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
4666 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
4667 | ASSERT_TRUE(listed.size() == created.size()); | |
4668 | objects.clear(); | |
4669 | listed.clear(); | |
4670 | ghobject_t current, next; | |
4671 | while (1) { | |
4672 | r = store->collection_list(cid, current, ghobject_t::get_max(), 60, | |
4673 | &objects, &next); | |
4674 | ASSERT_EQ(r, 0); | |
4675 | ASSERT_TRUE(sorted(objects)); | |
4676 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
4677 | i != objects.end(); ++i) { | |
4678 | if (listed.count(*i)) | |
4679 | cerr << *i << " repeated" << std::endl; | |
4680 | listed.insert(*i); | |
4681 | } | |
4682 | if (objects.size() < 50) { | |
4683 | ASSERT_TRUE(next.is_max()); | |
4684 | break; | |
4685 | } | |
4686 | objects.clear(); | |
4687 | current = next.get_boundary(); | |
4688 | } | |
4689 | cerr << "listed.size() is " << listed.size() << std::endl; | |
4690 | ASSERT_TRUE(listed.size() == created.size()); | |
4691 | for (set<ghobject_t>::iterator i = listed.begin(); | |
4692 | i != listed.end(); | |
4693 | ++i) { | |
4694 | ASSERT_TRUE(created.count(*i)); | |
4695 | } | |
4696 | ||
4697 | for (set<ghobject_t>::iterator i = created.begin(); | |
4698 | i != created.end(); | |
4699 | ++i) { | |
4700 | ObjectStore::Transaction t; | |
4701 | t.remove(cid, *i); | |
4702 | r = apply_transaction(store, &osr, std::move(t)); | |
4703 | ASSERT_EQ(r, 0); | |
4704 | } | |
4705 | ObjectStore::Transaction t; | |
4706 | t.remove_collection(cid); | |
4707 | r = apply_transaction(store, &osr, std::move(t)); | |
4708 | ASSERT_EQ(r, 0); | |
4709 | } | |
4710 | ||
4711 | ||
4712 | TEST_P(StoreTest, OMapTest) { | |
4713 | ObjectStore::Sequencer osr("test"); | |
4714 | coll_t cid; | |
4715 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
4716 | int r; | |
4717 | { | |
4718 | ObjectStore::Transaction t; | |
4719 | t.create_collection(cid, 0); | |
4720 | r = apply_transaction(store, &osr, std::move(t)); | |
4721 | ASSERT_EQ(r, 0); | |
4722 | } | |
4723 | ||
4724 | map<string, bufferlist> attrs; | |
4725 | { | |
4726 | ObjectStore::Transaction t; | |
4727 | t.touch(cid, hoid); | |
4728 | t.omap_clear(cid, hoid); | |
4729 | map<string, bufferlist> start_set; | |
4730 | t.omap_setkeys(cid, hoid, start_set); | |
4731 | r = apply_transaction(store, &osr, std::move(t)); | |
4732 | ASSERT_EQ(r, 0); | |
4733 | } | |
4734 | ||
4735 | for (int i = 0; i < 100; i++) { | |
4736 | if (!(i%5)) { | |
4737 | std::cout << "On iteration " << i << std::endl; | |
4738 | } | |
4739 | ObjectStore::Transaction t; | |
4740 | bufferlist bl; | |
4741 | map<string, bufferlist> cur_attrs; | |
4742 | r = store->omap_get(cid, hoid, &bl, &cur_attrs); | |
4743 | ASSERT_EQ(r, 0); | |
4744 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
4745 | j != attrs.end(); | |
4746 | ++j) { | |
4747 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
4748 | if (!correct) { | |
4749 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
4750 | if (cur_attrs.count(j->first) > 0) { | |
4751 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
4752 | } | |
4753 | } | |
4754 | ASSERT_EQ(correct, true); | |
4755 | } | |
4756 | ASSERT_EQ(attrs.size(), cur_attrs.size()); | |
4757 | ||
4758 | char buf[100]; | |
4759 | snprintf(buf, sizeof(buf), "%d", i); | |
4760 | bl.clear(); | |
4761 | bufferptr bp(buf, strlen(buf) + 1); | |
4762 | bl.append(bp); | |
4763 | map<string, bufferlist> to_add; | |
4764 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
4765 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
4766 | t.omap_setkeys(cid, hoid, to_add); | |
4767 | r = apply_transaction(store, &osr, std::move(t)); | |
4768 | ASSERT_EQ(r, 0); | |
4769 | } | |
4770 | ||
4771 | int i = 0; | |
4772 | while (attrs.size()) { | |
4773 | if (!(i%5)) { | |
4774 | std::cout << "removal: On iteration " << i << std::endl; | |
4775 | } | |
4776 | ObjectStore::Transaction t; | |
4777 | bufferlist bl; | |
4778 | map<string, bufferlist> cur_attrs; | |
4779 | r = store->omap_get(cid, hoid, &bl, &cur_attrs); | |
4780 | ASSERT_EQ(r, 0); | |
4781 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
4782 | j != attrs.end(); | |
4783 | ++j) { | |
4784 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
4785 | if (!correct) { | |
4786 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
4787 | if (cur_attrs.count(j->first) > 0) { | |
4788 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
4789 | } | |
4790 | } | |
4791 | ASSERT_EQ(correct, true); | |
4792 | } | |
4793 | ||
4794 | string to_remove = attrs.begin()->first; | |
4795 | set<string> keys_to_remove; | |
4796 | keys_to_remove.insert(to_remove); | |
4797 | t.omap_rmkeys(cid, hoid, keys_to_remove); | |
4798 | r = apply_transaction(store, &osr, std::move(t)); | |
4799 | ASSERT_EQ(r, 0); | |
4800 | ||
4801 | attrs.erase(to_remove); | |
4802 | ||
4803 | ++i; | |
4804 | } | |
4805 | ||
4806 | { | |
4807 | bufferlist bl1; | |
4808 | bl1.append("omap_header"); | |
4809 | ObjectStore::Transaction t; | |
4810 | t.omap_setheader(cid, hoid, bl1); | |
4811 | r = apply_transaction(store, &osr, std::move(t)); | |
4812 | ASSERT_EQ(r, 0); | |
4813 | t = ObjectStore::Transaction(); | |
4814 | ||
4815 | bufferlist bl2; | |
4816 | bl2.append("value"); | |
4817 | map<string, bufferlist> to_add; | |
4818 | to_add.insert(pair<string, bufferlist>("key", bl2)); | |
4819 | t.omap_setkeys(cid, hoid, to_add); | |
4820 | r = apply_transaction(store, &osr, std::move(t)); | |
4821 | ASSERT_EQ(r, 0); | |
4822 | ||
4823 | bufferlist bl3; | |
4824 | map<string, bufferlist> cur_attrs; | |
4825 | r = store->omap_get(cid, hoid, &bl3, &cur_attrs); | |
4826 | ASSERT_EQ(r, 0); | |
4827 | ASSERT_EQ(cur_attrs.size(), size_t(1)); | |
4828 | ASSERT_TRUE(bl_eq(bl1, bl3)); | |
4829 | ||
4830 | set<string> keys; | |
4831 | r = store->omap_get_keys(cid, hoid, &keys); | |
4832 | ASSERT_EQ(r, 0); | |
4833 | ASSERT_EQ(keys.size(), size_t(1)); | |
4834 | } | |
4835 | ||
4836 | // test omap_clear, omap_rmkey_range | |
4837 | { | |
4838 | { | |
4839 | map<string,bufferlist> to_set; | |
4840 | for (int n=0; n<10; ++n) { | |
4841 | to_set[stringify(n)].append("foo"); | |
4842 | } | |
4843 | bufferlist h; | |
4844 | h.append("header"); | |
4845 | ObjectStore::Transaction t; | |
4846 | t.remove(cid, hoid); | |
4847 | t.touch(cid, hoid); | |
4848 | t.omap_setheader(cid, hoid, h); | |
4849 | t.omap_setkeys(cid, hoid, to_set); | |
4850 | r = apply_transaction(store, &osr, std::move(t)); | |
4851 | ASSERT_EQ(r, 0); | |
4852 | } | |
4853 | { | |
4854 | ObjectStore::Transaction t; | |
4855 | t.omap_rmkeyrange(cid, hoid, "3", "7"); | |
4856 | r = apply_transaction(store, &osr, std::move(t)); | |
4857 | ASSERT_EQ(r, 0); | |
4858 | } | |
4859 | { | |
4860 | bufferlist hdr; | |
4861 | map<string,bufferlist> m; | |
4862 | store->omap_get(cid, hoid, &hdr, &m); | |
4863 | ASSERT_EQ(6u, hdr.length()); | |
4864 | ASSERT_TRUE(m.count("2")); | |
4865 | ASSERT_TRUE(!m.count("3")); | |
4866 | ASSERT_TRUE(!m.count("6")); | |
4867 | ASSERT_TRUE(m.count("7")); | |
4868 | ASSERT_TRUE(m.count("8")); | |
4869 | //cout << m << std::endl; | |
4870 | ASSERT_EQ(6u, m.size()); | |
4871 | } | |
4872 | { | |
4873 | ObjectStore::Transaction t; | |
4874 | t.omap_clear(cid, hoid); | |
4875 | r = apply_transaction(store, &osr, std::move(t)); | |
4876 | ASSERT_EQ(r, 0); | |
4877 | } | |
4878 | { | |
4879 | bufferlist hdr; | |
4880 | map<string,bufferlist> m; | |
4881 | store->omap_get(cid, hoid, &hdr, &m); | |
4882 | ASSERT_EQ(0u, hdr.length()); | |
4883 | ASSERT_EQ(0u, m.size()); | |
4884 | } | |
4885 | } | |
4886 | ||
4887 | ObjectStore::Transaction t; | |
4888 | t.remove(cid, hoid); | |
4889 | t.remove_collection(cid); | |
4890 | r = apply_transaction(store, &osr, std::move(t)); | |
4891 | ASSERT_EQ(r, 0); | |
4892 | } | |
4893 | ||
4894 | TEST_P(StoreTest, OMapIterator) { | |
4895 | ObjectStore::Sequencer osr("test"); | |
4896 | coll_t cid; | |
4897 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
4898 | int count = 0; | |
4899 | int r; | |
4900 | { | |
4901 | ObjectStore::Transaction t; | |
4902 | t.create_collection(cid, 0); | |
4903 | r = apply_transaction(store, &osr, std::move(t)); | |
4904 | ASSERT_EQ(r, 0); | |
4905 | } | |
4906 | ||
4907 | map<string, bufferlist> attrs; | |
4908 | { | |
4909 | ObjectStore::Transaction t; | |
4910 | t.touch(cid, hoid); | |
4911 | t.omap_clear(cid, hoid); | |
4912 | map<string, bufferlist> start_set; | |
4913 | t.omap_setkeys(cid, hoid, start_set); | |
4914 | r = apply_transaction(store, &osr, std::move(t)); | |
4915 | ASSERT_EQ(r, 0); | |
4916 | } | |
4917 | ObjectMap::ObjectMapIterator iter; | |
4918 | bool correct; | |
4919 | //basic iteration | |
4920 | for (int i = 0; i < 100; i++) { | |
4921 | if (!(i%5)) { | |
4922 | std::cout << "On iteration " << i << std::endl; | |
4923 | } | |
4924 | bufferlist bl; | |
4925 | ||
4926 | // FileStore may deadlock two active iterators over the same data | |
4927 | iter = ObjectMap::ObjectMapIterator(); | |
4928 | ||
4929 | iter = store->get_omap_iterator(cid, hoid); | |
4930 | for (iter->seek_to_first(), count=0; iter->valid(); iter->next(), count++) { | |
4931 | string key = iter->key(); | |
4932 | bufferlist value = iter->value(); | |
4933 | correct = attrs.count(key) && (string(value.c_str()) == string(attrs[key].c_str())); | |
4934 | if (!correct) { | |
4935 | if (attrs.count(key) > 0) { | |
4936 | std::cout << "key " << key << "in omap , " << value.c_str() << " : " << attrs[key].c_str() << std::endl; | |
4937 | } | |
4938 | else | |
4939 | std::cout << "key " << key << "should not exists in omap" << std::endl; | |
4940 | } | |
4941 | ASSERT_EQ(correct, true); | |
4942 | } | |
4943 | ASSERT_EQ((int)attrs.size(), count); | |
4944 | ||
4945 | // FileStore may deadlock an active iterator vs apply_transaction | |
4946 | iter = ObjectMap::ObjectMapIterator(); | |
4947 | ||
4948 | char buf[100]; | |
4949 | snprintf(buf, sizeof(buf), "%d", i); | |
4950 | bl.clear(); | |
4951 | bufferptr bp(buf, strlen(buf) + 1); | |
4952 | bl.append(bp); | |
4953 | map<string, bufferlist> to_add; | |
4954 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
4955 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
4956 | ObjectStore::Transaction t; | |
4957 | t.omap_setkeys(cid, hoid, to_add); | |
4958 | r = apply_transaction(store, &osr, std::move(t)); | |
4959 | ASSERT_EQ(r, 0); | |
4960 | } | |
4961 | ||
4962 | iter = store->get_omap_iterator(cid, hoid); | |
4963 | //lower bound | |
4964 | string bound_key = "key-5"; | |
4965 | iter->lower_bound(bound_key); | |
4966 | correct = bound_key <= iter->key(); | |
4967 | if (!correct) { | |
4968 | std::cout << "lower bound, bound key is " << bound_key << " < iter key is " << iter->key() << std::endl; | |
4969 | } | |
4970 | ASSERT_EQ(correct, true); | |
4971 | //upper bound | |
4972 | iter->upper_bound(bound_key); | |
4973 | correct = iter->key() > bound_key; | |
4974 | if (!correct) { | |
4975 | std::cout << "upper bound, bound key is " << bound_key << " >= iter key is " << iter->key() << std::endl; | |
4976 | } | |
4977 | ASSERT_EQ(correct, true); | |
4978 | ||
4979 | // FileStore may deadlock an active iterator vs apply_transaction | |
4980 | iter = ObjectMap::ObjectMapIterator(); | |
4981 | { | |
4982 | ObjectStore::Transaction t; | |
4983 | t.remove(cid, hoid); | |
4984 | t.remove_collection(cid); | |
4985 | r = apply_transaction(store, &osr, std::move(t)); | |
4986 | ASSERT_EQ(r, 0); | |
4987 | } | |
4988 | } | |
4989 | ||
4990 | TEST_P(StoreTest, XattrTest) { | |
4991 | ObjectStore::Sequencer osr("test"); | |
4992 | coll_t cid; | |
4993 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
4994 | bufferlist big; | |
4995 | for (unsigned i = 0; i < 10000; ++i) { | |
4996 | big.append('\0'); | |
4997 | } | |
4998 | bufferlist small; | |
4999 | for (unsigned i = 0; i < 10; ++i) { | |
5000 | small.append('\0'); | |
5001 | } | |
5002 | int r; | |
5003 | { | |
5004 | ObjectStore::Transaction t; | |
5005 | t.create_collection(cid, 0); | |
5006 | t.touch(cid, hoid); | |
5007 | r = apply_transaction(store, &osr, std::move(t)); | |
5008 | ASSERT_EQ(r, 0); | |
5009 | } | |
5010 | ||
5011 | map<string, bufferlist> attrs; | |
5012 | { | |
5013 | ObjectStore::Transaction t; | |
5014 | t.setattr(cid, hoid, "attr1", small); | |
5015 | attrs["attr1"] = small; | |
5016 | t.setattr(cid, hoid, "attr2", big); | |
5017 | attrs["attr2"] = big; | |
5018 | t.setattr(cid, hoid, "attr3", small); | |
5019 | attrs["attr3"] = small; | |
5020 | t.setattr(cid, hoid, "attr1", small); | |
5021 | attrs["attr1"] = small; | |
5022 | t.setattr(cid, hoid, "attr4", big); | |
5023 | attrs["attr4"] = big; | |
5024 | t.setattr(cid, hoid, "attr3", big); | |
5025 | attrs["attr3"] = big; | |
5026 | r = apply_transaction(store, &osr, std::move(t)); | |
5027 | ASSERT_EQ(r, 0); | |
5028 | } | |
5029 | ||
5030 | map<string, bufferptr> aset; | |
5031 | store->getattrs(cid, hoid, aset); | |
5032 | ASSERT_EQ(aset.size(), attrs.size()); | |
5033 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5034 | i != aset.end(); | |
5035 | ++i) { | |
5036 | bufferlist bl; | |
5037 | bl.push_back(i->second); | |
5038 | ASSERT_TRUE(attrs[i->first] == bl); | |
5039 | } | |
5040 | ||
5041 | { | |
5042 | ObjectStore::Transaction t; | |
5043 | t.rmattr(cid, hoid, "attr2"); | |
5044 | attrs.erase("attr2"); | |
5045 | r = apply_transaction(store, &osr, std::move(t)); | |
5046 | ASSERT_EQ(r, 0); | |
5047 | } | |
5048 | ||
5049 | aset.clear(); | |
5050 | store->getattrs(cid, hoid, aset); | |
5051 | ASSERT_EQ(aset.size(), attrs.size()); | |
5052 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5053 | i != aset.end(); | |
5054 | ++i) { | |
5055 | bufferlist bl; | |
5056 | bl.push_back(i->second); | |
5057 | ASSERT_TRUE(attrs[i->first] == bl); | |
5058 | } | |
5059 | ||
5060 | bufferptr bp; | |
5061 | r = store->getattr(cid, hoid, "attr2", bp); | |
5062 | ASSERT_EQ(r, -ENODATA); | |
5063 | ||
5064 | r = store->getattr(cid, hoid, "attr3", bp); | |
5065 | ASSERT_EQ(r, 0); | |
5066 | bufferlist bl2; | |
5067 | bl2.push_back(bp); | |
5068 | ASSERT_TRUE(bl2 == attrs["attr3"]); | |
5069 | ||
5070 | ObjectStore::Transaction t; | |
5071 | t.remove(cid, hoid); | |
5072 | t.remove_collection(cid); | |
5073 | r = apply_transaction(store, &osr, std::move(t)); | |
5074 | ASSERT_EQ(r, 0); | |
5075 | } | |
5076 | ||
5077 | void colsplittest( | |
5078 | ObjectStore *store, | |
5079 | unsigned num_objects, | |
5080 | unsigned common_suffix_size, | |
5081 | bool clones | |
5082 | ) { | |
5083 | ObjectStore::Sequencer osr("test"); | |
5084 | coll_t cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD)); | |
5085 | coll_t tid(spg_t(pg_t(1<<common_suffix_size,52),shard_id_t::NO_SHARD)); | |
5086 | int r = 0; | |
5087 | { | |
5088 | ObjectStore::Transaction t; | |
5089 | t.create_collection(cid, common_suffix_size); | |
5090 | r = apply_transaction(store, &osr, std::move(t)); | |
5091 | ASSERT_EQ(r, 0); | |
5092 | } | |
5093 | bufferlist small; | |
5094 | small.append("small"); | |
5095 | { | |
5096 | ObjectStore::Transaction t; | |
5097 | for (uint32_t i = 0; i < (2 - (int)clones)*num_objects; ++i) { | |
5098 | stringstream objname; | |
5099 | objname << "obj" << i; | |
5100 | ghobject_t a(hobject_t( | |
5101 | objname.str(), | |
5102 | "", | |
5103 | CEPH_NOSNAP, | |
5104 | i<<common_suffix_size, | |
5105 | 52, "")); | |
5106 | t.write(cid, a, 0, small.length(), small, | |
5107 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5108 | if (clones) { | |
5109 | objname << "-clone"; | |
5110 | ghobject_t b(hobject_t( | |
5111 | objname.str(), | |
5112 | "", | |
5113 | CEPH_NOSNAP, | |
5114 | i<<common_suffix_size, | |
5115 | 52, "")); | |
5116 | t.clone(cid, a, b); | |
5117 | } | |
5118 | if (i % 100) { | |
5119 | r = apply_transaction(store, &osr, std::move(t)); | |
5120 | ASSERT_EQ(r, 0); | |
5121 | t = ObjectStore::Transaction(); | |
5122 | } | |
5123 | } | |
5124 | r = apply_transaction(store, &osr, std::move(t)); | |
5125 | ASSERT_EQ(r, 0); | |
5126 | } | |
5127 | { | |
5128 | ObjectStore::Transaction t; | |
5129 | t.create_collection(tid, common_suffix_size + 1); | |
5130 | t.split_collection(cid, common_suffix_size+1, 1<<common_suffix_size, tid); | |
5131 | r = apply_transaction(store, &osr, std::move(t)); | |
5132 | ASSERT_EQ(r, 0); | |
5133 | } | |
5134 | ||
5135 | ObjectStore::Transaction t; | |
5136 | vector<ghobject_t> objects; | |
5137 | r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), | |
5138 | INT_MAX, &objects, 0); | |
5139 | ASSERT_EQ(r, 0); | |
5140 | ASSERT_EQ(objects.size(), num_objects); | |
5141 | unsigned size = 0; | |
5142 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5143 | i != objects.end(); | |
5144 | ++i) { | |
5145 | ASSERT_EQ(!!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
5146 | t.remove(cid, *i); | |
5147 | if (++size > 100) { | |
5148 | size = 0; | |
5149 | r = apply_transaction(store, &osr, std::move(t)); | |
5150 | ASSERT_EQ(r, 0); | |
5151 | t = ObjectStore::Transaction(); | |
5152 | } | |
5153 | } | |
5154 | ||
5155 | objects.clear(); | |
5156 | r = store->collection_list(tid, ghobject_t(), ghobject_t::get_max(), | |
5157 | INT_MAX, &objects, 0); | |
5158 | ASSERT_EQ(r, 0); | |
5159 | ASSERT_EQ(objects.size(), num_objects); | |
5160 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5161 | i != objects.end(); | |
5162 | ++i) { | |
5163 | ASSERT_EQ(!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
5164 | t.remove(tid, *i); | |
5165 | if (++size > 100) { | |
5166 | size = 0; | |
5167 | r = apply_transaction(store, &osr, std::move(t)); | |
5168 | ASSERT_EQ(r, 0); | |
5169 | t = ObjectStore::Transaction(); | |
5170 | } | |
5171 | } | |
5172 | ||
5173 | t.remove_collection(cid); | |
5174 | t.remove_collection(tid); | |
5175 | r = apply_transaction(store, &osr, std::move(t)); | |
5176 | ASSERT_EQ(r, 0); | |
5177 | } | |
5178 | ||
5179 | TEST_P(StoreTest, ColSplitTest1) { | |
5180 | colsplittest(store.get(), 10000, 11, false); | |
5181 | } | |
5182 | TEST_P(StoreTest, ColSplitTest1Clones) { | |
5183 | colsplittest(store.get(), 10000, 11, true); | |
5184 | } | |
5185 | TEST_P(StoreTest, ColSplitTest2) { | |
5186 | colsplittest(store.get(), 100, 7, false); | |
5187 | } | |
5188 | TEST_P(StoreTest, ColSplitTest2Clones) { | |
5189 | colsplittest(store.get(), 100, 7, true); | |
5190 | } | |
5191 | ||
5192 | #if 0 | |
5193 | TEST_P(StoreTest, ColSplitTest3) { | |
5194 | colsplittest(store.get(), 100000, 25); | |
5195 | } | |
5196 | #endif | |
5197 | ||
5198 | /** | |
5199 | * This test tests adding two different groups | |
5200 | * of objects, each with 1 common prefix and 1 | |
5201 | * different prefix. We then remove half | |
5202 | * in order to verify that the merging correctly | |
5203 | * stops at the common prefix subdir. See bug | |
5204 | * #5273 */ | |
5205 | TEST_P(StoreTest, TwoHash) { | |
5206 | ObjectStore::Sequencer osr("test"); | |
5207 | coll_t cid; | |
5208 | int r; | |
5209 | { | |
5210 | ObjectStore::Transaction t; | |
5211 | t.create_collection(cid, 0); | |
5212 | r = apply_transaction(store, &osr, std::move(t)); | |
5213 | ASSERT_EQ(r, 0); | |
5214 | } | |
5215 | std::cout << "Making objects" << std::endl; | |
5216 | for (int i = 0; i < 360; ++i) { | |
5217 | ObjectStore::Transaction t; | |
5218 | ghobject_t o; | |
5219 | o.hobj.pool = -1; | |
5220 | if (i < 8) { | |
5221 | o.hobj.set_hash((i << 16) | 0xA1); | |
5222 | t.touch(cid, o); | |
5223 | } | |
5224 | o.hobj.set_hash((i << 16) | 0xB1); | |
5225 | t.touch(cid, o); | |
5226 | r = apply_transaction(store, &osr, std::move(t)); | |
5227 | ASSERT_EQ(r, 0); | |
5228 | } | |
5229 | std::cout << "Removing half" << std::endl; | |
5230 | for (int i = 1; i < 8; ++i) { | |
5231 | ObjectStore::Transaction t; | |
5232 | ghobject_t o; | |
5233 | o.hobj.pool = -1; | |
5234 | o.hobj.set_hash((i << 16) | 0xA1); | |
5235 | t.remove(cid, o); | |
5236 | r = apply_transaction(store, &osr, std::move(t)); | |
5237 | ASSERT_EQ(r, 0); | |
5238 | } | |
5239 | std::cout << "Checking" << std::endl; | |
5240 | for (int i = 1; i < 8; ++i) { | |
5241 | ObjectStore::Transaction t; | |
5242 | ghobject_t o; | |
5243 | o.hobj.set_hash((i << 16) | 0xA1); | |
5244 | o.hobj.pool = -1; | |
5245 | bool exists = store->exists(cid, o); | |
5246 | ASSERT_EQ(exists, false); | |
5247 | } | |
5248 | { | |
5249 | ghobject_t o; | |
5250 | o.hobj.set_hash(0xA1); | |
5251 | o.hobj.pool = -1; | |
5252 | bool exists = store->exists(cid, o); | |
5253 | ASSERT_EQ(exists, true); | |
5254 | } | |
5255 | std::cout << "Cleanup" << std::endl; | |
5256 | for (int i = 0; i < 360; ++i) { | |
5257 | ObjectStore::Transaction t; | |
5258 | ghobject_t o; | |
5259 | o.hobj.set_hash((i << 16) | 0xA1); | |
5260 | o.hobj.pool = -1; | |
5261 | t.remove(cid, o); | |
5262 | o.hobj.set_hash((i << 16) | 0xB1); | |
5263 | t.remove(cid, o); | |
5264 | r = apply_transaction(store, &osr, std::move(t)); | |
5265 | ASSERT_EQ(r, 0); | |
5266 | } | |
5267 | ObjectStore::Transaction t; | |
5268 | t.remove_collection(cid); | |
5269 | r = apply_transaction(store, &osr, std::move(t)); | |
5270 | ASSERT_EQ(r, 0); | |
5271 | } | |
5272 | ||
5273 | TEST_P(StoreTest, Rename) { | |
5274 | ObjectStore::Sequencer osr("test"); | |
5275 | coll_t cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD)); | |
5276 | ghobject_t srcoid(hobject_t("src_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5277 | ghobject_t dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5278 | bufferlist a, b; | |
5279 | a.append("foo"); | |
5280 | b.append("bar"); | |
5281 | int r; | |
5282 | { | |
5283 | ObjectStore::Transaction t; | |
5284 | t.create_collection(cid, 0); | |
5285 | t.write(cid, srcoid, 0, a.length(), a); | |
5286 | r = apply_transaction(store, &osr, std::move(t)); | |
5287 | ASSERT_EQ(r, 0); | |
5288 | } | |
5289 | ASSERT_TRUE(store->exists(cid, srcoid)); | |
5290 | { | |
5291 | ObjectStore::Transaction t; | |
5292 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
5293 | t.write(cid, srcoid, 0, b.length(), b); | |
5294 | t.setattr(cid, srcoid, "attr", b); | |
5295 | r = apply_transaction(store, &osr, std::move(t)); | |
5296 | ASSERT_EQ(r, 0); | |
5297 | } | |
5298 | ASSERT_TRUE(store->exists(cid, srcoid)); | |
5299 | ASSERT_TRUE(store->exists(cid, dstoid)); | |
5300 | { | |
5301 | bufferlist bl; | |
5302 | store->read(cid, srcoid, 0, 3, bl); | |
5303 | ASSERT_TRUE(bl_eq(b, bl)); | |
5304 | store->read(cid, dstoid, 0, 3, bl); | |
5305 | ASSERT_TRUE(bl_eq(a, bl)); | |
5306 | } | |
5307 | { | |
5308 | ObjectStore::Transaction t; | |
5309 | t.remove(cid, dstoid); | |
5310 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
5311 | r = apply_transaction(store, &osr, std::move(t)); | |
5312 | ASSERT_EQ(r, 0); | |
5313 | } | |
5314 | ASSERT_TRUE(store->exists(cid, dstoid)); | |
5315 | ASSERT_FALSE(store->exists(cid, srcoid)); | |
5316 | { | |
5317 | bufferlist bl; | |
5318 | store->read(cid, dstoid, 0, 3, bl); | |
5319 | ASSERT_TRUE(bl_eq(b, bl)); | |
5320 | } | |
5321 | { | |
5322 | ObjectStore::Transaction t; | |
5323 | t.remove(cid, dstoid); | |
5324 | t.remove_collection(cid); | |
5325 | r = apply_transaction(store, &osr, std::move(t)); | |
5326 | ASSERT_EQ(r, 0); | |
5327 | } | |
5328 | } | |
5329 | ||
5330 | TEST_P(StoreTest, MoveRename) { | |
5331 | ObjectStore::Sequencer osr("test"); | |
5332 | coll_t cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD)); | |
5333 | ghobject_t temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5334 | ghobject_t oid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5335 | int r; | |
5336 | { | |
5337 | ObjectStore::Transaction t; | |
5338 | t.create_collection(cid, 0); | |
5339 | t.touch(cid, oid); | |
5340 | r = apply_transaction(store, &osr, std::move(t)); | |
5341 | ASSERT_EQ(r, 0); | |
5342 | } | |
5343 | ASSERT_TRUE(store->exists(cid, oid)); | |
5344 | bufferlist data, attr; | |
5345 | map<string, bufferlist> omap; | |
5346 | data.append("data payload"); | |
5347 | attr.append("attr value"); | |
5348 | omap["omap_key"].append("omap value"); | |
5349 | { | |
5350 | ObjectStore::Transaction t; | |
5351 | t.touch(cid, temp_oid); | |
5352 | t.write(cid, temp_oid, 0, data.length(), data); | |
5353 | t.setattr(cid, temp_oid, "attr", attr); | |
5354 | t.omap_setkeys(cid, temp_oid, omap); | |
5355 | r = apply_transaction(store, &osr, std::move(t)); | |
5356 | ASSERT_EQ(r, 0); | |
5357 | } | |
5358 | ASSERT_TRUE(store->exists(cid, temp_oid)); | |
5359 | { | |
5360 | ObjectStore::Transaction t; | |
5361 | t.remove(cid, oid); | |
5362 | t.collection_move_rename(cid, temp_oid, cid, oid); | |
5363 | r = apply_transaction(store, &osr, std::move(t)); | |
5364 | ASSERT_EQ(r, 0); | |
5365 | } | |
5366 | ASSERT_TRUE(store->exists(cid, oid)); | |
5367 | ASSERT_FALSE(store->exists(cid, temp_oid)); | |
5368 | { | |
5369 | bufferlist newdata; | |
5370 | r = store->read(cid, oid, 0, 1000, newdata); | |
5371 | ASSERT_GE(r, 0); | |
5372 | ASSERT_TRUE(bl_eq(data, newdata)); | |
5373 | bufferlist newattr; | |
5374 | r = store->getattr(cid, oid, "attr", newattr); | |
5375 | ASSERT_EQ(r, 0); | |
5376 | ASSERT_TRUE(bl_eq(attr, newattr)); | |
5377 | set<string> keys; | |
5378 | keys.insert("omap_key"); | |
5379 | map<string, bufferlist> newomap; | |
5380 | r = store->omap_get_values(cid, oid, keys, &newomap); | |
5381 | ASSERT_GE(r, 0); | |
5382 | ASSERT_EQ(1u, newomap.size()); | |
5383 | ASSERT_TRUE(newomap.count("omap_key")); | |
5384 | ASSERT_TRUE(bl_eq(omap["omap_key"], newomap["omap_key"])); | |
5385 | } | |
5386 | { | |
5387 | ObjectStore::Transaction t; | |
5388 | t.remove(cid, oid); | |
5389 | t.remove_collection(cid); | |
5390 | r = apply_transaction(store, &osr, std::move(t)); | |
5391 | ASSERT_EQ(r, 0); | |
5392 | } | |
5393 | } | |
5394 | ||
5395 | TEST_P(StoreTest, BigRGWObjectName) { | |
5396 | ObjectStore::Sequencer osr("test"); | |
5397 | coll_t cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD)); | |
5398 | ghobject_t oid( | |
5399 | hobject_t( | |
5400 | "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", | |
5401 | "", | |
5402 | CEPH_NOSNAP, | |
5403 | 0x81920472, | |
5404 | 12, | |
5405 | ""), | |
5406 | 15, | |
5407 | shard_id_t::NO_SHARD); | |
5408 | ghobject_t oid2(oid); | |
5409 | oid2.generation = 17; | |
5410 | ghobject_t oidhead(oid); | |
5411 | oidhead.generation = ghobject_t::NO_GEN; | |
5412 | ||
5413 | int r; | |
5414 | { | |
5415 | ObjectStore::Transaction t; | |
5416 | t.create_collection(cid, 0); | |
5417 | t.touch(cid, oidhead); | |
5418 | t.collection_move_rename(cid, oidhead, cid, oid); | |
5419 | t.touch(cid, oidhead); | |
5420 | t.collection_move_rename(cid, oidhead, cid, oid2); | |
5421 | r = apply_transaction(store, &osr, std::move(t)); | |
5422 | ASSERT_EQ(r, 0); | |
5423 | } | |
5424 | ||
5425 | { | |
5426 | ObjectStore::Transaction t; | |
5427 | t.remove(cid, oid); | |
5428 | r = apply_transaction(store, &osr, std::move(t)); | |
5429 | ASSERT_EQ(r, 0); | |
5430 | } | |
5431 | ||
5432 | { | |
5433 | vector<ghobject_t> objects; | |
5434 | r = store->collection_list(cid, ghobject_t(), ghobject_t::get_max(), | |
5435 | INT_MAX, &objects, 0); | |
5436 | ASSERT_EQ(r, 0); | |
5437 | ASSERT_EQ(objects.size(), 1u); | |
5438 | ASSERT_EQ(objects[0], oid2); | |
5439 | } | |
5440 | ||
5441 | ASSERT_FALSE(store->exists(cid, oid)); | |
5442 | ||
5443 | { | |
5444 | ObjectStore::Transaction t; | |
5445 | t.remove(cid, oid2); | |
5446 | t.remove_collection(cid); | |
5447 | r = apply_transaction(store, &osr, std::move(t)); | |
5448 | ASSERT_EQ(r, 0); | |
5449 | ||
5450 | } | |
5451 | } | |
5452 | ||
5453 | TEST_P(StoreTest, SetAllocHint) { | |
5454 | ObjectStore::Sequencer osr("test"); | |
5455 | coll_t cid; | |
5456 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, "")); | |
5457 | int r; | |
5458 | { | |
5459 | ObjectStore::Transaction t; | |
5460 | t.create_collection(cid, 0); | |
5461 | t.touch(cid, hoid); | |
5462 | r = apply_transaction(store, &osr, std::move(t)); | |
5463 | ASSERT_EQ(r, 0); | |
5464 | } | |
5465 | { | |
5466 | ObjectStore::Transaction t; | |
5467 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
5468 | r = apply_transaction(store, &osr, std::move(t)); | |
5469 | ASSERT_EQ(r, 0); | |
5470 | } | |
5471 | { | |
5472 | ObjectStore::Transaction t; | |
5473 | t.remove(cid, hoid); | |
5474 | r = apply_transaction(store, &osr, std::move(t)); | |
5475 | ASSERT_EQ(r, 0); | |
5476 | } | |
5477 | { | |
5478 | ObjectStore::Transaction t; | |
5479 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
5480 | r = apply_transaction(store, &osr, std::move(t)); | |
5481 | ASSERT_EQ(r, 0); | |
5482 | } | |
5483 | { | |
5484 | ObjectStore::Transaction t; | |
5485 | t.remove_collection(cid); | |
5486 | r = apply_transaction(store, &osr, std::move(t)); | |
5487 | ASSERT_EQ(r, 0); | |
5488 | } | |
5489 | } | |
5490 | ||
5491 | TEST_P(StoreTest, TryMoveRename) { | |
5492 | ObjectStore::Sequencer osr("test"); | |
5493 | coll_t cid; | |
5494 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
5495 | ghobject_t hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP, 0, -1, "")); | |
5496 | int r; | |
5497 | { | |
5498 | ObjectStore::Transaction t; | |
5499 | t.create_collection(cid, 0); | |
5500 | r = apply_transaction(store, &osr, std::move(t)); | |
5501 | ASSERT_EQ(r, 0); | |
5502 | } | |
5503 | { | |
5504 | ObjectStore::Transaction t; | |
5505 | t.try_rename(cid, hoid, hoid2); | |
5506 | r = apply_transaction(store, &osr, std::move(t)); | |
5507 | ASSERT_EQ(r, 0); | |
5508 | } | |
5509 | { | |
5510 | ObjectStore::Transaction t; | |
5511 | t.touch(cid, hoid); | |
5512 | r = apply_transaction(store, &osr, std::move(t)); | |
5513 | ASSERT_EQ(r, 0); | |
5514 | } | |
5515 | { | |
5516 | ObjectStore::Transaction t; | |
5517 | t.try_rename(cid, hoid, hoid2); | |
5518 | r = apply_transaction(store, &osr, std::move(t)); | |
5519 | ASSERT_EQ(r, 0); | |
5520 | } | |
5521 | struct stat st; | |
5522 | ASSERT_EQ(store->stat(cid, hoid, &st), -ENOENT); | |
5523 | ASSERT_EQ(store->stat(cid, hoid2, &st), 0); | |
5524 | } | |
5525 | ||
5526 | #if defined(HAVE_LIBAIO) | |
5527 | TEST_P(StoreTest, BluestoreOnOffCSumTest) { | |
5528 | if (string(GetParam()) != "bluestore") | |
5529 | return; | |
5530 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5531 | g_conf->apply_changes(NULL); | |
5532 | ||
5533 | ObjectStore::Sequencer osr("test"); | |
5534 | int r; | |
5535 | coll_t cid; | |
5536 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
5537 | { | |
5538 | bufferlist in; | |
5539 | r = store->read(cid, hoid, 0, 5, in); | |
5540 | ASSERT_EQ(-ENOENT, r); | |
5541 | } | |
5542 | { | |
5543 | ObjectStore::Transaction t; | |
5544 | t.create_collection(cid, 0); | |
5545 | cerr << "Creating collection " << cid << std::endl; | |
5546 | r = apply_transaction(store, &osr, std::move(t)); | |
5547 | ASSERT_EQ(r, 0); | |
5548 | } | |
5549 | { | |
5550 | //write with csum enabled followed by read with csum disabled | |
5551 | size_t block_size = 64*1024; | |
5552 | ObjectStore::Transaction t; | |
5553 | bufferlist bl, orig; | |
5554 | bl.append(std::string(block_size, 'a')); | |
5555 | orig = bl; | |
5556 | t.remove(cid, hoid); | |
5557 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
5558 | t.write(cid, hoid, 0, bl.length(), bl); | |
5559 | cerr << "Remove then create" << std::endl; | |
5560 | r = apply_transaction(store, &osr, std::move(t)); | |
5561 | ASSERT_EQ(r, 0); | |
5562 | ||
5563 | g_conf->set_val("bluestore_csum_type", "none"); | |
5564 | g_conf->apply_changes(NULL); | |
5565 | ||
5566 | bufferlist in; | |
5567 | r = store->read(cid, hoid, 0, block_size, in); | |
5568 | ASSERT_EQ((int)block_size, r); | |
5569 | ASSERT_TRUE(bl_eq(orig, in)); | |
5570 | ||
5571 | } | |
5572 | { | |
5573 | //write with csum disabled followed by read with csum enabled | |
5574 | ||
5575 | size_t block_size = 64*1024; | |
5576 | ObjectStore::Transaction t; | |
5577 | bufferlist bl, orig; | |
5578 | bl.append(std::string(block_size, 'a')); | |
5579 | orig = bl; | |
5580 | t.remove(cid, hoid); | |
5581 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
5582 | t.write(cid, hoid, 0, bl.length(), bl); | |
5583 | cerr << "Remove then create" << std::endl; | |
5584 | r = apply_transaction(store, &osr, std::move(t)); | |
5585 | ASSERT_EQ(r, 0); | |
5586 | ||
5587 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5588 | g_conf->apply_changes(NULL); | |
5589 | ||
5590 | bufferlist in; | |
5591 | r = store->read(cid, hoid, 0, block_size, in); | |
5592 | ASSERT_EQ((int)block_size, r); | |
5593 | ASSERT_TRUE(bl_eq(orig, in)); | |
5594 | } | |
5595 | { | |
5596 | //'mixed' non-overlapping writes to the same blob | |
5597 | ||
5598 | ObjectStore::Transaction t; | |
5599 | bufferlist bl, orig; | |
5600 | size_t block_size = 8000; | |
5601 | bl.append(std::string(block_size, 'a')); | |
5602 | orig = bl; | |
5603 | t.remove(cid, hoid); | |
5604 | t.write(cid, hoid, 0, bl.length(), bl); | |
5605 | cerr << "Remove then create" << std::endl; | |
5606 | r = apply_transaction(store, &osr, std::move(t)); | |
5607 | ASSERT_EQ(r, 0); | |
5608 | ||
5609 | g_conf->set_val("bluestore_csum_type", "none"); | |
5610 | g_conf->apply_changes(NULL); | |
5611 | ||
5612 | ObjectStore::Transaction t2; | |
5613 | t2.write(cid, hoid, block_size*2, bl.length(), bl); | |
5614 | cerr << "Append 'unprotected'" << std::endl; | |
5615 | r = apply_transaction(store, &osr, std::move(t2)); | |
5616 | ASSERT_EQ(r, 0); | |
5617 | ||
5618 | bufferlist in; | |
5619 | r = store->read(cid, hoid, 0, block_size, in); | |
5620 | ASSERT_EQ((int)block_size, r); | |
5621 | ASSERT_TRUE(bl_eq(orig, in)); | |
5622 | in.clear(); | |
5623 | r = store->read(cid, hoid, block_size*2, block_size, in); | |
5624 | ASSERT_EQ((int)block_size, r); | |
5625 | ASSERT_TRUE(bl_eq(orig, in)); | |
5626 | ||
5627 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5628 | g_conf->apply_changes(NULL); | |
5629 | in.clear(); | |
5630 | r = store->read(cid, hoid, 0, block_size, in); | |
5631 | ASSERT_EQ((int)block_size, r); | |
5632 | ASSERT_TRUE(bl_eq(orig, in)); | |
5633 | in.clear(); | |
5634 | r = store->read(cid, hoid, block_size*2, block_size, in); | |
5635 | ASSERT_EQ((int)block_size, r); | |
5636 | ASSERT_TRUE(bl_eq(orig, in)); | |
5637 | } | |
5638 | { | |
5639 | //partially blob overwrite under a different csum enablement mode | |
5640 | ||
5641 | ObjectStore::Transaction t; | |
5642 | bufferlist bl, orig, orig2; | |
5643 | size_t block_size0 = 0x10000; | |
5644 | size_t block_size = 9000; | |
5645 | size_t block_size2 = 5000; | |
5646 | bl.append(std::string(block_size0, 'a')); | |
5647 | t.remove(cid, hoid); | |
5648 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
5649 | t.write(cid, hoid, 0, bl.length(), bl); | |
5650 | cerr << "Remove then create" << std::endl; | |
5651 | r = apply_transaction(store, &osr, std::move(t)); | |
5652 | ASSERT_EQ(r, 0); | |
5653 | ||
5654 | g_conf->set_val("bluestore_csum_type", "none"); | |
5655 | g_conf->apply_changes(NULL); | |
5656 | ||
5657 | ObjectStore::Transaction t2; | |
5658 | bl.clear(); | |
5659 | bl.append(std::string(block_size, 'b')); | |
5660 | t2.write(cid, hoid, 0, bl.length(), bl); | |
5661 | t2.write(cid, hoid, block_size0, bl.length(), bl); | |
5662 | cerr << "Overwrite with unprotected data" << std::endl; | |
5663 | r = apply_transaction(store, &osr, std::move(t2)); | |
5664 | ASSERT_EQ(r, 0); | |
5665 | ||
5666 | orig = bl; | |
5667 | orig2 = bl; | |
5668 | orig.append( std::string(block_size0 - block_size, 'a')); | |
5669 | ||
5670 | bufferlist in; | |
5671 | r = store->read(cid, hoid, 0, block_size0, in); | |
5672 | ASSERT_EQ((int)block_size0, r); | |
5673 | ASSERT_TRUE(bl_eq(orig, in)); | |
5674 | ||
5675 | r = store->read(cid, hoid, block_size0, block_size, in); | |
5676 | ASSERT_EQ((int)block_size, r); | |
5677 | ASSERT_TRUE(bl_eq(orig2, in)); | |
5678 | ||
5679 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5680 | g_conf->apply_changes(NULL); | |
5681 | ||
5682 | ObjectStore::Transaction t3; | |
5683 | bl.clear(); | |
5684 | bl.append(std::string(block_size2, 'c')); | |
5685 | t3.write(cid, hoid, block_size0, bl.length(), bl); | |
5686 | cerr << "Overwrite with protected data" << std::endl; | |
5687 | r = apply_transaction(store, &osr, std::move(t3)); | |
5688 | ASSERT_EQ(r, 0); | |
5689 | ||
5690 | in.clear(); | |
5691 | orig = bl; | |
5692 | orig.append( std::string(block_size - block_size2, 'b')); | |
5693 | r = store->read(cid, hoid, block_size0, block_size, in); | |
5694 | ASSERT_EQ((int)block_size, r); | |
5695 | ASSERT_TRUE(bl_eq(orig, in)); | |
5696 | } | |
5697 | ||
5698 | { | |
5699 | ObjectStore::Transaction t; | |
5700 | t.remove(cid, hoid); | |
5701 | t.remove_collection(cid); | |
5702 | cerr << "Cleaning" << std::endl; | |
5703 | r = apply_transaction(store, &osr, std::move(t)); | |
5704 | ASSERT_EQ(r, 0); | |
5705 | } | |
5706 | } | |
5707 | #endif | |
5708 | ||
5709 | INSTANTIATE_TEST_CASE_P( | |
5710 | ObjectStore, | |
5711 | StoreTest, | |
5712 | ::testing::Values( | |
5713 | "memstore", | |
5714 | "filestore", | |
5715 | #if defined(HAVE_LIBAIO) | |
5716 | "bluestore", | |
5717 | #endif | |
5718 | "kstore")); | |
5719 | ||
5720 | // Note: instantiate all stores to preserve store numbering order only | |
5721 | INSTANTIATE_TEST_CASE_P( | |
5722 | ObjectStore, | |
5723 | StoreTestSpecificAUSize, | |
5724 | ::testing::Values( | |
5725 | "memstore", | |
5726 | "filestore", | |
5727 | #if defined(HAVE_LIBAIO) | |
5728 | "bluestore", | |
5729 | #endif | |
5730 | "kstore")); | |
5731 | ||
5732 | #else | |
5733 | ||
5734 | // Google Test may not support value-parameterized tests with some | |
5735 | // compilers. If we use conditional compilation to compile out all | |
5736 | // code referring to the gtest_main library, MSVC linker will not link | |
5737 | // that library at all and consequently complain about missing entry | |
5738 | // point defined in that library (fatal error LNK1561: entry point | |
5739 | // must be defined). This dummy test keeps gtest_main linked in. | |
5740 | TEST(DummyTest, ValueParameterizedTestsAreNotSupportedOnThisPlatform) {} | |
5741 | ||
5742 | #endif | |
5743 | ||
5744 | void doMany4KWritesTest(boost::scoped_ptr<ObjectStore>& store, | |
5745 | unsigned max_objects, | |
5746 | unsigned max_ops, | |
5747 | unsigned max_object_size, | |
5748 | unsigned max_write_size, | |
5749 | unsigned write_alignment, | |
5750 | store_statfs_t* res_stat) | |
5751 | { | |
5752 | ObjectStore::Sequencer osr("test"); | |
5753 | MixedGenerator gen(555); | |
5754 | gen_type rng(time(NULL)); | |
5755 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
5756 | ||
5757 | SyntheticWorkloadState test_obj(store.get(), | |
5758 | &gen, | |
5759 | &rng, | |
5760 | &osr, | |
5761 | cid, | |
5762 | max_object_size, | |
5763 | max_write_size, | |
5764 | write_alignment); | |
5765 | test_obj.init(); | |
5766 | for (unsigned i = 0; i < max_objects; ++i) { | |
5767 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
5768 | test_obj.touch(); | |
5769 | } | |
5770 | for (unsigned i = 0; i < max_ops; ++i) { | |
5771 | if (!(i % 200)) { | |
5772 | cerr << "Op " << i << std::endl; | |
5773 | test_obj.print_internal_state(); | |
5774 | } | |
5775 | test_obj.write(); | |
5776 | } | |
5777 | test_obj.wait_for_done(); | |
5778 | if (res_stat) { | |
5779 | test_obj.statfs(*res_stat); | |
5780 | } | |
5781 | test_obj.shutdown(); | |
5782 | } | |
5783 | ||
5784 | TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { | |
5785 | if (string(GetParam()) != "bluestore") | |
5786 | return; | |
5787 | ||
5788 | StartDeferred(0x10000); | |
5789 | ||
5790 | store_statfs_t res_stat; | |
5791 | unsigned max_object = 4*1024*1024; | |
5792 | ||
5793 | doMany4KWritesTest(store, 1, 1000, 4*1024*1024, 4*1024, 0, &res_stat); | |
5794 | ||
5795 | ASSERT_LE(res_stat.stored, max_object); | |
5796 | ASSERT_EQ(res_stat.allocated, max_object); | |
5797 | } | |
5798 | ||
5799 | TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { | |
5800 | if (string(GetParam()) != "bluestore") | |
5801 | return; | |
5802 | StartDeferred(0x10000); | |
5803 | g_conf->set_val("bluestore_csum_type", "none"); | |
5804 | g_ceph_context->_conf->apply_changes(NULL); | |
5805 | store_statfs_t res_stat; | |
5806 | unsigned max_object = 4*1024*1024; | |
5807 | ||
5808 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0, &res_stat ); | |
5809 | ||
5810 | ASSERT_LE(res_stat.stored, max_object); | |
5811 | ASSERT_EQ(res_stat.allocated, max_object); | |
5812 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5813 | } | |
5814 | ||
5815 | TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { | |
5816 | if (string(GetParam()) != "bluestore") | |
5817 | return; | |
5818 | StartDeferred(0x10000); | |
5819 | store_statfs_t res_stat; | |
5820 | unsigned max_object = 4*1024*1024; | |
5821 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0, &res_stat); | |
5822 | ASSERT_LE(res_stat.stored, max_object); | |
5823 | ASSERT_EQ(res_stat.allocated, max_object); | |
5824 | } | |
5825 | ||
5826 | #if defined(HAVE_LIBAIO) | |
5827 | void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items) | |
5828 | { | |
5829 | uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes(); | |
5830 | uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes(); | |
5831 | ||
5832 | uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items(); | |
5833 | uint64_t other_items = mempool::bluestore_cache_other::allocated_items(); | |
5834 | cout << "onode(" << onode_allocated << "/" << onode_items | |
5835 | << ") other(" << other_allocated << "/" << other_items | |
5836 | << ")" << std::endl; | |
5837 | *total_bytes = onode_allocated + other_allocated; | |
5838 | *total_items = onode_items; | |
5839 | } | |
5840 | ||
5841 | TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) { | |
5842 | ||
5843 | if (string(GetParam()) != "bluestore") | |
5844 | return; | |
5845 | ||
5846 | size_t block_size = 4096; | |
5847 | StartDeferred(block_size); | |
5848 | g_conf->set_val("bluestore_compression_mode", "none"); | |
5849 | g_conf->set_val("bluestore_csum_type", "none"); | |
5850 | g_conf->set_val("bluestore_cache_size_hdd", "400000000"); | |
5851 | g_conf->set_val("bluestore_cache_size_ssd", "400000000"); | |
5852 | g_conf->apply_changes(NULL); | |
5853 | ||
5854 | ObjectStore::Sequencer osr("test"); | |
5855 | int r; | |
5856 | coll_t cid; | |
5857 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
5858 | size_t obj_size = 4 * 1024 * 1024; | |
5859 | uint64_t total_bytes, total_bytes2; | |
5860 | uint64_t total_onodes; | |
5861 | get_mempool_stats(&total_bytes, &total_onodes); | |
5862 | ASSERT_EQ(total_onodes, 0u); | |
5863 | ||
5864 | { | |
5865 | ObjectStore::Transaction t; | |
5866 | t.create_collection(cid, 0); | |
5867 | r = apply_transaction(store, &osr, std::move(t)); | |
5868 | ASSERT_EQ(r, 0); | |
5869 | } | |
5870 | { | |
5871 | ObjectStore::Transaction t; | |
5872 | bufferlist bl, orig, orig2; | |
5873 | ||
5874 | bl.append(std::string(obj_size, 'a')); | |
5875 | t.write(cid, hoid, 0, bl.length(), bl); | |
5876 | r = apply_transaction(store, &osr, std::move(t)); | |
5877 | ASSERT_EQ(r, 0); | |
5878 | } | |
5879 | get_mempool_stats(&total_bytes, &total_onodes); | |
5880 | ASSERT_NE(total_bytes, 0u); | |
5881 | ASSERT_EQ(total_onodes, 1u); | |
5882 | ||
5883 | { | |
5884 | ObjectStore::Transaction t; | |
5885 | t.truncate(cid, hoid, 0); | |
5886 | r = apply_transaction(store, &osr, std::move(t)); | |
5887 | ASSERT_EQ(r, 0); | |
5888 | } | |
5889 | ||
5890 | for(size_t i = 0; i < 1; ++i) { | |
5891 | bufferlist bl; | |
5892 | bl.append(std::string(block_size * (i+1), 'a')); | |
5893 | for( size_t j = 0; j < obj_size; j+= bl.length()) { | |
5894 | ObjectStore::Transaction t; | |
5895 | t.write(cid, hoid, j, bl.length(), bl); | |
5896 | r = apply_transaction(store, &osr, std::move(t)); | |
5897 | ASSERT_EQ(r, 0); | |
5898 | } | |
5899 | get_mempool_stats(&total_bytes2, &total_onodes); | |
5900 | ASSERT_NE(total_bytes2, 0u); | |
5901 | ASSERT_EQ(total_onodes, 1u); | |
5902 | } | |
5903 | { | |
5904 | cout <<" mempool dump:\n"; | |
5905 | JSONFormatter f(true); | |
5906 | f.open_object_section("transaction"); | |
5907 | mempool::dump(&f); | |
5908 | f.close_section(); | |
5909 | f.flush(cout); | |
5910 | cout << std::endl; | |
5911 | } | |
5912 | { | |
5913 | bufferlist bl; | |
5914 | for (size_t i = 0; i < obj_size; i += 0x1000) { | |
5915 | store->read(cid, hoid, i, 0x1000, bl); | |
5916 | } | |
5917 | } | |
5918 | get_mempool_stats(&total_bytes, &total_onodes); | |
5919 | ASSERT_NE(total_bytes, 0u); | |
5920 | ASSERT_EQ(total_onodes, 1u); | |
5921 | ||
5922 | { | |
5923 | cout <<" mempool dump:\n"; | |
5924 | JSONFormatter f(true); | |
5925 | f.open_object_section("transaction"); | |
5926 | mempool::dump(&f); | |
5927 | f.close_section(); | |
5928 | f.flush(cout); | |
5929 | cout << std::endl; | |
5930 | } | |
5931 | { | |
5932 | ObjectStore::Transaction t; | |
5933 | t.remove(cid, hoid); | |
5934 | t.remove_collection(cid); | |
5935 | cerr << "Cleaning" << std::endl; | |
5936 | r = apply_transaction(store, &osr, std::move(t)); | |
5937 | ASSERT_EQ(r, 0); | |
5938 | } | |
5939 | g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000"); | |
5940 | g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000"); | |
5941 | g_conf->set_val("bluestore_compression_mode", "none"); | |
5942 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
5943 | ||
5944 | } | |
5945 | ||
5946 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) { | |
5947 | ||
5948 | if (string(GetParam()) != "bluestore") | |
5949 | return; | |
5950 | ||
5951 | size_t block_size = 4096; | |
5952 | StartDeferred(block_size); | |
5953 | g_conf->set_val("bluestore_max_blob_size", "65536"); | |
5954 | g_conf->apply_changes(NULL); | |
5955 | ||
5956 | ObjectStore::Sequencer osr("test"); | |
5957 | int r; | |
5958 | coll_t cid; | |
5959 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
5960 | ||
5961 | const PerfCounters* logger = store->get_perf_counters(); | |
5962 | ||
5963 | { | |
5964 | ObjectStore::Transaction t; | |
5965 | t.create_collection(cid, 0); | |
5966 | r = apply_transaction(store, &osr, std::move(t)); | |
5967 | ASSERT_EQ(r, 0); | |
5968 | } | |
5969 | { | |
5970 | ObjectStore::Transaction t; | |
5971 | bufferlist bl; | |
5972 | ||
5973 | bl.append(std::string(block_size * 2, 'a')); | |
5974 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5975 | r = apply_transaction(store, &osr, std::move(t)); | |
5976 | ASSERT_EQ(r, 0); | |
5977 | } | |
5978 | { | |
5979 | // overwrite at the beginning | |
5980 | ObjectStore::Transaction t; | |
5981 | bufferlist bl; | |
5982 | ||
5983 | bl.append(std::string(block_size, 'b')); | |
5984 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5985 | r = apply_transaction(store, &osr, std::move(t)); | |
5986 | ASSERT_EQ(r, 0); | |
5987 | } | |
5988 | { | |
5989 | // append | |
5990 | ObjectStore::Transaction t; | |
5991 | bufferlist bl; | |
5992 | ||
5993 | bl.append(std::string(block_size * 2, 'c')); | |
5994 | t.write(cid, hoid, block_size * 2, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5995 | r = apply_transaction(store, &osr, std::move(t)); | |
5996 | ASSERT_EQ(r, 0); | |
5997 | } | |
5998 | { | |
5999 | // append with a gap | |
6000 | ObjectStore::Transaction t; | |
6001 | bufferlist bl; | |
6002 | ||
6003 | bl.append(std::string(block_size * 2, 'd')); | |
6004 | t.write(cid, hoid, block_size * 5, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6005 | r = apply_transaction(store, &osr, std::move(t)); | |
6006 | ASSERT_EQ(r, 0); | |
6007 | } | |
6008 | { | |
6009 | // We need to issue a read to trigger cache stat update that refresh | |
6010 | // perf counters. additionally we need to wait some time for mempool | |
6011 | // thread to update stats. | |
6012 | sleep(1); | |
6013 | bufferlist bl, expected; | |
6014 | r = store->read(cid, hoid, 0, block_size, bl); | |
6015 | ASSERT_EQ(r, (int)block_size); | |
6016 | expected.append(string(block_size, 'b')); | |
6017 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6018 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6019 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6020 | } | |
6021 | { | |
6022 | // overwrite at end | |
6023 | ObjectStore::Transaction t; | |
6024 | bufferlist bl; | |
6025 | ||
6026 | bl.append(std::string(block_size * 2, 'e')); | |
6027 | ||
6028 | // Currently we are unable to reuse blob when overwriting in a single step | |
6029 | t.write(cid, hoid, block_size * 6, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6030 | r = apply_transaction(store, &osr, std::move(t)); | |
6031 | ASSERT_EQ(r, 0); | |
6032 | } | |
6033 | { | |
6034 | // We need to issue a read to trigger cache stat update that refresh | |
6035 | // perf counters. additionally we need to wait some time for mempool | |
6036 | // thread to update stats. | |
6037 | sleep(1); | |
6038 | bufferlist bl, expected; | |
6039 | r = store->read(cid, hoid, 0, block_size, bl); | |
6040 | ASSERT_EQ(r, (int)block_size); | |
6041 | expected.append(string(block_size, 'b')); | |
6042 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6043 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6044 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6045 | } | |
6046 | { | |
6047 | // fill the gap | |
6048 | ObjectStore::Transaction t; | |
6049 | bufferlist bl; | |
6050 | ||
6051 | bl.append(std::string(block_size, 'f')); | |
6052 | ||
6053 | t.write(cid, hoid, block_size * 4, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6054 | r = apply_transaction(store, &osr, std::move(t)); | |
6055 | ASSERT_EQ(r, 0); | |
6056 | } | |
6057 | { | |
6058 | // we need to wait some time for mempool | |
6059 | // thread to update stats to be able to check blob/extent numbers from | |
6060 | // perf counters. | |
6061 | sleep(1); | |
6062 | ||
6063 | bufferlist bl, expected; | |
6064 | r = store->read(cid, hoid, 0, block_size, bl); | |
6065 | ASSERT_EQ(r, (int)block_size); | |
6066 | expected.append(string(block_size, 'b')); | |
6067 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6068 | ||
6069 | bl.clear(); | |
6070 | expected.clear(); | |
6071 | r = store->read(cid, hoid, block_size, block_size, bl); | |
6072 | ASSERT_EQ(r, (int)block_size); | |
6073 | expected.append(string(block_size, 'a')); | |
6074 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6075 | ||
6076 | bl.clear(); | |
6077 | expected.clear(); | |
6078 | r = store->read(cid, hoid, block_size * 2, block_size * 2, bl); | |
6079 | ASSERT_EQ(r, (int)block_size * 2); | |
6080 | expected.append(string(block_size * 2, 'c')); | |
6081 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6082 | ||
6083 | bl.clear(); | |
6084 | expected.clear(); | |
6085 | r = store->read(cid, hoid, block_size * 4, block_size, bl); | |
6086 | ASSERT_EQ(r, (int)block_size); | |
6087 | expected.append(string(block_size, 'f')); | |
6088 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6089 | ||
6090 | bl.clear(); | |
6091 | expected.clear(); | |
6092 | r = store->read(cid, hoid, block_size * 5, block_size, bl); | |
6093 | ASSERT_EQ(r, (int)block_size); | |
6094 | expected.append(string(block_size, 'd')); | |
6095 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6096 | ||
6097 | bl.clear(); | |
6098 | expected.clear(); | |
6099 | r = store->read(cid, hoid, block_size * 5, block_size * 3, bl); | |
6100 | ASSERT_EQ(r, (int)block_size * 3); | |
6101 | expected.append(string(block_size, 'd')); | |
6102 | expected.append(string(block_size * 2, 'e')); | |
6103 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6104 | } | |
6105 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6106 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6107 | ||
6108 | ||
6109 | { | |
6110 | ObjectStore::Transaction t; | |
6111 | t.remove(cid, hoid); | |
6112 | t.remove_collection(cid); | |
6113 | cerr << "Cleaning" << std::endl; | |
6114 | r = apply_transaction(store, &osr, std::move(t)); | |
6115 | ASSERT_EQ(r, 0); | |
6116 | } | |
6117 | g_conf->set_val("bluestore_max_blob_size", "0"); | |
6118 | ||
6119 | } | |
6120 | ||
6121 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) { | |
6122 | ||
6123 | if (string(GetParam()) != "bluestore") | |
6124 | return; | |
6125 | ||
6126 | size_t block_size = 4096; | |
6127 | StartDeferred(block_size); | |
6128 | g_conf->set_val("bluestore_max_blob_size", "65536"); | |
6129 | ||
6130 | g_conf->apply_changes(NULL); | |
6131 | ||
6132 | ObjectStore::Sequencer osr("test"); | |
6133 | int r; | |
6134 | coll_t cid; | |
6135 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6136 | ||
6137 | const PerfCounters* logger = store->get_perf_counters(); | |
6138 | { | |
6139 | ObjectStore::Transaction t; | |
6140 | t.create_collection(cid, 0); | |
6141 | r = apply_transaction(store, &osr, std::move(t)); | |
6142 | ASSERT_EQ(r, 0); | |
6143 | } | |
6144 | { | |
6145 | ObjectStore::Transaction t; | |
6146 | bufferlist bl; | |
6147 | ||
6148 | bl.append(std::string(block_size * 2, 'a')); | |
6149 | t.write(cid, hoid, block_size * 10, bl.length(), bl, | |
6150 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6151 | r = apply_transaction(store, &osr, std::move(t)); | |
6152 | ASSERT_EQ(r, 0); | |
6153 | } | |
6154 | { | |
6155 | // prepend existing | |
6156 | ObjectStore::Transaction t; | |
6157 | bufferlist bl; | |
6158 | ||
6159 | bl.append(std::string(block_size, 'b')); | |
6160 | t.write(cid, hoid, block_size * 9, bl.length(), bl, | |
6161 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6162 | r = apply_transaction(store, &osr, std::move(t)); | |
6163 | ASSERT_EQ(r, 0); | |
6164 | } | |
6165 | { | |
6166 | // We need to issue a read to trigger cache stat update that refresh | |
6167 | // perf counters. additionally we need to wait some time for mempool | |
6168 | // thread to update stats. | |
6169 | sleep(1); | |
6170 | bufferlist bl, expected; | |
6171 | r = store->read(cid, hoid, block_size * 9, block_size * 2, bl); | |
6172 | ASSERT_EQ(r, (int)block_size * 2); | |
6173 | expected.append(string(block_size, 'b')); | |
6174 | expected.append(string(block_size, 'a')); | |
6175 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6176 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6177 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6178 | } | |
6179 | ||
6180 | ||
6181 | { | |
6182 | // prepend existing with a gap | |
6183 | ObjectStore::Transaction t; | |
6184 | bufferlist bl; | |
6185 | ||
6186 | bl.append(std::string(block_size, 'c')); | |
6187 | t.write(cid, hoid, block_size * 7, bl.length(), bl, | |
6188 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6189 | r = apply_transaction(store, &osr, std::move(t)); | |
6190 | ASSERT_EQ(r, 0); | |
6191 | } | |
6192 | { | |
6193 | // We need to issue a read to trigger cache stat update that refresh | |
6194 | // perf counters. additionally we need to wait some time for mempool | |
6195 | // thread to update stats. | |
6196 | sleep(1); | |
6197 | bufferlist bl, expected; | |
6198 | r = store->read(cid, hoid, block_size * 7, block_size * 3, bl); | |
6199 | ASSERT_EQ(r, (int)block_size * 3); | |
6200 | expected.append(string(block_size, 'c')); | |
6201 | expected.append(string(block_size, 0)); | |
6202 | expected.append(string(block_size, 'b')); | |
6203 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6204 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6205 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6206 | } | |
6207 | ||
6208 | { | |
6209 | // append after existing with a gap | |
6210 | ObjectStore::Transaction t; | |
6211 | bufferlist bl; | |
6212 | ||
6213 | bl.append(std::string(block_size, 'd')); | |
6214 | t.write(cid, hoid, block_size * 13, bl.length(), bl, | |
6215 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6216 | r = apply_transaction(store, &osr, std::move(t)); | |
6217 | ASSERT_EQ(r, 0); | |
6218 | } | |
6219 | { | |
6220 | // We need to issue a read to trigger cache stat update that refresh | |
6221 | // perf counters. additionally we need to wait some time for mempool | |
6222 | // thread to update stats. | |
6223 | sleep(1); | |
6224 | bufferlist bl, expected; | |
6225 | r = store->read(cid, hoid, block_size * 11, block_size * 3, bl); | |
6226 | ASSERT_EQ(r, (int)block_size * 3); | |
6227 | expected.append(string(block_size, 'a')); | |
6228 | expected.append(string(block_size, 0)); | |
6229 | expected.append(string(block_size, 'd')); | |
6230 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6231 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6232 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
6233 | } | |
6234 | ||
6235 | { | |
6236 | // append twice to the next max_blob slot | |
6237 | ObjectStore::Transaction t; | |
6238 | bufferlist bl; | |
6239 | ||
6240 | bl.append(std::string(block_size, 'e')); | |
6241 | t.write(cid, hoid, block_size * 17, bl.length(), bl, | |
6242 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6243 | t.write(cid, hoid, block_size * 19, bl.length(), bl, | |
6244 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6245 | r = apply_transaction(store, &osr, std::move(t)); | |
6246 | ASSERT_EQ(r, 0); | |
6247 | } | |
6248 | { | |
6249 | // We need to issue a read to trigger cache stat update that refresh | |
6250 | // perf counters. additionally we need to wait some time for mempool | |
6251 | // thread to update stats. | |
6252 | sleep(1); | |
6253 | bufferlist bl, expected; | |
6254 | r = store->read(cid, hoid, block_size * 17, block_size * 3, bl); | |
6255 | ASSERT_EQ(r, (int)block_size * 3); | |
6256 | expected.append(string(block_size, 'e')); | |
6257 | expected.append(string(block_size, 0)); | |
6258 | expected.append(string(block_size, 'e')); | |
6259 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6260 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
6261 | ASSERT_EQ(logger->get(l_bluestore_extents), 5u); | |
6262 | } | |
6263 | { | |
6264 | // fill gaps at the second slot | |
6265 | ObjectStore::Transaction t; | |
6266 | bufferlist bl; | |
6267 | ||
6268 | bl.append(std::string(block_size, 'f')); | |
6269 | t.write(cid, hoid, block_size * 16, bl.length(), bl, | |
6270 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6271 | t.write(cid, hoid, block_size * 18, bl.length(), bl, | |
6272 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6273 | r = apply_transaction(store, &osr, std::move(t)); | |
6274 | ASSERT_EQ(r, 0); | |
6275 | } | |
6276 | { | |
6277 | // We need to issue a read to trigger cache stat update that refresh | |
6278 | // perf counters. additionally we need to wait some time for mempool | |
6279 | // thread to update stats. | |
6280 | sleep(1); | |
6281 | bufferlist bl, expected; | |
6282 | r = store->read(cid, hoid, block_size * 16, block_size * 4, bl); | |
6283 | ASSERT_EQ(r, (int)block_size * 4); | |
6284 | expected.append(string(block_size, 'f')); | |
6285 | expected.append(string(block_size, 'e')); | |
6286 | expected.append(string(block_size, 'f')); | |
6287 | expected.append(string(block_size, 'e')); | |
6288 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6289 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
6290 | ASSERT_EQ(logger->get(l_bluestore_extents), 4u); | |
6291 | } | |
6292 | { | |
6293 | ObjectStore::Transaction t; | |
6294 | t.remove(cid, hoid); | |
6295 | t.remove_collection(cid); | |
6296 | cerr << "Cleaning" << std::endl; | |
6297 | r = apply_transaction(store, &osr, std::move(t)); | |
6298 | ASSERT_EQ(r, 0); | |
6299 | } | |
6300 | g_conf->set_val("bluestore_max_blob_size", "0"); | |
6301 | } | |
6302 | ||
6303 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) { | |
6304 | ||
6305 | if (string(GetParam()) != "bluestore") | |
6306 | return; | |
6307 | ||
6308 | size_t block_size = 4096; | |
6309 | StartDeferred(block_size); | |
6310 | g_conf->set_val("bluestore_max_blob_size", "65536"); | |
6311 | g_conf->apply_changes(NULL); | |
6312 | ||
6313 | ObjectStore::Sequencer osr("test"); | |
6314 | int r; | |
6315 | coll_t cid; | |
6316 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6317 | ||
6318 | const PerfCounters* logger = store->get_perf_counters(); | |
6319 | ||
6320 | { | |
6321 | ObjectStore::Transaction t; | |
6322 | t.create_collection(cid, 0); | |
6323 | r = apply_transaction(store, &osr, std::move(t)); | |
6324 | ASSERT_EQ(r, 0); | |
6325 | } | |
6326 | { | |
6327 | ObjectStore::Transaction t; | |
6328 | bufferlist bl; | |
6329 | ||
6330 | bl.append(std::string(block_size, 'a')); | |
6331 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6332 | t.write(cid, hoid, block_size * 2, bl.length(), bl, | |
6333 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6334 | r = apply_transaction(store, &osr, std::move(t)); | |
6335 | ASSERT_EQ(r, 0); | |
6336 | } | |
6337 | { | |
6338 | // write small into the gap | |
6339 | ObjectStore::Transaction t; | |
6340 | bufferlist bl; | |
6341 | ||
6342 | bl.append(std::string(3, 'b')); | |
6343 | t.write(cid, hoid, block_size + 1, bl.length(), bl, | |
6344 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6345 | r = apply_transaction(store, &osr, std::move(t)); | |
6346 | ASSERT_EQ(r, 0); | |
6347 | } | |
6348 | { | |
6349 | // We need to issue a read to trigger cache stat update that refresh | |
6350 | // perf counters. additionally we need to wait some time for mempool | |
6351 | // thread to update stats. | |
6352 | sleep(1); | |
6353 | bufferlist bl, expected; | |
6354 | r = store->read(cid, hoid, 0, block_size * 3, bl); | |
6355 | ASSERT_EQ(r, (int)block_size * 3); | |
6356 | expected.append(string(block_size, 'a')); | |
6357 | expected.append(string(1, 0)); | |
6358 | expected.append(string(3, 'b')); | |
6359 | expected.append(string(block_size - 4, 0)); | |
6360 | expected.append(string(block_size, 'a')); | |
6361 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6362 | ||
6363 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6364 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
6365 | } | |
6366 | { | |
6367 | ObjectStore::Transaction t; | |
6368 | t.remove(cid, hoid); | |
6369 | t.remove_collection(cid); | |
6370 | cerr << "Cleaning" << std::endl; | |
6371 | r = apply_transaction(store, &osr, std::move(t)); | |
6372 | ASSERT_EQ(r, 0); | |
6373 | } | |
6374 | g_conf->set_val("bluestore_max_blob_size", "0"); | |
6375 | } | |
6376 | ||
6377 | // The test case to reproduce an issue when write happens | |
6378 | // to a zero space between the extents sharing the same spanning blob | |
6379 | // with unloaded shard map. | |
6380 | // Second extent might be filled with zeros this way due to wrong result | |
6381 | // returned by has_any_extents() call in do_write_small. The latter is caused | |
6382 | // by incompletly loaded extent map. | |
6383 | TEST_P(StoreTestSpecificAUSize, SmallWriteOnShardedExtents) { | |
6384 | if (string(GetParam()) != "bluestore") | |
6385 | return; | |
6386 | ||
6387 | size_t block_size = 0x10000; | |
6388 | StartDeferred(block_size); | |
6389 | ||
6390 | g_conf->set_val("bluestore_csum_type", "xxhash64"); | |
6391 | g_conf->set_val("bluestore_max_target_blob", "524288"); // for sure | |
6392 | ||
6393 | g_conf->apply_changes(NULL); | |
6394 | ||
6395 | ObjectStore::Sequencer osr("test"); | |
6396 | int r; | |
6397 | coll_t cid; | |
6398 | ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6399 | ||
6400 | { | |
6401 | ObjectStore::Transaction t; | |
6402 | t.create_collection(cid, 0); | |
6403 | r = apply_transaction(store, &osr, std::move(t)); | |
6404 | ASSERT_EQ(r, 0); | |
6405 | } | |
6406 | { | |
6407 | //doing some tricks to have sharded extents/spanning objects | |
6408 | ObjectStore::Transaction t; | |
6409 | bufferlist bl, bl2; | |
6410 | ||
6411 | bl.append(std::string(0x80000, 'a')); | |
6412 | t.write(cid, hoid1, 0, bl.length(), bl, 0); | |
6413 | t.zero(cid, hoid1, 0x719e0, 0x75b0 ); | |
6414 | r = apply_transaction(store, &osr, std::move(t)); | |
6415 | ASSERT_EQ(r, 0); | |
6416 | ||
6417 | bl2.append(std::string(0x70000, 'b')); | |
6418 | t.write(cid, hoid1, 0, bl2.length(), bl2, 0); | |
6419 | t.zero(cid, hoid1, 0, 0x50000); | |
6420 | r = apply_transaction(store, &osr, std::move(t)); | |
6421 | ASSERT_EQ(r, 0); | |
6422 | ||
6423 | } | |
6424 | store->umount(); | |
6425 | store->mount(); | |
6426 | ||
6427 | { | |
6428 | // do a write to zero space in between some extents sharing the same blob | |
6429 | ObjectStore::Transaction t; | |
6430 | bufferlist bl, bl2; | |
6431 | ||
6432 | bl.append(std::string(0x6520, 'c')); | |
6433 | t.write(cid, hoid1, 0x71c00, bl.length(), bl, 0); | |
6434 | ||
6435 | r = apply_transaction(store, &osr, std::move(t)); | |
6436 | ASSERT_EQ(r, 0); | |
6437 | } | |
6438 | ||
6439 | { | |
6440 | ObjectStore::Transaction t; | |
6441 | bufferlist bl, expected; | |
6442 | ||
6443 | r = store->read(cid, hoid1, 0x70000, 0x9c00, bl); | |
6444 | ASSERT_EQ(r, (int)0x9c00); | |
6445 | expected.append(string(0x19e0, 'a')); | |
6446 | expected.append(string(0x220, 0)); | |
6447 | expected.append(string(0x6520, 'c')); | |
6448 | expected.append(string(0xe70, 0)); | |
6449 | expected.append(string(0xc70, 'a')); | |
6450 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6451 | bl.clear(); | |
6452 | ||
6453 | } | |
6454 | ||
6455 | { | |
6456 | ObjectStore::Transaction t; | |
6457 | t.remove(cid, hoid1); | |
6458 | t.remove_collection(cid); | |
6459 | cerr << "Cleaning" << std::endl; | |
6460 | r = apply_transaction(store, &osr, std::move(t)); | |
6461 | ASSERT_EQ(r, 0); | |
6462 | } | |
6463 | g_conf->set_val("bluestore_max_target_blob", "524288"); | |
6464 | g_conf->set_val("bluestore_csum_type", "crc32c"); | |
6465 | } | |
6466 | ||
6467 | TEST_P(StoreTestSpecificAUSize, ExcessiveFragmentation) { | |
6468 | if (string(GetParam()) != "bluestore") | |
6469 | return; | |
6470 | ||
6471 | g_conf->set_val("bluestore_block_size", | |
6472 | stringify((uint64_t)2048 * 1024 * 1024)); | |
6473 | ||
6474 | ASSERT_EQ(g_conf->get_val<uint64_t>("bluefs_alloc_size"), | |
6475 | 1024 * 1024); | |
6476 | ||
6477 | size_t block_size = 0x10000; | |
6478 | StartDeferred(block_size); | |
6479 | ||
6480 | ObjectStore::Sequencer osr("test"); | |
6481 | int r; | |
6482 | coll_t cid; | |
6483 | ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6484 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
6485 | ||
6486 | { | |
6487 | ObjectStore::Transaction t; | |
6488 | t.create_collection(cid, 0); | |
6489 | r = apply_transaction(store, &osr, std::move(t)); | |
6490 | ASSERT_EQ(r, 0); | |
6491 | } | |
6492 | { | |
6493 | // create 2x400MB objects in a way that their pextents are interleaved | |
6494 | ObjectStore::Transaction t; | |
6495 | bufferlist bl; | |
6496 | ||
6497 | bl.append(std::string(block_size * 4, 'a')); // 256KB | |
6498 | uint64_t offs = 0; | |
6499 | while(offs < (uint64_t)400 * 1024 * 1024) { | |
6500 | t.write(cid, hoid1, offs, bl.length(), bl, 0); | |
6501 | t.write(cid, hoid2, offs, bl.length(), bl, 0); | |
6502 | r = apply_transaction(store, &osr, std::move(t)); | |
6503 | ASSERT_EQ(r, 0); | |
6504 | offs += bl.length(); | |
6505 | if( (offs % (100 * 1024 * 1024)) == 0) { | |
6506 | std::cout<<"written " << offs << std::endl; | |
6507 | } | |
6508 | } | |
6509 | } | |
6510 | std::cout<<"written 800MB"<<std::endl; | |
6511 | { | |
6512 | // Partially overwrite objects with 100MB each leaving space | |
6513 | // fragmented and occuping still unfragmented space at the end | |
6514 | // So we'll have enough free space but it'll lack long enough (e.g. 1MB) | |
6515 | // contiguous pextents. | |
6516 | ObjectStore::Transaction t; | |
6517 | bufferlist bl; | |
6518 | ||
6519 | bl.append(std::string(block_size * 4, 'a')); | |
6520 | uint64_t offs = 0; | |
6521 | while(offs < 112 * 1024 * 1024) { | |
6522 | t.write(cid, hoid1, offs, bl.length(), bl, 0); | |
6523 | t.write(cid, hoid2, offs, bl.length(), bl, 0); | |
6524 | r = apply_transaction(store, &osr, std::move(t)); | |
6525 | ASSERT_EQ(r, 0); | |
6526 | // this will produce high fragmentation if original allocations | |
6527 | // were contiguous | |
6528 | offs += bl.length(); | |
6529 | if( (offs % (10 * 1024 * 1024)) == 0) { | |
6530 | std::cout<<"written " << offs << std::endl; | |
6531 | } | |
6532 | } | |
6533 | } | |
6534 | { | |
6535 | // remove one of the object producing much free space | |
6536 | // and hence triggering bluefs rebalance. | |
6537 | // Which should fail as there is no long enough pextents. | |
6538 | ObjectStore::Transaction t; | |
6539 | t.remove(cid, hoid2); | |
6540 | r = apply_transaction(store, &osr, std::move(t)); | |
6541 | ASSERT_EQ(r, 0); | |
6542 | } | |
6543 | ||
6544 | auto to_sleep = 5 * | |
6545 | (int)g_conf->get_val<double>("bluestore_bluefs_balance_interval"); | |
6546 | std::cout<<"sleeping... " << std::endl; | |
6547 | sleep(to_sleep); | |
6548 | ||
6549 | { | |
6550 | // touch another object to triggerrebalance | |
6551 | ObjectStore::Transaction t; | |
6552 | t.touch(cid, hoid1); | |
6553 | r = apply_transaction(store, &osr, std::move(t)); | |
6554 | ASSERT_EQ(r, 0); | |
6555 | } | |
6556 | { | |
6557 | ObjectStore::Transaction t; | |
6558 | t.remove(cid, hoid1); | |
6559 | t.remove(cid, hoid2); | |
6560 | t.remove_collection(cid); | |
6561 | cerr << "Cleaning" << std::endl; | |
6562 | r = apply_transaction(store, &osr, std::move(t)); | |
6563 | ASSERT_EQ(r, 0); | |
6564 | } | |
6565 | g_conf->set_val("bluestore_block_size", | |
6566 | stringify(DEF_STORE_TEST_BLOCKDEV_SIZE)); | |
6567 | } | |
6568 | ||
6569 | #endif //#if defined(HAVE_LIBAIO) | |
6570 | ||
6571 | TEST_P(StoreTest, KVDBHistogramTest) { | |
6572 | if (string(GetParam()) != "bluestore") | |
6573 | return; | |
6574 | ||
6575 | ObjectStore::Sequencer osr("test"); | |
6576 | int NUM_OBJS = 200; | |
6577 | int r = 0; | |
6578 | coll_t cid; | |
6579 | string base("testobj."); | |
6580 | bufferlist a; | |
6581 | bufferptr ap(0x1000); | |
6582 | memset(ap.c_str(), 'a', 0x1000); | |
6583 | a.append(ap); | |
6584 | { | |
6585 | ObjectStore::Transaction t; | |
6586 | t.create_collection(cid, 0); | |
6587 | r = apply_transaction(store, &osr, std::move(t)); | |
6588 | ASSERT_EQ(r, 0); | |
6589 | } | |
6590 | for (int i = 0; i < NUM_OBJS; ++i) { | |
6591 | ObjectStore::Transaction t; | |
6592 | char buf[100]; | |
6593 | snprintf(buf, sizeof(buf), "%d", i); | |
6594 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
6595 | t.write(cid, hoid, 0, 0x1000, a); | |
6596 | r = apply_transaction(store, &osr, std::move(t)); | |
6597 | ASSERT_EQ(r, 0); | |
6598 | } | |
6599 | ||
6600 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
6601 | store->generate_db_histogram(f); | |
6602 | f->flush(cout); | |
6603 | cout << std::endl; | |
6604 | } | |
6605 | ||
6606 | TEST_P(StoreTest, KVDBStatsTest) { | |
6607 | if (string(GetParam()) != "bluestore") | |
6608 | return; | |
6609 | ||
6610 | g_conf->set_val("rocksdb_perf", "true"); | |
6611 | g_conf->set_val("rocksdb_collect_compaction_stats", "true"); | |
6612 | g_conf->set_val("rocksdb_collect_extended_stats","true"); | |
6613 | g_conf->set_val("rocksdb_collect_memory_stats","true"); | |
6614 | g_ceph_context->_conf->apply_changes(NULL); | |
6615 | int r = store->umount(); | |
6616 | ASSERT_EQ(r, 0); | |
6617 | r = store->mount(); //to force rocksdb stats | |
6618 | ASSERT_EQ(r, 0); | |
6619 | ||
6620 | ObjectStore::Sequencer osr("test"); | |
6621 | int NUM_OBJS = 200; | |
6622 | coll_t cid; | |
6623 | string base("testobj."); | |
6624 | bufferlist a; | |
6625 | bufferptr ap(0x1000); | |
6626 | memset(ap.c_str(), 'a', 0x1000); | |
6627 | a.append(ap); | |
6628 | { | |
6629 | ObjectStore::Transaction t; | |
6630 | t.create_collection(cid, 0); | |
6631 | r = apply_transaction(store, &osr, std::move(t)); | |
6632 | ASSERT_EQ(r, 0); | |
6633 | } | |
6634 | for (int i = 0; i < NUM_OBJS; ++i) { | |
6635 | ObjectStore::Transaction t; | |
6636 | char buf[100]; | |
6637 | snprintf(buf, sizeof(buf), "%d", i); | |
6638 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
6639 | t.write(cid, hoid, 0, 0x1000, a); | |
6640 | r = apply_transaction(store, &osr, std::move(t)); | |
6641 | ASSERT_EQ(r, 0); | |
6642 | } | |
6643 | ||
6644 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
6645 | store->get_db_statistics(f); | |
6646 | f->flush(cout); | |
6647 | cout << std::endl; | |
6648 | g_conf->set_val("rocksdb_perf", "false"); | |
6649 | g_conf->set_val("rocksdb_collect_compaction_stats", "false"); | |
6650 | g_conf->set_val("rocksdb_collect_extended_stats","false"); | |
6651 | g_conf->set_val("rocksdb_collect_memory_stats","false"); | |
6652 | } | |
6653 | ||
6654 | #if defined(HAVE_LIBAIO) | |
6655 | TEST_P(StoreTestSpecificAUSize, garbageCollection) { | |
6656 | ObjectStore::Sequencer osr("test"); | |
6657 | int r; | |
6658 | coll_t cid; | |
6659 | int buf_len = 256 * 1024; | |
6660 | int overlap_offset = 64 * 1024; | |
6661 | int write_offset = buf_len; | |
6662 | if (string(GetParam()) != "bluestore") | |
6663 | return; | |
6664 | ||
6665 | #define WRITE_AT(offset, _length) {\ | |
6666 | ObjectStore::Transaction t;\ | |
6667 | if ((uint64_t)_length != bl.length()) { \ | |
6668 | buffer::ptr p(bl.c_str(), _length);\ | |
6669 | bufferlist bl_tmp;\ | |
6670 | bl_tmp.push_back(p);\ | |
6671 | t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\ | |
6672 | } else {\ | |
6673 | t.write(cid, hoid, offset, bl.length(), bl);\ | |
6674 | }\ | |
6675 | r = apply_transaction(store, &osr, std::move(t));\ | |
6676 | ASSERT_EQ(r, 0);\ | |
6677 | } | |
6678 | ||
6679 | StartDeferred(65536); | |
6680 | ||
6681 | g_conf->set_val("bluestore_compression_max_blob_size", "524288"); | |
6682 | g_conf->set_val("bluestore_compression_min_blob_size", "262144"); | |
6683 | g_conf->set_val("bluestore_compression_mode", "force"); | |
6684 | g_conf->apply_changes(NULL); | |
6685 | ||
6686 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6687 | { | |
6688 | bufferlist in; | |
6689 | r = store->read(cid, hoid, 0, 5, in); | |
6690 | ASSERT_EQ(-ENOENT, r); | |
6691 | } | |
6692 | { | |
6693 | ObjectStore::Transaction t; | |
6694 | t.create_collection(cid, 0); | |
6695 | cerr << "Creating collection " << cid << std::endl; | |
6696 | r = apply_transaction(store, &osr, std::move(t)); | |
6697 | ASSERT_EQ(r, 0); | |
6698 | } | |
6699 | ||
6700 | std::string data; | |
6701 | data.resize(buf_len); | |
6702 | ||
6703 | { | |
6704 | { | |
6705 | bool exists = store->exists(cid, hoid); | |
6706 | ASSERT_TRUE(!exists); | |
6707 | ||
6708 | ObjectStore::Transaction t; | |
6709 | t.touch(cid, hoid); | |
6710 | cerr << "Creating object " << hoid << std::endl; | |
6711 | r = apply_transaction(store, &osr, std::move(t)); | |
6712 | ASSERT_EQ(r, 0); | |
6713 | ||
6714 | exists = store->exists(cid, hoid); | |
6715 | ASSERT_EQ(true, exists); | |
6716 | } | |
6717 | bufferlist bl; | |
6718 | ||
6719 | for(size_t i = 0; i < data.size(); i++) | |
6720 | data[i] = i % 256; | |
6721 | ||
6722 | bl.append(data); | |
6723 | ||
6724 | { | |
6725 | struct store_statfs_t statfs; | |
6726 | WRITE_AT(0, buf_len); | |
6727 | int r = store->statfs(&statfs); | |
6728 | ASSERT_EQ(r, 0); | |
6729 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
6730 | } | |
6731 | { | |
6732 | struct store_statfs_t statfs; | |
6733 | WRITE_AT(write_offset - 2 * overlap_offset, buf_len); | |
6734 | int r = store->statfs(&statfs); | |
6735 | ASSERT_EQ(r, 0); | |
6736 | ASSERT_EQ(statfs.compressed_allocated, 0x20000); | |
6737 | const PerfCounters* counters = store->get_perf_counters(); | |
6738 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0u); | |
6739 | } | |
6740 | ||
6741 | { | |
6742 | struct store_statfs_t statfs; | |
6743 | WRITE_AT(write_offset - overlap_offset, buf_len); | |
6744 | int r = store->statfs(&statfs); | |
6745 | ASSERT_EQ(r, 0); | |
6746 | ASSERT_EQ(statfs.compressed_allocated, 0x20000); | |
6747 | const PerfCounters* counters = store->get_perf_counters(); | |
6748 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x10000u); | |
6749 | } | |
6750 | { | |
6751 | struct store_statfs_t statfs; | |
6752 | WRITE_AT(write_offset - 3 * overlap_offset, buf_len); | |
6753 | int r = store->statfs(&statfs); | |
6754 | ASSERT_EQ(r, 0); | |
6755 | ASSERT_EQ(statfs.compressed_allocated, 0x20000); | |
6756 | const PerfCounters* counters = store->get_perf_counters(); | |
6757 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
6758 | } | |
6759 | { | |
6760 | struct store_statfs_t statfs; | |
6761 | WRITE_AT(write_offset + 1, overlap_offset-1); | |
6762 | int r = store->statfs(&statfs); | |
6763 | ASSERT_EQ(r, 0); | |
6764 | ASSERT_EQ(statfs.compressed_allocated, 0x20000); | |
6765 | const PerfCounters* counters = store->get_perf_counters(); | |
6766 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
6767 | } | |
6768 | { | |
6769 | struct store_statfs_t statfs; | |
6770 | WRITE_AT(write_offset + 1, overlap_offset); | |
6771 | int r = store->statfs(&statfs); | |
6772 | ASSERT_EQ(r, 0); | |
6773 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
6774 | const PerfCounters* counters = store->get_perf_counters(); | |
6775 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x3ffffu); | |
6776 | } | |
6777 | { | |
6778 | struct store_statfs_t statfs; | |
6779 | WRITE_AT(0, buf_len-1); | |
6780 | int r = store->statfs(&statfs); | |
6781 | ASSERT_EQ(r, 0); | |
6782 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
6783 | const PerfCounters* counters = store->get_perf_counters(); | |
6784 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
6785 | } | |
6786 | g_conf->set_val("bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0 | |
6787 | { | |
6788 | struct store_statfs_t statfs; | |
6789 | WRITE_AT(1, overlap_offset-2); | |
6790 | WRITE_AT(overlap_offset * 2 + 1, overlap_offset-2); | |
6791 | int r = store->statfs(&statfs); | |
6792 | ASSERT_EQ(r, 0); | |
6793 | ASSERT_EQ(statfs.compressed_allocated, 0x10000); | |
6794 | const PerfCounters* counters = store->get_perf_counters(); | |
6795 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
6796 | } | |
6797 | { | |
6798 | struct store_statfs_t statfs; | |
6799 | WRITE_AT(overlap_offset + 1, overlap_offset-2); | |
6800 | int r = store->statfs(&statfs); | |
6801 | ASSERT_EQ(r, 0); | |
6802 | ASSERT_EQ(statfs.compressed_allocated, 0x0); | |
6803 | const PerfCounters* counters = store->get_perf_counters(); | |
6804 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40007u); | |
6805 | } | |
6806 | { | |
6807 | ObjectStore::Transaction t; | |
6808 | t.remove(cid, hoid); | |
6809 | cerr << "Cleaning" << std::endl; | |
6810 | r = apply_transaction(store, &osr, std::move(t)); | |
6811 | ASSERT_EQ(r, 0); | |
6812 | } | |
6813 | } | |
6814 | g_conf->set_val("bluestore_gc_enable_total_threshold", "0"); | |
6815 | g_conf->set_val("bluestore_compression_min_blob_size", "0"); | |
6816 | g_conf->set_val("bluestore_compression_max_blob_size", "0"); | |
6817 | g_conf->set_val("bluestore_compression_mode", "none"); | |
6818 | g_conf->apply_changes(NULL); | |
6819 | } | |
6820 | #endif | |
6821 | ||
6822 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice) { | |
6823 | if (string(GetParam()) != "bluestore") | |
6824 | return; | |
6825 | ||
6826 | g_conf->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 4K | |
6827 | g_conf->set_val("bluestore_fsck_on_mount", "false"); | |
6828 | g_conf->set_val("bluestore_fsck_on_umount", "false"); | |
6829 | StartDeferred(0x4000); | |
6830 | store->umount(); | |
6831 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
6832 | store->mount(); | |
6833 | ||
6834 | g_conf->set_val("bluestore_fsck_on_mount", "true"); | |
6835 | g_conf->set_val("bluestore_fsck_on_umount", "true"); | |
6836 | g_conf->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb | |
6837 | g_conf->apply_changes(NULL); | |
6838 | } | |
6839 | ||
6840 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice2) { | |
6841 | if (string(GetParam()) != "bluestore") | |
6842 | return; | |
6843 | ||
6844 | g_conf->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 20K | |
6845 | g_conf->set_val("bluestore_fsck_on_mount", "false"); | |
6846 | g_conf->set_val("bluestore_fsck_on_umount", "false"); | |
6847 | StartDeferred(0x1000); | |
6848 | store->umount(); | |
6849 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
6850 | store->mount(); | |
6851 | ||
6852 | g_conf->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb | |
6853 | g_conf->set_val("bluestore_fsck_on_mount", "true"); | |
6854 | g_conf->set_val("bluestore_fsck_on_umount", "true"); | |
6855 | g_conf->apply_changes(NULL); | |
6856 | } | |
6857 | ||
6858 | int main(int argc, char **argv) { | |
6859 | vector<const char*> args; | |
6860 | argv_to_vec(argc, (const char **)argv, args); | |
6861 | env_to_vec(args); | |
6862 | ||
6863 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
6864 | CODE_ENVIRONMENT_UTILITY, 0); | |
6865 | common_init_finish(g_ceph_context); | |
6866 | ||
6867 | g_ceph_context->_conf->set_val("osd_journal_size", "400"); | |
6868 | g_ceph_context->_conf->set_val("filestore_index_retry_probability", "0.5"); | |
6869 | g_ceph_context->_conf->set_val("filestore_op_thread_timeout", "1000"); | |
6870 | g_ceph_context->_conf->set_val("filestore_op_thread_suicide_timeout", "10000"); | |
6871 | //g_ceph_context->_conf->set_val("filestore_fiemap", "true"); | |
6872 | g_ceph_context->_conf->set_val("bluestore_fsck_on_mount", "true"); | |
6873 | g_ceph_context->_conf->set_val("bluestore_fsck_on_umount", "true"); | |
6874 | g_ceph_context->_conf->set_val("bluestore_debug_misc", "true"); | |
6875 | g_ceph_context->_conf->set_val("bluestore_debug_small_allocations", "4"); | |
6876 | g_ceph_context->_conf->set_val("bluestore_debug_freelist", "true"); | |
6877 | g_ceph_context->_conf->set_val("bluestore_clone_cow", "true"); | |
6878 | g_ceph_context->_conf->set_val("bluestore_max_alloc_size", "196608"); | |
6879 | ||
6880 | // set small cache sizes so we see trimming during Synthetic tests | |
6881 | g_ceph_context->_conf->set_val("bluestore_cache_size_hdd", "4000000"); | |
6882 | g_ceph_context->_conf->set_val("bluestore_cache_size_ssd", "4000000"); | |
6883 | ||
6884 | // very short *_max prealloc so that we fall back to async submits | |
6885 | g_ceph_context->_conf->set_val("bluestore_blobid_prealloc", "10"); | |
6886 | g_ceph_context->_conf->set_val("bluestore_nid_prealloc", "10"); | |
6887 | g_ceph_context->_conf->set_val("bluestore_debug_randomize_serial_transaction", | |
6888 | "10"); | |
6889 | ||
6890 | g_ceph_context->_conf->set_val("bdev_debug_aio", "true"); | |
6891 | ||
6892 | // specify device size | |
6893 | g_ceph_context->_conf->set_val("bluestore_block_size", | |
6894 | stringify(DEF_STORE_TEST_BLOCKDEV_SIZE)); | |
6895 | ||
6896 | g_ceph_context->_conf->set_val( | |
6897 | "enable_experimental_unrecoverable_data_corrupting_features", "*"); | |
6898 | g_ceph_context->_conf->apply_changes(NULL); | |
6899 | ||
6900 | ::testing::InitGoogleTest(&argc, argv); | |
6901 | return RUN_ALL_TESTS(); | |
6902 | } | |
6903 | ||
6904 | /* | |
6905 | * Local Variables: | |
6906 | * compile-command: "cd ../.. ; make ceph_test_objectstore && | |
6907 | * ./ceph_test_objectstore \ | |
6908 | * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20 | |
6909 | * " | |
6910 | * End: | |
6911 | */ |