]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <glob.h> | |
16 | #include <stdio.h> | |
17 | #include <string.h> | |
18 | #include <iostream> | |
19 | #include <time.h> | |
20 | #include <sys/mount.h> | |
21 | #include <boost/scoped_ptr.hpp> | |
22 | #include <boost/random/mersenne_twister.hpp> | |
23 | #include <boost/random/uniform_int.hpp> | |
24 | #include <boost/random/binomial_distribution.hpp> | |
25 | #include <gtest/gtest.h> | |
26 | ||
27 | #include "os/ObjectStore.h" | |
28 | #include "os/filestore/FileStore.h" | |
11fdf7f2 | 29 | #if defined(WITH_BLUESTORE) |
7c673cae | 30 | #include "os/bluestore/BlueStore.h" |
9f95a23c | 31 | #include "os/bluestore/BlueFS.h" |
7c673cae FG |
32 | #endif |
33 | #include "include/Context.h" | |
34 | #include "common/ceph_argparse.h" | |
9f95a23c | 35 | #include "common/admin_socket.h" |
7c673cae | 36 | #include "global/global_init.h" |
9f95a23c | 37 | #include "common/ceph_mutex.h" |
7c673cae FG |
38 | #include "common/Cond.h" |
39 | #include "common/errno.h" | |
40 | #include "include/stringify.h" | |
41 | #include "include/coredumpctl.h" | |
42 | ||
43 | #include "include/unordered_map.h" | |
44 | #include "store_test_fixture.h" | |
45 | ||
11fdf7f2 | 46 | using namespace std::placeholders; |
7c673cae FG |
47 | |
48 | typedef boost::mt11213b gen_type; | |
49 | ||
94b18763 | 50 | const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000; |
7c673cae FG |
51 | #define dout_context g_ceph_context |
52 | ||
7c673cae FG |
53 | static bool bl_eq(bufferlist& expected, bufferlist& actual) |
54 | { | |
55 | if (expected.contents_equal(actual)) | |
56 | return true; | |
57 | ||
58 | unsigned first = 0; | |
59 | if(expected.length() != actual.length()) { | |
60 | cout << "--- buffer lengths mismatch " << std::hex | |
61 | << "expected 0x" << expected.length() << " != actual 0x" | |
62 | << actual.length() << std::dec << std::endl; | |
63 | derr << "--- buffer lengths mismatch " << std::hex | |
64 | << "expected 0x" << expected.length() << " != actual 0x" | |
65 | << actual.length() << std::dec << dendl; | |
66 | } | |
11fdf7f2 | 67 | auto len = std::min(expected.length(), actual.length()); |
7c673cae FG |
68 | while ( first<len && expected[first] == actual[first]) |
69 | ++first; | |
70 | unsigned last = len; | |
71 | while (last > 0 && expected[last-1] == actual[last-1]) | |
72 | --last; | |
73 | if(len > 0) { | |
74 | cout << "--- buffer mismatch between offset 0x" << std::hex << first | |
75 | << " and 0x" << last << ", total 0x" << len << std::dec | |
76 | << std::endl; | |
77 | derr << "--- buffer mismatch between offset 0x" << std::hex << first | |
78 | << " and 0x" << last << ", total 0x" << len << std::dec | |
79 | << dendl; | |
80 | cout << "--- expected:\n"; | |
81 | expected.hexdump(cout); | |
82 | cout << "--- actual:\n"; | |
83 | actual.hexdump(cout); | |
84 | } | |
85 | return false; | |
86 | } | |
87 | ||
88 | ||
89 | ||
90 | template <typename T> | |
11fdf7f2 | 91 | int queue_transaction( |
7c673cae | 92 | T &store, |
11fdf7f2 | 93 | ObjectStore::CollectionHandle ch, |
7c673cae FG |
94 | ObjectStore::Transaction &&t) { |
95 | if (rand() % 2) { | |
96 | ObjectStore::Transaction t2; | |
97 | t2.append(t); | |
11fdf7f2 | 98 | return store->queue_transaction(ch, std::move(t2)); |
7c673cae | 99 | } else { |
11fdf7f2 | 100 | return store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
101 | } |
102 | } | |
103 | ||
104 | ||
105 | bool sorted(const vector<ghobject_t> &in) { | |
106 | ghobject_t start; | |
107 | for (vector<ghobject_t>::const_iterator i = in.begin(); | |
108 | i != in.end(); | |
109 | ++i) { | |
110 | if (start > *i) { | |
111 | cout << start << " should follow " << *i << std::endl; | |
112 | return false; | |
113 | } | |
114 | start = *i; | |
115 | } | |
116 | return true; | |
117 | } | |
118 | ||
119 | class StoreTest : public StoreTestFixture, | |
120 | public ::testing::WithParamInterface<const char*> { | |
121 | public: | |
122 | StoreTest() | |
123 | : StoreTestFixture(GetParam()) | |
124 | {} | |
11fdf7f2 TL |
125 | void doCompressionTest(); |
126 | void doSyntheticTest( | |
127 | int num_ops, | |
128 | uint64_t max_obj, uint64_t max_wr, uint64_t align); | |
7c673cae FG |
129 | }; |
130 | ||
131 | class StoreTestDeferredSetup : public StoreTest { | |
132 | void SetUp() override { | |
133 | //do nothing | |
134 | } | |
135 | ||
136 | protected: | |
137 | void DeferredSetup() { | |
138 | StoreTest::SetUp(); | |
139 | } | |
140 | ||
141 | public: | |
142 | }; | |
143 | ||
144 | class StoreTestSpecificAUSize : public StoreTestDeferredSetup { | |
145 | ||
146 | public: | |
147 | typedef | |
148 | std::function<void( | |
7c673cae FG |
149 | uint64_t num_ops, |
150 | uint64_t max_obj, | |
151 | uint64_t max_wr, | |
152 | uint64_t align)> MatrixTest; | |
153 | ||
154 | void StartDeferred(size_t min_alloc_size) { | |
11fdf7f2 | 155 | SetVal(g_conf(), "bluestore_min_alloc_size", stringify(min_alloc_size).c_str()); |
7c673cae FG |
156 | DeferredSetup(); |
157 | } | |
7c673cae FG |
158 | |
159 | private: | |
160 | // bluestore matrix testing | |
161 | uint64_t max_write = 40 * 1024; | |
162 | uint64_t max_size = 400 * 1024; | |
163 | uint64_t alignment = 0; | |
164 | uint64_t num_ops = 10000; | |
165 | ||
166 | protected: | |
167 | string matrix_get(const char *k) { | |
168 | if (string(k) == "max_write") { | |
169 | return stringify(max_write); | |
170 | } else if (string(k) == "max_size") { | |
171 | return stringify(max_size); | |
172 | } else if (string(k) == "alignment") { | |
173 | return stringify(alignment); | |
174 | } else if (string(k) == "num_ops") { | |
175 | return stringify(num_ops); | |
176 | } else { | |
177 | char *buf; | |
11fdf7f2 | 178 | g_conf().get_val(k, &buf, -1); |
7c673cae FG |
179 | string v = buf; |
180 | free(buf); | |
181 | return v; | |
182 | } | |
183 | } | |
184 | ||
185 | void matrix_set(const char *k, const char *v) { | |
186 | if (string(k) == "max_write") { | |
187 | max_write = atoll(v); | |
188 | } else if (string(k) == "max_size") { | |
189 | max_size = atoll(v); | |
190 | } else if (string(k) == "alignment") { | |
191 | alignment = atoll(v); | |
192 | } else if (string(k) == "num_ops") { | |
193 | num_ops = atoll(v); | |
194 | } else { | |
11fdf7f2 | 195 | SetVal(g_conf(), k, v); |
7c673cae FG |
196 | } |
197 | } | |
198 | ||
199 | void do_matrix_choose(const char *matrix[][10], | |
200 | int i, int pos, int num, | |
7c673cae FG |
201 | MatrixTest fn) { |
202 | if (matrix[i][0]) { | |
203 | int count; | |
204 | for (count = 0; matrix[i][count+1]; ++count) ; | |
205 | for (int j = 1; matrix[i][j]; ++j) { | |
206 | matrix_set(matrix[i][0], matrix[i][j]); | |
207 | do_matrix_choose(matrix, | |
208 | i + 1, | |
209 | pos * count + j - 1, | |
210 | num * count, | |
7c673cae FG |
211 | fn); |
212 | } | |
213 | } else { | |
214 | cout << "---------------------- " << (pos + 1) << " / " << num | |
215 | << " ----------------------" << std::endl; | |
216 | for (unsigned k=0; matrix[k][0]; ++k) { | |
217 | cout << " " << matrix[k][0] << " = " << matrix_get(matrix[k][0]) | |
218 | << std::endl; | |
219 | } | |
11fdf7f2 TL |
220 | g_ceph_context->_conf.apply_changes(nullptr); |
221 | fn(num_ops, max_size, max_write, alignment); | |
7c673cae FG |
222 | } |
223 | } | |
224 | ||
225 | void do_matrix(const char *matrix[][10], | |
7c673cae | 226 | MatrixTest fn) { |
7c673cae FG |
227 | |
228 | if (strcmp(matrix[0][0], "bluestore_min_alloc_size") == 0) { | |
229 | int count; | |
230 | for (count = 0; matrix[0][count+1]; ++count) ; | |
231 | for (size_t j = 1; matrix[0][j]; ++j) { | |
232 | if (j > 1) { | |
233 | TearDown(); | |
234 | } | |
235 | StartDeferred(strtoll(matrix[0][j], NULL, 10)); | |
11fdf7f2 | 236 | do_matrix_choose(matrix, 1, j - 1, count, fn); |
7c673cae FG |
237 | } |
238 | } else { | |
239 | StartDeferred(0); | |
11fdf7f2 | 240 | do_matrix_choose(matrix, 0, 0, 1, fn); |
7c673cae | 241 | } |
7c673cae FG |
242 | } |
243 | ||
244 | }; | |
245 | ||
246 | TEST_P(StoreTest, collect_metadata) { | |
247 | map<string,string> pm; | |
248 | store->collect_metadata(&pm); | |
249 | if (GetParam() == string("filestore")) { | |
250 | ASSERT_NE(pm.count("filestore_backend"), 0u); | |
251 | ASSERT_NE(pm.count("filestore_f_type"), 0u); | |
252 | ASSERT_NE(pm.count("backend_filestore_partition_path"), 0u); | |
253 | ASSERT_NE(pm.count("backend_filestore_dev_node"), 0u); | |
254 | } | |
255 | } | |
256 | ||
257 | TEST_P(StoreTest, Trivial) { | |
258 | } | |
259 | ||
260 | TEST_P(StoreTest, TrivialRemount) { | |
261 | int r = store->umount(); | |
262 | ASSERT_EQ(0, r); | |
263 | r = store->mount(); | |
264 | ASSERT_EQ(0, r); | |
265 | } | |
266 | ||
267 | TEST_P(StoreTest, SimpleRemount) { | |
7c673cae FG |
268 | coll_t cid; |
269 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
270 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
271 | bufferlist bl; | |
272 | bl.append("1234512345"); | |
273 | int r; | |
11fdf7f2 | 274 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
275 | { |
276 | cerr << "create collection + write" << std::endl; | |
277 | ObjectStore::Transaction t; | |
278 | t.create_collection(cid, 0); | |
279 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 280 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
281 | ASSERT_EQ(r, 0); |
282 | } | |
11fdf7f2 | 283 | ch.reset(); |
7c673cae FG |
284 | r = store->umount(); |
285 | ASSERT_EQ(0, r); | |
286 | r = store->mount(); | |
287 | ASSERT_EQ(0, r); | |
11fdf7f2 | 288 | ch = store->open_collection(cid); |
7c673cae FG |
289 | { |
290 | ObjectStore::Transaction t; | |
291 | t.write(cid, hoid2, 0, bl.length(), bl); | |
11fdf7f2 | 292 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
293 | ASSERT_EQ(r, 0); |
294 | } | |
295 | { | |
296 | ObjectStore::Transaction t; | |
297 | t.remove(cid, hoid); | |
298 | t.remove(cid, hoid2); | |
299 | t.remove_collection(cid); | |
300 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 301 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
302 | ASSERT_EQ(r, 0); |
303 | } | |
11fdf7f2 | 304 | ch.reset(); |
7c673cae FG |
305 | r = store->umount(); |
306 | ASSERT_EQ(0, r); | |
307 | r = store->mount(); | |
308 | ASSERT_EQ(0, r); | |
11fdf7f2 | 309 | ch = store->create_new_collection(cid); |
7c673cae FG |
310 | { |
311 | ObjectStore::Transaction t; | |
312 | t.create_collection(cid, 0); | |
11fdf7f2 | 313 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 314 | ASSERT_EQ(r, 0); |
11fdf7f2 | 315 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
316 | ASSERT_TRUE(!exists); |
317 | } | |
318 | { | |
319 | ObjectStore::Transaction t; | |
320 | t.remove_collection(cid); | |
321 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 322 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
323 | ASSERT_EQ(r, 0); |
324 | } | |
325 | } | |
326 | ||
327 | TEST_P(StoreTest, IORemount) { | |
7c673cae FG |
328 | coll_t cid; |
329 | bufferlist bl; | |
330 | bl.append("1234512345"); | |
331 | int r; | |
11fdf7f2 | 332 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
333 | { |
334 | cerr << "create collection + objects" << std::endl; | |
335 | ObjectStore::Transaction t; | |
336 | t.create_collection(cid, 0); | |
337 | for (int n=1; n<=100; ++n) { | |
338 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
339 | t.write(cid, hoid, 0, bl.length(), bl); | |
340 | } | |
11fdf7f2 | 341 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
342 | ASSERT_EQ(r, 0); |
343 | } | |
344 | // overwrites | |
345 | { | |
346 | cout << "overwrites" << std::endl; | |
347 | for (int n=1; n<=100; ++n) { | |
348 | ObjectStore::Transaction t; | |
349 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
350 | t.write(cid, hoid, 1, bl.length(), bl); | |
11fdf7f2 | 351 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
352 | ASSERT_EQ(r, 0); |
353 | } | |
354 | } | |
11fdf7f2 | 355 | ch.reset(); |
7c673cae FG |
356 | r = store->umount(); |
357 | ASSERT_EQ(0, r); | |
358 | r = store->mount(); | |
359 | ASSERT_EQ(0, r); | |
360 | { | |
361 | ObjectStore::Transaction t; | |
362 | for (int n=1; n<=100; ++n) { | |
363 | ghobject_t hoid(hobject_t(sobject_t("Object " + stringify(n), CEPH_NOSNAP))); | |
364 | t.remove(cid, hoid); | |
365 | } | |
366 | t.remove_collection(cid); | |
11fdf7f2 TL |
367 | auto ch = store->open_collection(cid); |
368 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
369 | ASSERT_EQ(r, 0); |
370 | } | |
371 | } | |
372 | ||
373 | TEST_P(StoreTest, UnprintableCharsName) { | |
7c673cae FG |
374 | coll_t cid; |
375 | string name = "funnychars_"; | |
376 | for (unsigned i = 0; i < 256; ++i) { | |
377 | name.push_back(i); | |
378 | } | |
379 | ghobject_t oid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
380 | int r; | |
11fdf7f2 | 381 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
382 | { |
383 | cerr << "create collection + object" << std::endl; | |
384 | ObjectStore::Transaction t; | |
385 | t.create_collection(cid, 0); | |
386 | t.touch(cid, oid); | |
11fdf7f2 | 387 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
388 | ASSERT_EQ(r, 0); |
389 | } | |
11fdf7f2 | 390 | ch.reset(); |
7c673cae FG |
391 | r = store->umount(); |
392 | ASSERT_EQ(0, r); | |
393 | r = store->mount(); | |
394 | ASSERT_EQ(0, r); | |
395 | { | |
396 | cout << "removing" << std::endl; | |
397 | ObjectStore::Transaction t; | |
398 | t.remove(cid, oid); | |
399 | t.remove_collection(cid); | |
11fdf7f2 TL |
400 | auto ch = store->open_collection(cid); |
401 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
402 | ASSERT_EQ(r, 0); |
403 | } | |
404 | } | |
405 | ||
406 | TEST_P(StoreTest, FiemapEmpty) { | |
7c673cae FG |
407 | coll_t cid; |
408 | int r = 0; | |
409 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
11fdf7f2 | 410 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
411 | { |
412 | ObjectStore::Transaction t; | |
413 | t.create_collection(cid, 0); | |
414 | t.touch(cid, oid); | |
415 | t.truncate(cid, oid, 100000); | |
11fdf7f2 | 416 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
417 | ASSERT_EQ(r, 0); |
418 | } | |
419 | { | |
420 | bufferlist bl; | |
11fdf7f2 | 421 | store->fiemap(ch, oid, 0, 100000, bl); |
7c673cae | 422 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
423 | auto p = bl.cbegin(); |
424 | decode(m, p); | |
7c673cae FG |
425 | cout << " got " << m << std::endl; |
426 | e[0] = 100000; | |
427 | EXPECT_TRUE(m == e || m.empty()); | |
428 | } | |
429 | { | |
430 | ObjectStore::Transaction t; | |
431 | t.remove(cid, oid); | |
432 | t.remove_collection(cid); | |
433 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 434 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
435 | ASSERT_EQ(r, 0); |
436 | } | |
437 | } | |
438 | ||
439 | TEST_P(StoreTest, FiemapHoles) { | |
7c673cae FG |
440 | const uint64_t MAX_EXTENTS = 4000; |
441 | const uint64_t SKIP_STEP = 65536; | |
442 | coll_t cid; | |
443 | int r = 0; | |
444 | ghobject_t oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP))); | |
445 | bufferlist bl; | |
446 | bl.append("foo"); | |
11fdf7f2 | 447 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
448 | { |
449 | ObjectStore::Transaction t; | |
450 | t.create_collection(cid, 0); | |
451 | t.touch(cid, oid); | |
452 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) | |
453 | t.write(cid, oid, SKIP_STEP * i, 3, bl); | |
11fdf7f2 | 454 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
455 | ASSERT_EQ(r, 0); |
456 | } | |
457 | { | |
458 | //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3 | |
459 | bufferlist bl; | |
11fdf7f2 | 460 | store->fiemap(ch, oid, 0, SKIP_STEP * (MAX_EXTENTS - 1) + 3, bl); |
7c673cae | 461 | map<uint64_t,uint64_t> m, e; |
11fdf7f2 TL |
462 | auto p = bl.cbegin(); |
463 | decode(m, p); | |
7c673cae FG |
464 | cout << " got " << m << std::endl; |
465 | ASSERT_TRUE(!m.empty()); | |
466 | ASSERT_GE(m[0], 3u); | |
11fdf7f2 TL |
467 | auto last = m.crbegin(); |
468 | if (m.size() == 1) { | |
469 | ASSERT_EQ(0u, last->first); | |
470 | } else if (m.size() == MAX_EXTENTS) { | |
471 | for (uint64_t i = 0; i < MAX_EXTENTS; i++) { | |
472 | ASSERT_TRUE(m.count(SKIP_STEP * i)); | |
473 | } | |
7c673cae | 474 | } |
11fdf7f2 TL |
475 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); |
476 | } | |
477 | { | |
7c673cae | 478 | // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3 |
11fdf7f2 TL |
479 | bufferlist bl; |
480 | store->fiemap(ch, oid, SKIP_STEP, SKIP_STEP * (MAX_EXTENTS - 2) + 3, bl); | |
481 | map<uint64_t,uint64_t> m, e; | |
482 | auto p = bl.cbegin(); | |
483 | decode(m, p); | |
7c673cae FG |
484 | cout << " got " << m << std::endl; |
485 | ASSERT_TRUE(!m.empty()); | |
11fdf7f2 TL |
486 | // kstore always returns [0, object_size] regardless of offset and length |
487 | // FIXME: if fiemap logic in kstore is refined | |
488 | if (string(GetParam()) != "kstore") { | |
489 | ASSERT_GE(m[SKIP_STEP], 3u); | |
490 | auto last = m.crbegin(); | |
491 | if (m.size() == 1) { | |
492 | ASSERT_EQ(SKIP_STEP, last->first); | |
493 | } else if (m.size() == MAX_EXTENTS - 2) { | |
494 | for (uint64_t i = 1; i < MAX_EXTENTS - 1; i++) { | |
495 | ASSERT_TRUE(m.count(SKIP_STEP*i)); | |
496 | } | |
497 | } | |
498 | ASSERT_GT(last->first + last->second, SKIP_STEP * (MAX_EXTENTS - 1)); | |
7c673cae | 499 | } |
7c673cae FG |
500 | } |
501 | { | |
502 | ObjectStore::Transaction t; | |
503 | t.remove(cid, oid); | |
504 | t.remove_collection(cid); | |
505 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 506 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
507 | ASSERT_EQ(r, 0); |
508 | } | |
509 | } | |
510 | ||
511 | TEST_P(StoreTest, SimpleMetaColTest) { | |
7c673cae FG |
512 | coll_t cid; |
513 | int r = 0; | |
514 | { | |
11fdf7f2 | 515 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
516 | ObjectStore::Transaction t; |
517 | t.create_collection(cid, 0); | |
518 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 519 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
520 | ASSERT_EQ(r, 0); |
521 | } | |
522 | { | |
523 | ObjectStore::Transaction t; | |
524 | t.remove_collection(cid); | |
525 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
526 | auto ch = store->open_collection(cid); |
527 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
528 | ASSERT_EQ(r, 0); |
529 | } | |
530 | { | |
11fdf7f2 | 531 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
532 | ObjectStore::Transaction t; |
533 | t.create_collection(cid, 0); | |
534 | cerr << "add collection" << std::endl; | |
11fdf7f2 | 535 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
536 | ASSERT_EQ(r, 0); |
537 | } | |
538 | { | |
539 | ObjectStore::Transaction t; | |
540 | t.remove_collection(cid); | |
541 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
542 | auto ch = store->open_collection(cid); |
543 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
544 | ASSERT_EQ(r, 0); |
545 | } | |
546 | } | |
547 | ||
548 | TEST_P(StoreTest, SimplePGColTest) { | |
7c673cae FG |
549 | coll_t cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD)); |
550 | int r = 0; | |
551 | { | |
552 | ObjectStore::Transaction t; | |
11fdf7f2 | 553 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
554 | t.create_collection(cid, 4); |
555 | cerr << "create collection" << std::endl; | |
11fdf7f2 | 556 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
557 | ASSERT_EQ(r, 0); |
558 | } | |
559 | { | |
560 | ObjectStore::Transaction t; | |
561 | t.remove_collection(cid); | |
562 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
563 | auto ch = store->open_collection(cid); |
564 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
565 | ASSERT_EQ(r, 0); |
566 | } | |
567 | { | |
568 | ObjectStore::Transaction t; | |
569 | t.create_collection(cid, 4); | |
570 | cerr << "add collection" << std::endl; | |
11fdf7f2 TL |
571 | auto ch = store->create_new_collection(cid); |
572 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
573 | ASSERT_EQ(r, 0); |
574 | } | |
575 | { | |
576 | ObjectStore::Transaction t; | |
577 | t.remove_collection(cid); | |
578 | cerr << "remove collection" << std::endl; | |
11fdf7f2 TL |
579 | auto ch = store->open_collection(cid); |
580 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
581 | ASSERT_EQ(r, 0); |
582 | } | |
583 | } | |
584 | ||
585 | TEST_P(StoreTest, SimpleColPreHashTest) { | |
7c673cae FG |
586 | // Firstly we will need to revert the value making sure |
587 | // collection hint actually works | |
588 | int merge_threshold = g_ceph_context->_conf->filestore_merge_threshold; | |
589 | std::ostringstream oss; | |
590 | if (merge_threshold > 0) { | |
591 | oss << "-" << merge_threshold; | |
11fdf7f2 | 592 | SetVal(g_conf(), "filestore_merge_threshold", oss.str().c_str()); |
7c673cae FG |
593 | } |
594 | ||
595 | uint32_t pg_num = 128; | |
596 | ||
597 | boost::uniform_int<> pg_id_range(0, pg_num); | |
598 | gen_type rng(time(NULL)); | |
599 | int pg_id = pg_id_range(rng); | |
600 | ||
601 | int objs_per_folder = abs(merge_threshold) * 16 * g_ceph_context->_conf->filestore_split_multiple; | |
602 | boost::uniform_int<> folders_range(5, 256); | |
603 | uint64_t expected_num_objs = (uint64_t)objs_per_folder * (uint64_t)folders_range(rng); | |
604 | ||
605 | coll_t cid(spg_t(pg_t(pg_id, 15), shard_id_t::NO_SHARD)); | |
606 | int r; | |
11fdf7f2 | 607 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
608 | { |
609 | // Create a collection along with a hint | |
610 | ObjectStore::Transaction t; | |
611 | t.create_collection(cid, 5); | |
612 | cerr << "create collection" << std::endl; | |
613 | bufferlist hint; | |
11fdf7f2 TL |
614 | encode(pg_num, hint); |
615 | encode(expected_num_objs, hint); | |
7c673cae FG |
616 | t.collection_hint(cid, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS, hint); |
617 | cerr << "collection hint" << std::endl; | |
11fdf7f2 | 618 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
619 | ASSERT_EQ(r, 0); |
620 | } | |
621 | { | |
622 | // Remove the collection | |
623 | ObjectStore::Transaction t; | |
624 | t.remove_collection(cid); | |
625 | cerr << "remove collection" << std::endl; | |
11fdf7f2 | 626 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
627 | ASSERT_EQ(r, 0); |
628 | } | |
7c673cae FG |
629 | } |
630 | ||
631 | TEST_P(StoreTest, SmallBlockWrites) { | |
7c673cae FG |
632 | int r; |
633 | coll_t cid; | |
11fdf7f2 | 634 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
635 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
636 | { | |
637 | ObjectStore::Transaction t; | |
638 | t.create_collection(cid, 0); | |
639 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 640 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
641 | ASSERT_EQ(r, 0); |
642 | } | |
643 | bufferlist a; | |
644 | bufferptr ap(0x1000); | |
645 | memset(ap.c_str(), 'a', 0x1000); | |
646 | a.append(ap); | |
647 | bufferlist b; | |
648 | bufferptr bp(0x1000); | |
649 | memset(bp.c_str(), 'b', 0x1000); | |
650 | b.append(bp); | |
651 | bufferlist c; | |
652 | bufferptr cp(0x1000); | |
653 | memset(cp.c_str(), 'c', 0x1000); | |
654 | c.append(cp); | |
655 | bufferptr zp(0x1000); | |
656 | zp.zero(); | |
657 | bufferlist z; | |
658 | z.append(zp); | |
659 | { | |
660 | ObjectStore::Transaction t; | |
661 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 662 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
663 | ASSERT_EQ(r, 0); |
664 | ||
665 | bufferlist in, exp; | |
11fdf7f2 | 666 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
667 | ASSERT_EQ(0x1000, r); |
668 | exp.append(a); | |
669 | ASSERT_TRUE(bl_eq(exp, in)); | |
670 | } | |
671 | { | |
672 | ObjectStore::Transaction t; | |
673 | t.write(cid, hoid, 0x1000, 0x1000, b); | |
11fdf7f2 | 674 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
675 | ASSERT_EQ(r, 0); |
676 | ||
677 | bufferlist in, exp; | |
11fdf7f2 | 678 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
679 | ASSERT_EQ(0x2000, r); |
680 | exp.append(a); | |
681 | exp.append(b); | |
682 | ASSERT_TRUE(bl_eq(exp, in)); | |
683 | } | |
684 | { | |
685 | ObjectStore::Transaction t; | |
686 | t.write(cid, hoid, 0x3000, 0x1000, c); | |
11fdf7f2 | 687 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
688 | ASSERT_EQ(r, 0); |
689 | ||
690 | bufferlist in, exp; | |
11fdf7f2 | 691 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
692 | ASSERT_EQ(0x4000, r); |
693 | exp.append(a); | |
694 | exp.append(b); | |
695 | exp.append(z); | |
696 | exp.append(c); | |
697 | ASSERT_TRUE(bl_eq(exp, in)); | |
698 | } | |
699 | { | |
700 | ObjectStore::Transaction t; | |
701 | t.write(cid, hoid, 0x2000, 0x1000, a); | |
11fdf7f2 | 702 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
703 | ASSERT_EQ(r, 0); |
704 | ||
705 | bufferlist in, exp; | |
11fdf7f2 | 706 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
707 | ASSERT_EQ(0x4000, r); |
708 | exp.append(a); | |
709 | exp.append(b); | |
710 | exp.append(a); | |
711 | exp.append(c); | |
712 | ASSERT_TRUE(bl_eq(exp, in)); | |
713 | } | |
714 | { | |
715 | ObjectStore::Transaction t; | |
716 | t.write(cid, hoid, 0, 0x1000, c); | |
11fdf7f2 | 717 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
718 | ASSERT_EQ(r, 0); |
719 | } | |
720 | { | |
721 | bufferlist in, exp; | |
11fdf7f2 | 722 | r = store->read(ch, hoid, 0, 0x4000, in); |
7c673cae FG |
723 | ASSERT_EQ(0x4000, r); |
724 | exp.append(c); | |
725 | exp.append(b); | |
726 | exp.append(a); | |
727 | exp.append(c); | |
728 | ASSERT_TRUE(bl_eq(exp, in)); | |
729 | } | |
730 | { | |
731 | ObjectStore::Transaction t; | |
732 | t.remove(cid, hoid); | |
733 | t.remove_collection(cid); | |
734 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 735 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
736 | ASSERT_EQ(r, 0); |
737 | } | |
738 | } | |
739 | ||
740 | TEST_P(StoreTest, BufferCacheReadTest) { | |
7c673cae FG |
741 | int r; |
742 | coll_t cid; | |
743 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
744 | { | |
11fdf7f2 TL |
745 | auto ch = store->open_collection(cid); |
746 | ASSERT_FALSE(ch); | |
7c673cae | 747 | } |
11fdf7f2 | 748 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
749 | { |
750 | ObjectStore::Transaction t; | |
751 | t.create_collection(cid, 0); | |
752 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 753 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
754 | ASSERT_EQ(r, 0); |
755 | } | |
756 | { | |
11fdf7f2 | 757 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
758 | ASSERT_TRUE(!exists); |
759 | ||
760 | ObjectStore::Transaction t; | |
761 | t.touch(cid, hoid); | |
762 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 763 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
764 | ASSERT_EQ(r, 0); |
765 | ||
11fdf7f2 | 766 | exists = store->exists(ch, hoid); |
7c673cae FG |
767 | ASSERT_EQ(true, exists); |
768 | } | |
769 | { | |
770 | ObjectStore::Transaction t; | |
771 | bufferlist bl, newdata; | |
772 | bl.append("abcde"); | |
773 | t.write(cid, hoid, 0, 5, bl); | |
774 | t.write(cid, hoid, 10, 5, bl); | |
775 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 776 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
777 | ASSERT_EQ(r, 0); |
778 | ||
11fdf7f2 | 779 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
780 | ASSERT_EQ(r, 15); |
781 | { | |
782 | bufferlist expected; | |
783 | expected.append(bl); | |
784 | expected.append_zero(5); | |
785 | expected.append(bl); | |
786 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
787 | } | |
788 | } | |
789 | //overwrite over the same extents | |
790 | { | |
791 | ObjectStore::Transaction t; | |
792 | bufferlist bl, newdata; | |
793 | bl.append("edcba"); | |
794 | t.write(cid, hoid, 0, 5, bl); | |
795 | t.write(cid, hoid, 10, 5, bl); | |
796 | cerr << "TwinWrite" << std::endl; | |
11fdf7f2 | 797 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
798 | ASSERT_EQ(r, 0); |
799 | ||
11fdf7f2 | 800 | r = store->read(ch, hoid, 0, 15, newdata); |
7c673cae FG |
801 | ASSERT_EQ(r, 15); |
802 | { | |
803 | bufferlist expected; | |
804 | expected.append(bl); | |
805 | expected.append_zero(5); | |
806 | expected.append(bl); | |
807 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
808 | } | |
809 | } | |
810 | //additional write to an unused region of some blob | |
811 | { | |
812 | ObjectStore::Transaction t; | |
813 | bufferlist bl2, newdata; | |
814 | bl2.append("1234567890"); | |
815 | ||
816 | t.write(cid, hoid, 20, bl2.length(), bl2); | |
817 | cerr << "Append" << std::endl; | |
11fdf7f2 | 818 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
819 | ASSERT_EQ(r, 0); |
820 | ||
11fdf7f2 | 821 | r = store->read(ch, hoid, 0, 30, newdata); |
7c673cae FG |
822 | ASSERT_EQ(r, 30); |
823 | { | |
824 | bufferlist expected; | |
825 | expected.append("edcba"); | |
826 | expected.append_zero(5); | |
827 | expected.append("edcba"); | |
828 | expected.append_zero(5); | |
829 | expected.append(bl2); | |
830 | ||
831 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
832 | } | |
833 | } | |
834 | //additional write to an unused region of some blob and partial owerite over existing extents | |
835 | { | |
836 | ObjectStore::Transaction t; | |
837 | bufferlist bl, bl2, bl3, newdata; | |
838 | bl.append("DCB"); | |
839 | bl2.append("1234567890"); | |
840 | bl3.append("BA"); | |
841 | ||
842 | t.write(cid, hoid, 30, bl2.length(), bl2); | |
843 | t.write(cid, hoid, 1, bl.length(), bl); | |
844 | t.write(cid, hoid, 13, bl3.length(), bl3); | |
845 | cerr << "TripleWrite" << std::endl; | |
11fdf7f2 | 846 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
847 | ASSERT_EQ(r, 0); |
848 | ||
11fdf7f2 | 849 | r = store->read(ch, hoid, 0, 40, newdata); |
7c673cae FG |
850 | ASSERT_EQ(r, 40); |
851 | { | |
852 | bufferlist expected; | |
853 | expected.append("eDCBa"); | |
854 | expected.append_zero(5); | |
855 | expected.append("edcBA"); | |
856 | expected.append_zero(5); | |
857 | expected.append(bl2); | |
858 | expected.append(bl2); | |
859 | ||
860 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
861 | } | |
862 | } | |
863 | } | |
864 | ||
11fdf7f2 | 865 | void StoreTest::doCompressionTest() |
7c673cae | 866 | { |
7c673cae FG |
867 | int r; |
868 | coll_t cid; | |
869 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
870 | { | |
11fdf7f2 TL |
871 | auto ch = store->open_collection(cid); |
872 | ASSERT_FALSE(ch); | |
7c673cae | 873 | } |
11fdf7f2 | 874 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
875 | { |
876 | ObjectStore::Transaction t; | |
877 | t.create_collection(cid, 0); | |
878 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 879 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
880 | ASSERT_EQ(r, 0); |
881 | } | |
882 | { | |
11fdf7f2 | 883 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
884 | ASSERT_TRUE(!exists); |
885 | ||
886 | ObjectStore::Transaction t; | |
887 | t.touch(cid, hoid); | |
888 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 889 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
890 | ASSERT_EQ(r, 0); |
891 | ||
11fdf7f2 | 892 | exists = store->exists(ch, hoid); |
7c673cae FG |
893 | ASSERT_EQ(true, exists); |
894 | } | |
895 | std::string data; | |
896 | data.resize(0x10000 * 4); | |
897 | for(size_t i = 0;i < data.size(); i++) | |
898 | data[i] = i / 256; | |
899 | { | |
900 | ObjectStore::Transaction t; | |
901 | bufferlist bl, newdata; | |
902 | bl.append(data); | |
903 | t.write(cid, hoid, 0, bl.length(), bl); | |
904 | cerr << "CompressibleData (4xAU) Write" << std::endl; | |
11fdf7f2 | 905 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
906 | ASSERT_EQ(r, 0); |
907 | ||
11fdf7f2 | 908 | r = store->read(ch, hoid, 0, data.size() , newdata); |
7c673cae FG |
909 | |
910 | ASSERT_EQ(r, (int)data.size()); | |
911 | { | |
912 | bufferlist expected; | |
913 | expected.append(data); | |
914 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
915 | } | |
916 | newdata.clear(); | |
11fdf7f2 | 917 | r = store->read(ch, hoid, 0, 711 , newdata); |
7c673cae FG |
918 | ASSERT_EQ(r, 711); |
919 | { | |
920 | bufferlist expected; | |
921 | expected.append(data.substr(0,711)); | |
922 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
923 | } | |
924 | newdata.clear(); | |
11fdf7f2 | 925 | r = store->read(ch, hoid, 0xf00f, data.size(), newdata); |
7c673cae FG |
926 | ASSERT_EQ(r, int(data.size() - 0xf00f) ); |
927 | { | |
928 | bufferlist expected; | |
929 | expected.append(data.substr(0xf00f)); | |
930 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
931 | } | |
932 | { | |
933 | struct store_statfs_t statfs; | |
934 | int r = store->statfs(&statfs); | |
935 | ASSERT_EQ(r, 0); | |
11fdf7f2 TL |
936 | ASSERT_EQ(statfs.data_stored, (unsigned)data.size()); |
937 | ASSERT_LE(statfs.data_compressed, (unsigned)data.size()); | |
938 | ASSERT_EQ(statfs.data_compressed_original, (unsigned)data.size()); | |
939 | ASSERT_LE(statfs.data_compressed_allocated, (unsigned)data.size()); | |
7c673cae FG |
940 | } |
941 | } | |
942 | std::string data2; | |
943 | data2.resize(0x10000 * 4 - 0x9000); | |
944 | for(size_t i = 0;i < data2.size(); i++) | |
945 | data2[i] = (i+1) / 256; | |
946 | { | |
947 | ObjectStore::Transaction t; | |
948 | bufferlist bl, newdata; | |
949 | bl.append(data2); | |
950 | t.write(cid, hoid, 0x8000, bl.length(), bl); | |
951 | cerr << "CompressibleData partial overwrite" << std::endl; | |
11fdf7f2 | 952 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
953 | ASSERT_EQ(r, 0); |
954 | ||
11fdf7f2 | 955 | r = store->read(ch, hoid, 0, 0x10000, newdata); |
7c673cae FG |
956 | ASSERT_EQ(r, (int)0x10000); |
957 | { | |
958 | bufferlist expected; | |
959 | expected.append(data.substr(0, 0x8000)); | |
960 | expected.append(data2.substr(0, 0x8000)); | |
961 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
962 | } | |
963 | newdata.clear(); | |
11fdf7f2 | 964 | r = store->read(ch, hoid, 0x9000, 711 , newdata); |
7c673cae FG |
965 | ASSERT_EQ(r, 711); |
966 | { | |
967 | bufferlist expected; | |
968 | expected.append(data2.substr(0x1000,711)); | |
969 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
970 | } | |
971 | newdata.clear(); | |
11fdf7f2 | 972 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
973 | ASSERT_EQ(r, int(0x40000) ); |
974 | { | |
975 | bufferlist expected; | |
976 | expected.append(data.substr(0, 0x8000)); | |
977 | expected.append(data2.substr(0, 0x37000)); | |
978 | expected.append(data.substr(0x3f000, 0x1000)); | |
979 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
980 | } | |
981 | } | |
982 | data2.resize(0x3f000); | |
983 | for(size_t i = 0;i < data2.size(); i++) | |
984 | data2[i] = (i+2) / 256; | |
985 | { | |
986 | ObjectStore::Transaction t; | |
987 | bufferlist bl, newdata; | |
988 | bl.append(data2); | |
989 | t.write(cid, hoid, 0, bl.length(), bl); | |
990 | cerr << "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 991 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
992 | ASSERT_EQ(r, 0); |
993 | ||
11fdf7f2 | 994 | r = store->read(ch, hoid, 0, 0x3e000 - 1, newdata); |
7c673cae FG |
995 | ASSERT_EQ(r, (int)0x3e000 - 1); |
996 | { | |
997 | bufferlist expected; | |
998 | expected.append(data2.substr(0, 0x3e000 - 1)); | |
999 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1000 | } | |
1001 | newdata.clear(); | |
11fdf7f2 | 1002 | r = store->read(ch, hoid, 0x3e000-1, 0x2001, newdata); |
7c673cae FG |
1003 | ASSERT_EQ(r, 0x2001); |
1004 | { | |
1005 | bufferlist expected; | |
1006 | expected.append(data2.substr(0x3e000-1, 0x1001)); | |
1007 | expected.append(data.substr(0x3f000, 0x1000)); | |
1008 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1009 | } | |
1010 | newdata.clear(); | |
11fdf7f2 | 1011 | r = store->read(ch, hoid, 0x0, 0x40000, newdata); |
7c673cae FG |
1012 | ASSERT_EQ(r, int(0x40000) ); |
1013 | { | |
1014 | bufferlist expected; | |
1015 | expected.append(data2.substr(0, 0x3f000)); | |
1016 | expected.append(data.substr(0x3f000, 0x1000)); | |
1017 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1018 | } | |
1019 | } | |
1020 | data.resize(0x1001); | |
1021 | for(size_t i = 0;i < data.size(); i++) | |
1022 | data[i] = (i+3) / 256; | |
1023 | { | |
1024 | ObjectStore::Transaction t; | |
1025 | bufferlist bl, newdata; | |
1026 | bl.append(data); | |
1027 | t.write(cid, hoid, 0x3f000-1, bl.length(), bl); | |
1028 | cerr << "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl; | |
11fdf7f2 | 1029 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1030 | ASSERT_EQ(r, 0); |
1031 | ||
11fdf7f2 | 1032 | r = store->read(ch, hoid, 0x3e000, 0x2000, newdata); |
7c673cae FG |
1033 | ASSERT_EQ(r, (int)0x2000); |
1034 | { | |
1035 | bufferlist expected; | |
1036 | expected.append(data2.substr(0x3e000, 0x1000 - 1)); | |
1037 | expected.append(data.substr(0, 0x1001)); | |
1038 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
1039 | } | |
1040 | } | |
1041 | { | |
1042 | ObjectStore::Transaction t; | |
1043 | t.remove(cid, hoid); | |
1044 | cerr << "Cleaning object" << std::endl; | |
11fdf7f2 | 1045 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1046 | ASSERT_EQ(r, 0); |
1047 | } | |
1048 | //force fsck | |
11fdf7f2 | 1049 | ch.reset(); |
7c673cae | 1050 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1051 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1052 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 TL |
1053 | ch = store->open_collection(cid); |
1054 | auto settingsBookmark = BookmarkSettings(); | |
1055 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
1056 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 1057 | { |
7c673cae FG |
1058 | data.resize(0x10000*6); |
1059 | ||
1060 | for(size_t i = 0;i < data.size(); i++) | |
1061 | data[i] = i / 256; | |
1062 | ObjectStore::Transaction t; | |
1063 | bufferlist bl, newdata; | |
1064 | bl.append(data); | |
1065 | t.write(cid, hoid, 0, bl.length(), bl); | |
1066 | cerr << "CompressibleData large blob" << std::endl; | |
11fdf7f2 | 1067 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1068 | ASSERT_EQ(r, 0); |
1069 | } | |
1070 | //force fsck | |
11fdf7f2 | 1071 | ch.reset(); |
7c673cae | 1072 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1073 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1074 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1075 | ch = store->open_collection(cid); |
7c673cae FG |
1076 | { |
1077 | ObjectStore::Transaction t; | |
1078 | t.remove(cid, hoid); | |
1079 | t.remove_collection(cid); | |
1080 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1081 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1082 | ASSERT_EQ(r, 0); |
1083 | } | |
7c673cae FG |
1084 | } |
1085 | ||
1086 | TEST_P(StoreTest, CompressionTest) { | |
1087 | if (string(GetParam()) != "bluestore") | |
1088 | return; | |
1089 | ||
11fdf7f2 TL |
1090 | SetVal(g_conf(), "bluestore_compression_algorithm", "snappy"); |
1091 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
1092 | g_ceph_context->_conf.apply_changes(nullptr); | |
1093 | doCompressionTest(); | |
7c673cae | 1094 | |
11fdf7f2 TL |
1095 | SetVal(g_conf(), "bluestore_compression_algorithm", "zlib"); |
1096 | SetVal(g_conf(), "bluestore_compression_mode", "aggressive"); | |
1097 | g_ceph_context->_conf.apply_changes(nullptr); | |
1098 | doCompressionTest(); | |
7c673cae FG |
1099 | } |
1100 | ||
1101 | TEST_P(StoreTest, SimpleObjectTest) { | |
7c673cae FG |
1102 | int r; |
1103 | coll_t cid; | |
1104 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1105 | { | |
11fdf7f2 TL |
1106 | auto ch = store->open_collection(cid); |
1107 | ASSERT_FALSE(ch); | |
7c673cae | 1108 | } |
11fdf7f2 | 1109 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1110 | { |
1111 | ObjectStore::Transaction t; | |
1112 | t.create_collection(cid, 0); | |
1113 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1114 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1115 | ASSERT_EQ(r, 0); |
1116 | } | |
1117 | { | |
11fdf7f2 | 1118 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1119 | ASSERT_TRUE(!exists); |
1120 | ||
1121 | ObjectStore::Transaction t; | |
1122 | t.touch(cid, hoid); | |
1123 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1124 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1125 | ASSERT_EQ(r, 0); |
1126 | ||
11fdf7f2 | 1127 | exists = store->exists(ch, hoid); |
7c673cae FG |
1128 | ASSERT_EQ(true, exists); |
1129 | } | |
1130 | { | |
1131 | ObjectStore::Transaction t; | |
1132 | t.remove(cid, hoid); | |
1133 | t.touch(cid, hoid); | |
1134 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1135 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1136 | ASSERT_EQ(r, 0); |
1137 | } | |
1138 | { | |
1139 | ObjectStore::Transaction t; | |
1140 | bufferlist bl, orig; | |
1141 | bl.append("abcde"); | |
1142 | orig = bl; | |
1143 | t.remove(cid, hoid); | |
1144 | t.write(cid, hoid, 0, 5, bl); | |
1145 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 1146 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1147 | ASSERT_EQ(r, 0); |
1148 | ||
1149 | bufferlist in; | |
11fdf7f2 | 1150 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
1151 | ASSERT_EQ(5, r); |
1152 | ASSERT_TRUE(bl_eq(orig, in)); | |
1153 | } | |
1154 | { | |
1155 | ObjectStore::Transaction t; | |
1156 | bufferlist bl, exp; | |
1157 | bl.append("abcde"); | |
1158 | exp = bl; | |
1159 | exp.append(bl); | |
1160 | t.write(cid, hoid, 5, 5, bl); | |
1161 | cerr << "Append" << std::endl; | |
11fdf7f2 | 1162 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1163 | ASSERT_EQ(r, 0); |
1164 | ||
1165 | bufferlist in; | |
11fdf7f2 | 1166 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1167 | ASSERT_EQ(10, r); |
1168 | ASSERT_TRUE(bl_eq(exp, in)); | |
1169 | } | |
1170 | { | |
1171 | ObjectStore::Transaction t; | |
1172 | bufferlist bl, exp; | |
1173 | bl.append("abcdeabcde"); | |
1174 | exp = bl; | |
1175 | t.write(cid, hoid, 0, 10, bl); | |
1176 | cerr << "Full overwrite" << std::endl; | |
11fdf7f2 | 1177 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1178 | ASSERT_EQ(r, 0); |
1179 | ||
1180 | bufferlist in; | |
11fdf7f2 | 1181 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1182 | ASSERT_EQ(10, r); |
1183 | ASSERT_TRUE(bl_eq(exp, in)); | |
1184 | } | |
1185 | { | |
1186 | ObjectStore::Transaction t; | |
1187 | bufferlist bl; | |
1188 | bl.append("abcde"); | |
1189 | t.write(cid, hoid, 3, 5, bl); | |
1190 | cerr << "Partial overwrite" << std::endl; | |
11fdf7f2 | 1191 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1192 | ASSERT_EQ(r, 0); |
1193 | ||
1194 | bufferlist in, exp; | |
1195 | exp.append("abcabcdede"); | |
11fdf7f2 | 1196 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1197 | ASSERT_EQ(10, r); |
1198 | in.hexdump(cout); | |
1199 | ASSERT_TRUE(bl_eq(exp, in)); | |
1200 | } | |
1201 | { | |
1202 | { | |
1203 | ObjectStore::Transaction t; | |
1204 | bufferlist bl; | |
1205 | bl.append("fghij"); | |
1206 | t.truncate(cid, hoid, 0); | |
1207 | t.write(cid, hoid, 5, 5, bl); | |
1208 | cerr << "Truncate + hole" << std::endl; | |
11fdf7f2 | 1209 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1210 | ASSERT_EQ(r, 0); |
1211 | } | |
1212 | { | |
1213 | ObjectStore::Transaction t; | |
1214 | bufferlist bl; | |
1215 | bl.append("abcde"); | |
1216 | t.write(cid, hoid, 0, 5, bl); | |
1217 | cerr << "Reverse fill-in" << std::endl; | |
11fdf7f2 | 1218 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1219 | ASSERT_EQ(r, 0); |
1220 | } | |
1221 | ||
1222 | bufferlist in, exp; | |
1223 | exp.append("abcdefghij"); | |
11fdf7f2 | 1224 | r = store->read(ch, hoid, 0, 10, in); |
7c673cae FG |
1225 | ASSERT_EQ(10, r); |
1226 | in.hexdump(cout); | |
1227 | ASSERT_TRUE(bl_eq(exp, in)); | |
1228 | } | |
1229 | { | |
1230 | ObjectStore::Transaction t; | |
1231 | bufferlist bl; | |
1232 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1233 | t.write(cid, hoid, 0, bl.length(), bl); | |
1234 | cerr << "larger overwrite" << std::endl; | |
11fdf7f2 | 1235 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1236 | ASSERT_EQ(r, 0); |
1237 | ||
1238 | bufferlist in; | |
11fdf7f2 | 1239 | r = store->read(ch, hoid, 0, bl.length(), in); |
7c673cae FG |
1240 | ASSERT_EQ((int)bl.length(), r); |
1241 | in.hexdump(cout); | |
1242 | ASSERT_TRUE(bl_eq(bl, in)); | |
1243 | } | |
1244 | { | |
1245 | bufferlist bl; | |
1246 | bl.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234"); | |
1247 | ||
1248 | //test: offset=len=0 mean read all data | |
1249 | bufferlist in; | |
11fdf7f2 | 1250 | r = store->read(ch, hoid, 0, 0, in); |
7c673cae FG |
1251 | ASSERT_EQ((int)bl.length(), r); |
1252 | in.hexdump(cout); | |
1253 | ASSERT_TRUE(bl_eq(bl, in)); | |
1254 | } | |
1255 | { | |
1256 | //verifying unaligned csums | |
1257 | std::string s1("1"), s2(0x1000, '2'), s3("00"); | |
1258 | { | |
1259 | ObjectStore::Transaction t; | |
1260 | bufferlist bl; | |
1261 | bl.append(s1); | |
1262 | bl.append(s2); | |
1263 | t.truncate(cid, hoid, 0); | |
1264 | t.write(cid, hoid, 0x1000-1, bl.length(), bl); | |
1265 | cerr << "Write unaligned csum, stage 1" << std::endl; | |
11fdf7f2 | 1266 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1267 | ASSERT_EQ(r, 0); |
1268 | } | |
1269 | ||
1270 | bufferlist in, exp1, exp2, exp3; | |
1271 | exp1.append(s1); | |
1272 | exp2.append(s2); | |
1273 | exp3.append(s3); | |
11fdf7f2 | 1274 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1275 | ASSERT_EQ(1, r); |
1276 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1277 | in.clear(); | |
11fdf7f2 | 1278 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1279 | ASSERT_EQ(0x1000, r); |
1280 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1281 | ||
1282 | { | |
1283 | ObjectStore::Transaction t; | |
1284 | bufferlist bl; | |
1285 | bl.append(s3); | |
1286 | t.write(cid, hoid, 1, bl.length(), bl); | |
1287 | cerr << "Write unaligned csum, stage 2" << std::endl; | |
11fdf7f2 | 1288 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1289 | ASSERT_EQ(r, 0); |
1290 | } | |
1291 | in.clear(); | |
11fdf7f2 | 1292 | r = store->read(ch, hoid, 1, 2, in); |
7c673cae FG |
1293 | ASSERT_EQ(2, r); |
1294 | ASSERT_TRUE(bl_eq(exp3, in)); | |
1295 | in.clear(); | |
11fdf7f2 | 1296 | r = store->read(ch, hoid, 0x1000-1, 1, in); |
7c673cae FG |
1297 | ASSERT_EQ(1, r); |
1298 | ASSERT_TRUE(bl_eq(exp1, in)); | |
1299 | in.clear(); | |
11fdf7f2 | 1300 | r = store->read(ch, hoid, 0x1000, 0x1000, in); |
7c673cae FG |
1301 | ASSERT_EQ(0x1000, r); |
1302 | ASSERT_TRUE(bl_eq(exp2, in)); | |
1303 | ||
1304 | } | |
1305 | ||
1306 | { | |
1307 | ObjectStore::Transaction t; | |
1308 | t.remove(cid, hoid); | |
1309 | t.remove_collection(cid); | |
1310 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1311 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1312 | ASSERT_EQ(r, 0); |
1313 | } | |
1314 | } | |
1315 | ||
11fdf7f2 TL |
1316 | #if defined(WITH_BLUESTORE) |
1317 | ||
1911f103 TL |
1318 | TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { |
1319 | if(string(GetParam()) != "bluestore") | |
1320 | return; | |
1321 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
1322 | g_conf().apply_changes(nullptr); | |
1323 | StartDeferred(65536); | |
1324 | ||
1325 | int r; | |
1326 | coll_t cid; | |
1327 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
1328 | const PerfCounters* logger = store->get_perf_counters(); | |
1329 | auto ch = store->create_new_collection(cid); | |
1330 | { | |
1331 | ObjectStore::Transaction t; | |
1332 | t.create_collection(cid, 0); | |
1333 | cerr << "Creating collection " << cid << std::endl; | |
1334 | r = queue_transaction(store, ch, std::move(t)); | |
1335 | ASSERT_EQ(r, 0); | |
1336 | } | |
1337 | { | |
1338 | bool exists = store->exists(ch, hoid); | |
1339 | ASSERT_TRUE(!exists); | |
1340 | ||
1341 | ObjectStore::Transaction t; | |
1342 | t.touch(cid, hoid); | |
1343 | cerr << "Creating object " << hoid << std::endl; | |
1344 | r = queue_transaction(store, ch, std::move(t)); | |
1345 | ASSERT_EQ(r, 0); | |
1346 | ||
1347 | exists = store->exists(ch, hoid); | |
1348 | ASSERT_EQ(true, exists); | |
1349 | } | |
1350 | { | |
1351 | ObjectStore::Transaction t; | |
1352 | bufferlist bl, orig; | |
1353 | string s(4096, 'a'); | |
1354 | bl.append(s); | |
1355 | t.write(cid, hoid, 0x11000, bl.length(), bl); | |
1356 | cerr << "write1" << std::endl; | |
1357 | r = queue_transaction(store, ch, std::move(t)); | |
1358 | ASSERT_EQ(r, 0); | |
1359 | } | |
1360 | { | |
1361 | ObjectStore::Transaction t; | |
1362 | bufferlist bl, orig; | |
1363 | string s(4096 * 3, 'a'); | |
1364 | bl.append(s); | |
1365 | t.write(cid, hoid, 0x15000, bl.length(), bl); | |
1366 | cerr << "write2" << std::endl; | |
1367 | r = queue_transaction(store, ch, std::move(t)); | |
1368 | ASSERT_EQ(r, 0); | |
1369 | } | |
1370 | ASSERT_EQ(logger->get(l_bluestore_write_small), 2u); | |
1371 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 1u); | |
1372 | ||
1373 | { | |
1374 | ObjectStore::Transaction t; | |
1375 | bufferlist bl, orig; | |
1376 | string s(4096 * 2, 'a'); | |
1377 | bl.append(s); | |
1378 | t.write(cid, hoid, 0xe000, bl.length(), bl); | |
1379 | cerr << "write3" << std::endl; | |
1380 | r = queue_transaction(store, ch, std::move(t)); | |
1381 | ASSERT_EQ(r, 0); | |
1382 | } | |
1383 | ASSERT_EQ(logger->get(l_bluestore_write_small), 3u); | |
1384 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1385 | ||
1386 | ||
1387 | { | |
1388 | ObjectStore::Transaction t; | |
1389 | bufferlist bl, orig; | |
1390 | string s(4096, 'a'); | |
1391 | bl.append(s); | |
1392 | t.write(cid, hoid, 0xf000, bl.length(), bl); | |
1393 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1394 | cerr << "write3" << std::endl; | |
1395 | r = queue_transaction(store, ch, std::move(t)); | |
1396 | ASSERT_EQ(r, 0); | |
1397 | } | |
1398 | ASSERT_EQ(logger->get(l_bluestore_write_small), 5u); | |
1399 | ASSERT_EQ(logger->get(l_bluestore_write_small_unused), 2u); | |
1400 | { | |
1401 | ObjectStore::Transaction t; | |
1402 | t.remove(cid, hoid); | |
1403 | t.remove_collection(cid); | |
1404 | cerr << "Cleaning" << std::endl; | |
1405 | r = queue_transaction(store, ch, std::move(t)); | |
1406 | ASSERT_EQ(r, 0); | |
1407 | } | |
1408 | } | |
1409 | ||
1410 | ||
7c673cae FG |
1411 | TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { |
1412 | if(string(GetParam()) != "bluestore") | |
1413 | return; | |
1414 | StartDeferred(65536); | |
11fdf7f2 | 1415 | SetVal(g_conf(), "bluestore_compression_mode", "force"); |
eafe8130 | 1416 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); |
7c673cae | 1417 | // just a big number to disble gc |
11fdf7f2 TL |
1418 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "100000"); |
1419 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
1420 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
1421 | int r; |
1422 | ||
11fdf7f2 TL |
1423 | int poolid = 4373; |
1424 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
1425 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
1426 | string(), | |
1427 | 0, | |
1428 | poolid, | |
1429 | string())); | |
7c673cae FG |
1430 | ghobject_t hoid2 = hoid; |
1431 | hoid2.hobj.snap = 1; | |
1432 | { | |
11fdf7f2 TL |
1433 | auto ch = store->open_collection(cid); |
1434 | ASSERT_FALSE(ch); | |
7c673cae | 1435 | } |
11fdf7f2 | 1436 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1437 | { |
1438 | ObjectStore::Transaction t; | |
1439 | t.create_collection(cid, 0); | |
1440 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1441 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1442 | ASSERT_EQ(r, 0); |
1443 | } | |
1444 | { | |
11fdf7f2 | 1445 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1446 | ASSERT_TRUE(!exists); |
1447 | ||
1448 | ObjectStore::Transaction t; | |
1449 | t.touch(cid, hoid); | |
1450 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1451 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1452 | ASSERT_EQ(r, 0); |
1453 | ||
11fdf7f2 | 1454 | exists = store->exists(ch, hoid); |
7c673cae FG |
1455 | ASSERT_EQ(true, exists); |
1456 | } | |
1457 | { | |
1458 | struct store_statfs_t statfs; | |
1459 | int r = store->statfs(&statfs); | |
1460 | ASSERT_EQ(r, 0); | |
1461 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
1462 | ASSERT_EQ( 0u, statfs.data_stored); |
1463 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); | |
1464 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
1465 | ||
1466 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1467 | bool per_pool_omap; |
1468 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1469 | ASSERT_EQ(r, 0); |
1470 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
1471 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
1472 | ||
7c673cae | 1473 | //force fsck |
11fdf7f2 | 1474 | ch.reset(); |
7c673cae | 1475 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1476 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1477 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1478 | ch = store->open_collection(cid); |
7c673cae FG |
1479 | } |
1480 | { | |
1481 | ObjectStore::Transaction t; | |
1482 | bufferlist bl; | |
1483 | bl.append("abcde"); | |
1484 | t.write(cid, hoid, 0, 5, bl); | |
1485 | cerr << "Append 5 bytes" << std::endl; | |
11fdf7f2 | 1486 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1487 | ASSERT_EQ(r, 0); |
1488 | ||
1489 | struct store_statfs_t statfs; | |
1490 | int r = store->statfs(&statfs); | |
1491 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1492 | ASSERT_EQ(5, statfs.data_stored); |
7c673cae | 1493 | ASSERT_EQ(0x10000, statfs.allocated); |
11fdf7f2 TL |
1494 | ASSERT_EQ(0, statfs.data_compressed); |
1495 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1496 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1497 | ||
1498 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1499 | bool per_pool_omap; |
1500 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1501 | ASSERT_EQ(r, 0); |
1502 | ASSERT_EQ(5, statfs_pool.data_stored); | |
1503 | ASSERT_EQ(0x10000, statfs_pool.allocated); | |
1504 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1505 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1506 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1507 | ||
1508 | // accessing unknown pool | |
9f95a23c | 1509 | r = store->pool_statfs(poolid + 1, &statfs_pool, &per_pool_omap); |
11fdf7f2 TL |
1510 | ASSERT_EQ(r, 0); |
1511 | ASSERT_EQ(0, statfs_pool.data_stored); | |
1512 | ASSERT_EQ(0, statfs_pool.allocated); | |
1513 | ASSERT_EQ(0, statfs_pool.data_compressed); | |
1514 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1515 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
1516 | ||
7c673cae | 1517 | //force fsck |
11fdf7f2 | 1518 | ch.reset(); |
7c673cae | 1519 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1520 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1521 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1522 | ch = store->open_collection(cid); |
7c673cae FG |
1523 | } |
1524 | { | |
1525 | ObjectStore::Transaction t; | |
1526 | std::string s(0x30000, 'a'); | |
1527 | bufferlist bl; | |
1528 | bl.append(s); | |
1529 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1530 | cerr << "Append 0x30000 compressible bytes" << std::endl; | |
11fdf7f2 | 1531 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1532 | ASSERT_EQ(r, 0); |
1533 | ||
1534 | struct store_statfs_t statfs; | |
1535 | int r = store->statfs(&statfs); | |
1536 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1537 | ASSERT_EQ(0x30005, statfs.data_stored); |
7c673cae | 1538 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1539 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1540 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1541 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1542 | ||
1543 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1544 | bool per_pool_omap; |
1545 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1546 | ASSERT_EQ(r, 0); |
1547 | ASSERT_EQ(0x30005, statfs_pool.data_stored); | |
1548 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1549 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1550 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1551 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1552 | //force fsck |
11fdf7f2 | 1553 | ch.reset(); |
7c673cae | 1554 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1555 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1556 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1557 | ch = store->open_collection(cid); |
7c673cae FG |
1558 | } |
1559 | { | |
1560 | ObjectStore::Transaction t; | |
1561 | t.zero(cid, hoid, 1, 3); | |
1562 | t.zero(cid, hoid, 0x20000, 9); | |
1563 | cerr << "Punch hole at 1~3, 0x20000~9" << std::endl; | |
11fdf7f2 | 1564 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1565 | ASSERT_EQ(r, 0); |
1566 | ||
1567 | struct store_statfs_t statfs; | |
1568 | int r = store->statfs(&statfs); | |
1569 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1570 | ASSERT_EQ(0x30005 - 3 - 9, statfs.data_stored); |
7c673cae | 1571 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1572 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1573 | ASSERT_EQ(0x20000 - 9, statfs.data_compressed_original); | |
1574 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1575 | ||
1576 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1577 | bool per_pool_omap; |
1578 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1579 | ASSERT_EQ(r, 0); |
1580 | ASSERT_EQ(0x30005 - 3 - 9, statfs_pool.data_stored); | |
1581 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1582 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1583 | ASSERT_EQ(0x20000 - 9, statfs_pool.data_compressed_original); | |
1584 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1585 | //force fsck |
11fdf7f2 | 1586 | ch.reset(); |
7c673cae | 1587 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1588 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1589 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1590 | ch = store->open_collection(cid); |
7c673cae FG |
1591 | } |
1592 | { | |
1593 | ObjectStore::Transaction t; | |
1594 | std::string s(0x1000, 'b'); | |
1595 | bufferlist bl; | |
1596 | bl.append(s); | |
1597 | t.write(cid, hoid, 1, bl.length(), bl); | |
1598 | t.write(cid, hoid, 0x10001, bl.length(), bl); | |
1599 | cerr << "Overwrite first and second(compressible) extents" << std::endl; | |
11fdf7f2 | 1600 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1601 | ASSERT_EQ(r, 0); |
1602 | ||
1603 | struct store_statfs_t statfs; | |
1604 | int r = store->statfs(&statfs); | |
1605 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1606 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs.data_stored); |
7c673cae | 1607 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1608 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1609 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs.data_compressed_original); | |
1610 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); | |
1611 | ||
1612 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1613 | bool per_pool_omap; |
1614 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1615 | ASSERT_EQ(r, 0); |
1616 | ASSERT_EQ(0x30001 - 9 + 0x1000, statfs_pool.data_stored); | |
1617 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1618 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1619 | ASSERT_EQ(0x20000 - 9 - 0x1000, statfs_pool.data_compressed_original); | |
1620 | ASSERT_EQ(statfs_pool.data_compressed_allocated, 0x10000); | |
7c673cae | 1621 | //force fsck |
11fdf7f2 | 1622 | ch.reset(); |
7c673cae | 1623 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1624 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1625 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1626 | ch = store->open_collection(cid); |
7c673cae FG |
1627 | } |
1628 | { | |
1629 | ObjectStore::Transaction t; | |
1630 | std::string s(0x10000, 'c'); | |
1631 | bufferlist bl; | |
1632 | bl.append(s); | |
1633 | t.write(cid, hoid, 0x10000, bl.length(), bl); | |
1634 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
1635 | t.write(cid, hoid, 0x30000, bl.length(), bl); | |
1636 | cerr << "Overwrite compressed extent with 3 uncompressible ones" << std::endl; | |
11fdf7f2 | 1637 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1638 | ASSERT_EQ(r, 0); |
1639 | ||
1640 | struct store_statfs_t statfs; | |
1641 | int r = store->statfs(&statfs); | |
1642 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1643 | ASSERT_EQ(0x30000 + 0x1001, statfs.data_stored); |
7c673cae | 1644 | ASSERT_EQ(0x40000, statfs.allocated); |
11fdf7f2 TL |
1645 | ASSERT_LE(statfs.data_compressed, 0); |
1646 | ASSERT_EQ(0, statfs.data_compressed_original); | |
1647 | ASSERT_EQ(0, statfs.data_compressed_allocated); | |
1648 | ||
1649 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1650 | bool per_pool_omap; |
1651 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1652 | ASSERT_EQ(r, 0); |
1653 | ASSERT_EQ(0x30000 + 0x1001, statfs_pool.data_stored); | |
1654 | ASSERT_EQ(0x40000, statfs_pool.allocated); | |
1655 | ASSERT_LE(statfs_pool.data_compressed, 0); | |
1656 | ASSERT_EQ(0, statfs_pool.data_compressed_original); | |
1657 | ASSERT_EQ(0, statfs_pool.data_compressed_allocated); | |
7c673cae | 1658 | //force fsck |
11fdf7f2 | 1659 | ch.reset(); |
7c673cae | 1660 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1661 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1662 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1663 | ch = store->open_collection(cid); |
7c673cae FG |
1664 | } |
1665 | { | |
1666 | ObjectStore::Transaction t; | |
1667 | t.zero(cid, hoid, 0, 0x40000); | |
1668 | cerr << "Zero object" << std::endl; | |
11fdf7f2 | 1669 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1670 | ASSERT_EQ(r, 0); |
1671 | struct store_statfs_t statfs; | |
1672 | int r = store->statfs(&statfs); | |
1673 | ASSERT_EQ(r, 0); | |
1674 | ASSERT_EQ(0u, statfs.allocated); | |
11fdf7f2 TL |
1675 | ASSERT_EQ(0u, statfs.data_stored); |
1676 | ASSERT_EQ(0u, statfs.data_compressed_original); | |
1677 | ASSERT_EQ(0u, statfs.data_compressed); | |
1678 | ASSERT_EQ(0u, statfs.data_compressed_allocated); | |
1679 | ||
1680 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1681 | bool per_pool_omap; |
1682 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1683 | ASSERT_EQ(r, 0); |
1684 | ASSERT_EQ(0u, statfs_pool.allocated); | |
1685 | ASSERT_EQ(0u, statfs_pool.data_stored); | |
1686 | ASSERT_EQ(0u, statfs_pool.data_compressed_original); | |
1687 | ASSERT_EQ(0u, statfs_pool.data_compressed); | |
1688 | ASSERT_EQ(0u, statfs_pool.data_compressed_allocated); | |
7c673cae | 1689 | //force fsck |
11fdf7f2 | 1690 | ch.reset(); |
7c673cae | 1691 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1692 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1693 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1694 | ch = store->open_collection(cid); |
7c673cae FG |
1695 | } |
1696 | { | |
1697 | ObjectStore::Transaction t; | |
1698 | std::string s(0x10000, 'c'); | |
1699 | bufferlist bl; | |
1700 | bl.append(s); | |
1701 | bl.append(s); | |
1702 | bl.append(s); | |
1703 | bl.append(s.substr(0, 0x10000-2)); | |
1704 | t.write(cid, hoid, 0, bl.length(), bl); | |
1705 | cerr << "Yet another compressible write" << std::endl; | |
11fdf7f2 | 1706 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1707 | ASSERT_EQ(r, 0); |
1708 | struct store_statfs_t statfs; | |
1709 | r = store->statfs(&statfs); | |
1710 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1711 | ASSERT_EQ(0x40000 - 2, statfs.data_stored); |
7c673cae | 1712 | ASSERT_EQ(0x30000, statfs.allocated); |
11fdf7f2 TL |
1713 | ASSERT_LE(statfs.data_compressed, 0x10000); |
1714 | ASSERT_EQ(0x20000, statfs.data_compressed_original); | |
1715 | ASSERT_EQ(0x10000, statfs.data_compressed_allocated); | |
1716 | ||
1717 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1718 | bool per_pool_omap; |
1719 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1720 | ASSERT_EQ(r, 0); |
1721 | ASSERT_EQ(0x40000 - 2, statfs_pool.data_stored); | |
1722 | ASSERT_EQ(0x30000, statfs_pool.allocated); | |
1723 | ASSERT_LE(statfs_pool.data_compressed, 0x10000); | |
1724 | ASSERT_EQ(0x20000, statfs_pool.data_compressed_original); | |
1725 | ASSERT_EQ(0x10000, statfs_pool.data_compressed_allocated); | |
7c673cae | 1726 | //force fsck |
11fdf7f2 | 1727 | ch.reset(); |
7c673cae | 1728 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 1729 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 1730 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 1731 | ch = store->open_collection(cid); |
7c673cae FG |
1732 | } |
1733 | { | |
1734 | struct store_statfs_t statfs; | |
1735 | r = store->statfs(&statfs); | |
1736 | ASSERT_EQ(r, 0); | |
1737 | ||
11fdf7f2 | 1738 | struct store_statfs_t statfs_pool; |
9f95a23c TL |
1739 | bool per_pool_omap; |
1740 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1741 | ASSERT_EQ(r, 0); |
1742 | ||
7c673cae FG |
1743 | ObjectStore::Transaction t; |
1744 | t.clone(cid, hoid, hoid2); | |
1745 | cerr << "Clone compressed objecte" << std::endl; | |
11fdf7f2 | 1746 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1747 | ASSERT_EQ(r, 0); |
1748 | struct store_statfs_t statfs2; | |
1749 | r = store->statfs(&statfs2); | |
1750 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1751 | ASSERT_GT(statfs2.data_stored, statfs.data_stored); |
7c673cae | 1752 | ASSERT_EQ(statfs2.allocated, statfs.allocated); |
11fdf7f2 TL |
1753 | ASSERT_GT(statfs2.data_compressed, statfs.data_compressed); |
1754 | ASSERT_GT(statfs2.data_compressed_original, statfs.data_compressed_original); | |
1755 | ASSERT_EQ(statfs2.data_compressed_allocated, statfs.data_compressed_allocated); | |
1756 | ||
1757 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1758 | r = store->pool_statfs(poolid, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1759 | ASSERT_EQ(r, 0); |
1760 | ASSERT_GT(statfs2_pool.data_stored, statfs_pool.data_stored); | |
1761 | ASSERT_EQ(statfs2_pool.allocated, statfs_pool.allocated); | |
1762 | ASSERT_GT(statfs2_pool.data_compressed, statfs_pool.data_compressed); | |
1763 | ASSERT_GT(statfs2_pool.data_compressed_original, | |
1764 | statfs_pool.data_compressed_original); | |
1765 | ASSERT_EQ(statfs2_pool.data_compressed_allocated, | |
1766 | statfs_pool.data_compressed_allocated); | |
1767 | } | |
1768 | ||
1769 | { | |
1770 | // verify no | |
1771 | auto poolid2 = poolid + 1; | |
1772 | coll_t cid2 = coll_t(spg_t(pg_t(20, poolid2), shard_id_t::NO_SHARD)); | |
1773 | ghobject_t hoid(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
1774 | string(), | |
1775 | 0, | |
1776 | poolid2, | |
1777 | string())); | |
1778 | auto ch = store->create_new_collection(cid2); | |
1779 | ||
1780 | { | |
1781 | ||
1782 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
1783 | bool per_pool_omap; |
1784 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
1785 | ASSERT_EQ(r, 0); |
1786 | ||
1787 | cerr << "Creating second collection " << cid2 << std::endl; | |
1788 | ObjectStore::Transaction t; | |
1789 | t.create_collection(cid2, 0); | |
1790 | r = queue_transaction(store, ch, std::move(t)); | |
1791 | ASSERT_EQ(r, 0); | |
1792 | ||
1793 | t = ObjectStore::Transaction(); | |
1794 | bufferlist bl; | |
1795 | bl.append("abcde"); | |
1796 | t.write(cid2, hoid, 0, 5, bl); | |
1797 | r = queue_transaction(store, ch, std::move(t)); | |
1798 | ASSERT_EQ(r, 0); | |
1799 | ||
1800 | struct store_statfs_t statfs2_pool; | |
9f95a23c | 1801 | r = store->pool_statfs(poolid2, &statfs2_pool, &per_pool_omap); |
11fdf7f2 TL |
1802 | ASSERT_EQ(r, 0); |
1803 | ASSERT_EQ(5, statfs2_pool.data_stored); | |
1804 | ASSERT_EQ(0x10000, statfs2_pool.allocated); | |
1805 | ASSERT_EQ(0, statfs2_pool.data_compressed); | |
1806 | ASSERT_EQ(0, statfs2_pool.data_compressed_original); | |
1807 | ASSERT_EQ(0, statfs2_pool.data_compressed_allocated); | |
1808 | ||
1809 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 1810 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1811 | ASSERT_EQ(r, 0); |
1812 | // adjust 'available' since it has changed | |
1813 | statfs1_pool_again.available = statfs1_pool.available; | |
1814 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
1815 | ||
1816 | t = ObjectStore::Transaction(); | |
1817 | t.remove(cid2, hoid); | |
1818 | t.remove_collection(cid2); | |
1819 | cerr << "Cleaning" << std::endl; | |
1820 | r = queue_transaction(store, ch, std::move(t)); | |
1821 | ASSERT_EQ(r, 0); | |
1822 | } | |
1823 | } | |
1824 | ||
1825 | { | |
1826 | // verify ops on temporary object | |
1827 | ||
1828 | auto poolid3 = poolid + 2; | |
1829 | coll_t cid3 = coll_t(spg_t(pg_t(20, poolid3), shard_id_t::NO_SHARD)); | |
1830 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP), | |
1831 | string(), | |
1832 | 0, | |
1833 | poolid3, | |
1834 | string())); | |
1835 | ghobject_t hoid3_temp; | |
1836 | hoid3_temp.hobj = hoid3.hobj.make_temp_hobject("Object 3 temp"); | |
1837 | auto ch3 = store->create_new_collection(cid3); | |
1838 | { | |
1839 | struct store_statfs_t statfs1_pool; | |
9f95a23c TL |
1840 | bool per_pool_omap; |
1841 | int r = store->pool_statfs(poolid, &statfs1_pool, &per_pool_omap); | |
11fdf7f2 TL |
1842 | ASSERT_EQ(r, 0); |
1843 | ||
1844 | cerr << "Creating third collection " << cid3 << std::endl; | |
1845 | ObjectStore::Transaction t; | |
1846 | t.create_collection(cid3, 0); | |
1847 | r = queue_transaction(store, ch3, std::move(t)); | |
1848 | ASSERT_EQ(r, 0); | |
1849 | ||
1850 | t = ObjectStore::Transaction(); | |
1851 | bufferlist bl; | |
1852 | bl.append("abcde"); | |
1853 | t.write(cid3, hoid3_temp, 0, 5, bl); | |
1854 | r = queue_transaction(store, ch3, std::move(t)); | |
1855 | ASSERT_EQ(r, 0); | |
1856 | ||
1857 | struct store_statfs_t statfs3_pool; | |
9f95a23c | 1858 | r = store->pool_statfs(poolid3, &statfs3_pool, &per_pool_omap); |
11fdf7f2 TL |
1859 | ASSERT_EQ(r, 0); |
1860 | ASSERT_EQ(5, statfs3_pool.data_stored); | |
1861 | ASSERT_EQ(0x10000, statfs3_pool.allocated); | |
1862 | ASSERT_EQ(0, statfs3_pool.data_compressed); | |
1863 | ASSERT_EQ(0, statfs3_pool.data_compressed_original); | |
1864 | ASSERT_EQ(0, statfs3_pool.data_compressed_allocated); | |
1865 | ||
1866 | struct store_statfs_t statfs1_pool_again; | |
9f95a23c | 1867 | r = store->pool_statfs(poolid, &statfs1_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1868 | ASSERT_EQ(r, 0); |
1869 | // adjust 'available' since it has changed | |
1870 | statfs1_pool_again.available = statfs1_pool.available; | |
1871 | ASSERT_EQ(statfs1_pool_again, statfs1_pool); | |
1872 | ||
1873 | //force fsck | |
1874 | ch.reset(); | |
1875 | ch3.reset(); | |
1876 | EXPECT_EQ(store->umount(), 0); | |
1877 | EXPECT_EQ(store->mount(), 0); | |
1878 | ch = store->open_collection(cid); | |
1879 | ch3 = store->open_collection(cid3); | |
1880 | ||
1881 | t = ObjectStore::Transaction(); | |
1882 | t.collection_move_rename( | |
1883 | cid3, hoid3_temp, | |
1884 | cid3, hoid3); | |
1885 | r = queue_transaction(store, ch3, std::move(t)); | |
1886 | ASSERT_EQ(r, 0); | |
1887 | ||
1888 | struct store_statfs_t statfs3_pool_again; | |
9f95a23c | 1889 | r = store->pool_statfs(poolid3, &statfs3_pool_again, &per_pool_omap); |
11fdf7f2 TL |
1890 | ASSERT_EQ(r, 0); |
1891 | ASSERT_EQ(statfs3_pool_again, statfs3_pool); | |
1892 | ||
1893 | //force fsck | |
1894 | ch.reset(); | |
1895 | ch3.reset(); | |
1896 | EXPECT_EQ(store->umount(), 0); | |
1897 | EXPECT_EQ(store->mount(), 0); | |
1898 | ch = store->open_collection(cid); | |
1899 | ch3 = store->open_collection(cid3); | |
1900 | ||
1901 | t = ObjectStore::Transaction(); | |
1902 | t.remove(cid3, hoid3); | |
1903 | t.remove_collection(cid3); | |
1904 | cerr << "Cleaning" << std::endl; | |
1905 | r = queue_transaction(store, ch3, std::move(t)); | |
1906 | ASSERT_EQ(r, 0); | |
1907 | } | |
7c673cae FG |
1908 | } |
1909 | ||
1910 | { | |
1911 | ObjectStore::Transaction t; | |
1912 | t.remove(cid, hoid); | |
1913 | t.remove(cid, hoid2); | |
1914 | t.remove_collection(cid); | |
1915 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 1916 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1917 | ASSERT_EQ(r, 0); |
1918 | ||
1919 | struct store_statfs_t statfs; | |
1920 | r = store->statfs(&statfs); | |
1921 | ASSERT_EQ(r, 0); | |
1922 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
1923 | ASSERT_EQ( 0u, statfs.data_stored); |
1924 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
1925 | ASSERT_EQ( 0u, statfs.data_compressed); | |
1926 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
1927 | ||
1928 | struct store_statfs_t statfs_pool; | |
9f95a23c TL |
1929 | bool per_pool_omap; |
1930 | r = store->pool_statfs(poolid, &statfs_pool, &per_pool_omap); | |
11fdf7f2 TL |
1931 | ASSERT_EQ(r, 0); |
1932 | ASSERT_EQ( 0u, statfs_pool.allocated); | |
1933 | ASSERT_EQ( 0u, statfs_pool.data_stored); | |
1934 | ASSERT_EQ( 0u, statfs_pool.data_compressed_original); | |
1935 | ASSERT_EQ( 0u, statfs_pool.data_compressed); | |
1936 | ASSERT_EQ( 0u, statfs_pool.data_compressed_allocated); | |
1937 | } | |
7c673cae FG |
1938 | } |
1939 | ||
1940 | TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) { | |
1941 | if(string(GetParam()) != "bluestore") | |
1942 | return; | |
1943 | StartDeferred(0x10000); | |
1944 | ||
7c673cae FG |
1945 | int r; |
1946 | coll_t cid; | |
1947 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 1948 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
1949 | { |
1950 | ObjectStore::Transaction t; | |
1951 | t.create_collection(cid, 0); | |
1952 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 1953 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1954 | ASSERT_EQ(r, 0); |
1955 | } | |
1956 | { | |
11fdf7f2 | 1957 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
1958 | ASSERT_TRUE(!exists); |
1959 | ||
1960 | ObjectStore::Transaction t; | |
1961 | t.touch(cid, hoid); | |
1962 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 1963 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1964 | ASSERT_EQ(r, 0); |
1965 | ||
11fdf7f2 | 1966 | exists = store->exists(ch, hoid); |
7c673cae FG |
1967 | ASSERT_EQ(true, exists); |
1968 | } | |
1969 | { | |
1970 | struct store_statfs_t statfs; | |
1971 | int r = store->statfs(&statfs); | |
1972 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1973 | ASSERT_EQ(g_conf()->bluestore_block_size, statfs.total); |
7c673cae | 1974 | ASSERT_EQ(0u, statfs.allocated); |
11fdf7f2 TL |
1975 | ASSERT_EQ(0u, statfs.data_stored); |
1976 | ASSERT_TRUE(statfs.available > 0u && statfs.available < g_conf()->bluestore_block_size); | |
7c673cae FG |
1977 | } |
1978 | std::string data; | |
1979 | data.resize(0x10000 * 3); | |
1980 | { | |
1981 | ObjectStore::Transaction t; | |
1982 | for(size_t i = 0;i < data.size(); i++) | |
1983 | data[i] = i / 256 + 1; | |
1984 | bufferlist bl, newdata; | |
1985 | bl.append(data); | |
1986 | t.write(cid, hoid, 0, bl.length(), bl); | |
1987 | t.zero(cid, hoid, 0x10000, 0x10000); | |
1988 | cerr << "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl; | |
11fdf7f2 | 1989 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
1990 | ASSERT_EQ(r, 0); |
1991 | ||
1992 | struct store_statfs_t statfs; | |
1993 | int r = store->statfs(&statfs); | |
1994 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 1995 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae FG |
1996 | ASSERT_EQ(0x20000, statfs.allocated); |
1997 | ||
11fdf7f2 | 1998 | r = store->read(ch, hoid, 0, data.size(), newdata); |
7c673cae FG |
1999 | ASSERT_EQ(r, (int)data.size()); |
2000 | { | |
2001 | bufferlist expected; | |
2002 | expected.append(data.substr(0, 0x10000)); | |
2003 | expected.append(string(0x10000, 0)); | |
2004 | expected.append(data.substr(0x20000, 0x10000)); | |
2005 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2006 | } | |
2007 | newdata.clear(); | |
2008 | ||
11fdf7f2 | 2009 | r = store->read(ch, hoid, 1, data.size()-2, newdata); |
7c673cae FG |
2010 | ASSERT_EQ(r, (int)data.size()-2); |
2011 | { | |
2012 | bufferlist expected; | |
2013 | expected.append(data.substr(1, 0x10000-1)); | |
2014 | expected.append(string(0x10000, 0)); | |
2015 | expected.append(data.substr(0x20000, 0x10000 - 1)); | |
2016 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2017 | } | |
2018 | newdata.clear(); | |
2019 | } | |
2020 | //force fsck | |
11fdf7f2 | 2021 | ch.reset(); |
7c673cae | 2022 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2023 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2024 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2025 | ch = store->open_collection(cid); |
7c673cae FG |
2026 | |
2027 | { | |
2028 | ObjectStore::Transaction t; | |
2029 | std::string data2(3, 'b'); | |
2030 | bufferlist bl, newdata; | |
2031 | bl.append(data2); | |
2032 | t.write(cid, hoid, 0x20000, bl.length(), bl); | |
2033 | cerr << "Write 3 bytes after the hole" << std::endl; | |
11fdf7f2 | 2034 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2035 | ASSERT_EQ(r, 0); |
2036 | ||
2037 | struct store_statfs_t statfs; | |
2038 | int r = store->statfs(&statfs); | |
2039 | ASSERT_EQ(r, 0); | |
2040 | ASSERT_EQ(0x20000, statfs.allocated); | |
11fdf7f2 | 2041 | ASSERT_EQ(0x20000, statfs.data_stored); |
7c673cae | 2042 | |
11fdf7f2 | 2043 | r = store->read(ch, hoid, 0x20000-1, 21, newdata); |
7c673cae FG |
2044 | ASSERT_EQ(r, (int)21); |
2045 | { | |
2046 | bufferlist expected; | |
2047 | expected.append(string(0x1, 0)); | |
2048 | expected.append(string(data2)); | |
2049 | expected.append(data.substr(0x20003, 21-4)); | |
2050 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2051 | } | |
2052 | newdata.clear(); | |
2053 | } | |
2054 | //force fsck | |
11fdf7f2 | 2055 | ch.reset(); |
7c673cae | 2056 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2057 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2058 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2059 | ch = store->open_collection(cid); |
7c673cae FG |
2060 | |
2061 | { | |
2062 | ObjectStore::Transaction t; | |
2063 | std::string data2(3, 'a'); | |
2064 | bufferlist bl, newdata; | |
2065 | bl.append(data2); | |
2066 | t.write(cid, hoid, 0x10000+1, bl.length(), bl); | |
2067 | cerr << "Write 3 bytes to the hole" << std::endl; | |
11fdf7f2 | 2068 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2069 | ASSERT_EQ(r, 0); |
2070 | ||
2071 | struct store_statfs_t statfs; | |
2072 | int r = store->statfs(&statfs); | |
2073 | ASSERT_EQ(r, 0); | |
2074 | ASSERT_EQ(0x30000, statfs.allocated); | |
11fdf7f2 | 2075 | ASSERT_EQ(0x20003, statfs.data_stored); |
7c673cae | 2076 | |
11fdf7f2 | 2077 | r = store->read(ch, hoid, 0x10000-1, 0x10000+22, newdata); |
7c673cae FG |
2078 | ASSERT_EQ(r, (int)0x10000+22); |
2079 | { | |
2080 | bufferlist expected; | |
2081 | expected.append(data.substr(0x10000-1, 1)); | |
2082 | expected.append(string(0x1, 0)); | |
2083 | expected.append(data2); | |
2084 | expected.append(string(0x10000-4, 0)); | |
2085 | expected.append(string(0x3, 'b')); | |
2086 | expected.append(data.substr(0x20004, 21-3)); | |
2087 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2088 | } | |
2089 | newdata.clear(); | |
2090 | } | |
2091 | { | |
2092 | ObjectStore::Transaction t; | |
2093 | bufferlist bl, newdata; | |
2094 | bl.append(string(0x30000, 'c')); | |
2095 | t.write(cid, hoid, 0, 0x30000, bl); | |
2096 | t.zero(cid, hoid, 0, 0x10000); | |
2097 | t.zero(cid, hoid, 0x20000, 0x10000); | |
11fdf7f2 TL |
2098 | cerr << "Rewrite an object and create two holes at the beginning and the end" << std::endl; |
2099 | r = queue_transaction(store, ch, std::move(t)); | |
7c673cae FG |
2100 | ASSERT_EQ(r, 0); |
2101 | ||
2102 | struct store_statfs_t statfs; | |
2103 | int r = store->statfs(&statfs); | |
2104 | ASSERT_EQ(r, 0); | |
2105 | ASSERT_EQ(0x10000, statfs.allocated); | |
11fdf7f2 | 2106 | ASSERT_EQ(0x10000, statfs.data_stored); |
7c673cae | 2107 | |
11fdf7f2 | 2108 | r = store->read(ch, hoid, 0, 0x30000, newdata); |
7c673cae FG |
2109 | ASSERT_EQ(r, (int)0x30000); |
2110 | { | |
2111 | bufferlist expected; | |
2112 | expected.append(string(0x10000, 0)); | |
2113 | expected.append(string(0x10000, 'c')); | |
2114 | expected.append(string(0x10000, 0)); | |
2115 | ASSERT_TRUE(bl_eq(expected, newdata)); | |
2116 | } | |
2117 | newdata.clear(); | |
2118 | } | |
2119 | ||
2120 | //force fsck | |
11fdf7f2 | 2121 | ch.reset(); |
7c673cae | 2122 | EXPECT_EQ(store->umount(), 0); |
11fdf7f2 | 2123 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly |
7c673cae | 2124 | EXPECT_EQ(store->mount(), 0); |
11fdf7f2 | 2125 | ch = store->open_collection(cid); |
7c673cae FG |
2126 | |
2127 | { | |
2128 | ObjectStore::Transaction t; | |
2129 | t.remove(cid, hoid); | |
2130 | t.remove_collection(cid); | |
2131 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2132 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2133 | ASSERT_EQ(r, 0); |
2134 | ||
2135 | struct store_statfs_t statfs; | |
2136 | r = store->statfs(&statfs); | |
2137 | ASSERT_EQ(r, 0); | |
2138 | ASSERT_EQ( 0u, statfs.allocated); | |
11fdf7f2 TL |
2139 | ASSERT_EQ( 0u, statfs.data_stored); |
2140 | ASSERT_EQ( 0u, statfs.data_compressed_original); | |
2141 | ASSERT_EQ( 0u, statfs.data_compressed); | |
2142 | ASSERT_EQ( 0u, statfs.data_compressed_allocated); | |
7c673cae FG |
2143 | } |
2144 | } | |
2145 | #endif | |
2146 | ||
2147 | TEST_P(StoreTest, ManySmallWrite) { | |
7c673cae FG |
2148 | int r; |
2149 | coll_t cid; | |
2150 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2151 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2152 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2153 | { |
2154 | ObjectStore::Transaction t; | |
2155 | t.create_collection(cid, 0); | |
2156 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2157 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2158 | ASSERT_EQ(r, 0); |
2159 | } | |
2160 | bufferlist bl; | |
2161 | bufferptr bp(4096); | |
2162 | bp.zero(); | |
2163 | bl.append(bp); | |
2164 | for (int i=0; i<100; ++i) { | |
2165 | ObjectStore::Transaction t; | |
2166 | t.write(cid, a, i*4096, 4096, bl, 0); | |
11fdf7f2 | 2167 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2168 | ASSERT_EQ(r, 0); |
2169 | } | |
2170 | for (int i=0; i<100; ++i) { | |
2171 | ObjectStore::Transaction t; | |
2172 | t.write(cid, b, (rand() % 1024)*4096, 4096, bl, 0); | |
11fdf7f2 | 2173 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2174 | ASSERT_EQ(r, 0); |
2175 | } | |
2176 | { | |
2177 | ObjectStore::Transaction t; | |
2178 | t.remove(cid, a); | |
2179 | t.remove(cid, b); | |
2180 | t.remove_collection(cid); | |
2181 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2182 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2183 | ASSERT_EQ(r, 0); |
2184 | } | |
2185 | } | |
2186 | ||
2187 | TEST_P(StoreTest, MultiSmallWriteSameBlock) { | |
7c673cae FG |
2188 | int r; |
2189 | coll_t cid; | |
2190 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2191 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2192 | { |
2193 | ObjectStore::Transaction t; | |
2194 | t.create_collection(cid, 0); | |
2195 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2196 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2197 | ASSERT_EQ(r, 0); |
2198 | } | |
2199 | bufferlist bl; | |
2200 | bl.append("short"); | |
2201 | C_SaferCond c, d; | |
2202 | // touch same block in both same transaction, tls, and pipelined txns | |
2203 | { | |
2204 | ObjectStore::Transaction t, u; | |
2205 | t.write(cid, a, 0, 5, bl, 0); | |
2206 | t.write(cid, a, 5, 5, bl, 0); | |
2207 | t.write(cid, a, 4094, 5, bl, 0); | |
2208 | t.write(cid, a, 9000, 5, bl, 0); | |
2209 | u.write(cid, a, 10, 5, bl, 0); | |
2210 | u.write(cid, a, 7000, 5, bl, 0); | |
11fdf7f2 | 2211 | t.register_on_commit(&c); |
7c673cae | 2212 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2213 | store->queue_transactions(ch, v); |
7c673cae FG |
2214 | } |
2215 | { | |
2216 | ObjectStore::Transaction t, u; | |
2217 | t.write(cid, a, 40, 5, bl, 0); | |
2218 | t.write(cid, a, 45, 5, bl, 0); | |
2219 | t.write(cid, a, 4094, 5, bl, 0); | |
2220 | t.write(cid, a, 6000, 5, bl, 0); | |
2221 | u.write(cid, a, 610, 5, bl, 0); | |
2222 | u.write(cid, a, 11000, 5, bl, 0); | |
11fdf7f2 | 2223 | t.register_on_commit(&d); |
7c673cae | 2224 | vector<ObjectStore::Transaction> v = {t, u}; |
11fdf7f2 | 2225 | store->queue_transactions(ch, v); |
7c673cae FG |
2226 | } |
2227 | c.wait(); | |
2228 | d.wait(); | |
2229 | { | |
2230 | bufferlist bl2; | |
11fdf7f2 | 2231 | r = store->read(ch, a, 0, 16000, bl2); |
7c673cae FG |
2232 | ASSERT_GE(r, 0); |
2233 | } | |
2234 | { | |
2235 | ObjectStore::Transaction t; | |
2236 | t.remove(cid, a); | |
2237 | t.remove_collection(cid); | |
2238 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2239 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2240 | ASSERT_EQ(r, 0); |
2241 | } | |
2242 | } | |
2243 | ||
2244 | TEST_P(StoreTest, SmallSkipFront) { | |
7c673cae FG |
2245 | int r; |
2246 | coll_t cid; | |
2247 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2248 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2249 | { |
2250 | ObjectStore::Transaction t; | |
2251 | t.create_collection(cid, 0); | |
2252 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2253 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2254 | ASSERT_EQ(r, 0); |
2255 | } | |
2256 | { | |
2257 | ObjectStore::Transaction t; | |
2258 | t.touch(cid, a); | |
2259 | t.truncate(cid, a, 3000); | |
11fdf7f2 | 2260 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2261 | ASSERT_EQ(r, 0); |
2262 | } | |
2263 | { | |
2264 | bufferlist bl; | |
2265 | bufferptr bp(4096); | |
2266 | memset(bp.c_str(), 1, 4096); | |
2267 | bl.append(bp); | |
2268 | ObjectStore::Transaction t; | |
2269 | t.write(cid, a, 4096, 4096, bl); | |
11fdf7f2 | 2270 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2271 | ASSERT_EQ(r, 0); |
2272 | } | |
2273 | { | |
2274 | bufferlist bl; | |
11fdf7f2 | 2275 | ASSERT_EQ(8192, store->read(ch, a, 0, 8192, bl)); |
7c673cae FG |
2276 | for (unsigned i=0; i<4096; ++i) |
2277 | ASSERT_EQ(0, bl[i]); | |
2278 | for (unsigned i=4096; i<8192; ++i) | |
2279 | ASSERT_EQ(1, bl[i]); | |
2280 | } | |
2281 | { | |
2282 | ObjectStore::Transaction t; | |
2283 | t.remove(cid, a); | |
2284 | t.remove_collection(cid); | |
2285 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2286 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2287 | ASSERT_EQ(r, 0); |
2288 | } | |
2289 | } | |
2290 | ||
2291 | TEST_P(StoreTest, AppendDeferredVsTailCache) { | |
7c673cae FG |
2292 | int r; |
2293 | coll_t cid; | |
2294 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
11fdf7f2 | 2295 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2296 | { |
2297 | ObjectStore::Transaction t; | |
2298 | t.create_collection(cid, 0); | |
2299 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2300 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2301 | ASSERT_EQ(r, 0); |
2302 | } | |
11fdf7f2 | 2303 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2304 | unsigned size = min_alloc / 3; |
2305 | bufferptr bpa(size); | |
2306 | memset(bpa.c_str(), 1, bpa.length()); | |
2307 | bufferlist bla; | |
2308 | bla.append(bpa); | |
2309 | { | |
2310 | ObjectStore::Transaction t; | |
2311 | t.write(cid, a, 0, bla.length(), bla, 0); | |
11fdf7f2 | 2312 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2313 | ASSERT_EQ(r, 0); |
2314 | } | |
2315 | ||
2316 | // force cached tail to clear ... | |
2317 | { | |
11fdf7f2 | 2318 | ch.reset(); |
7c673cae FG |
2319 | int r = store->umount(); |
2320 | ASSERT_EQ(0, r); | |
2321 | r = store->mount(); | |
2322 | ASSERT_EQ(0, r); | |
11fdf7f2 | 2323 | ch = store->open_collection(cid); |
7c673cae FG |
2324 | } |
2325 | ||
2326 | bufferptr bpb(size); | |
2327 | memset(bpb.c_str(), 2, bpb.length()); | |
2328 | bufferlist blb; | |
2329 | blb.append(bpb); | |
2330 | { | |
2331 | ObjectStore::Transaction t; | |
2332 | t.write(cid, a, bla.length(), blb.length(), blb, 0); | |
11fdf7f2 | 2333 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2334 | ASSERT_EQ(r, 0); |
2335 | } | |
2336 | bufferptr bpc(size); | |
2337 | memset(bpc.c_str(), 3, bpc.length()); | |
2338 | bufferlist blc; | |
2339 | blc.append(bpc); | |
2340 | { | |
2341 | ObjectStore::Transaction t; | |
2342 | t.write(cid, a, bla.length() + blb.length(), blc.length(), blc, 0); | |
11fdf7f2 | 2343 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2344 | ASSERT_EQ(r, 0); |
2345 | } | |
2346 | bufferlist final; | |
2347 | final.append(bla); | |
2348 | final.append(blb); | |
2349 | final.append(blc); | |
2350 | bufferlist actual; | |
2351 | { | |
2352 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2353 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2354 | ASSERT_TRUE(bl_eq(final, actual)); |
2355 | } | |
2356 | { | |
2357 | ObjectStore::Transaction t; | |
2358 | t.remove(cid, a); | |
2359 | t.remove_collection(cid); | |
2360 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2361 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2362 | ASSERT_EQ(r, 0); |
2363 | } | |
7c673cae FG |
2364 | } |
2365 | ||
2366 | TEST_P(StoreTest, AppendZeroTrailingSharedBlock) { | |
7c673cae FG |
2367 | int r; |
2368 | coll_t cid; | |
2369 | ghobject_t a(hobject_t(sobject_t("fooo", CEPH_NOSNAP))); | |
2370 | ghobject_t b = a; | |
2371 | b.hobj.snap = 1; | |
11fdf7f2 | 2372 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2373 | { |
2374 | ObjectStore::Transaction t; | |
2375 | t.create_collection(cid, 0); | |
2376 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2377 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2378 | ASSERT_EQ(r, 0); |
2379 | } | |
11fdf7f2 | 2380 | unsigned min_alloc = g_conf()->bluestore_min_alloc_size; |
7c673cae FG |
2381 | unsigned size = min_alloc / 3; |
2382 | bufferptr bpa(size); | |
2383 | memset(bpa.c_str(), 1, bpa.length()); | |
2384 | bufferlist bla; | |
2385 | bla.append(bpa); | |
2386 | // make sure there is some trailing gunk in the last block | |
2387 | { | |
2388 | bufferlist bt; | |
2389 | bt.append(bla); | |
2390 | bt.append("BADBADBADBAD"); | |
2391 | ObjectStore::Transaction t; | |
2392 | t.write(cid, a, 0, bt.length(), bt, 0); | |
11fdf7f2 | 2393 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2394 | ASSERT_EQ(r, 0); |
2395 | } | |
2396 | { | |
2397 | ObjectStore::Transaction t; | |
2398 | t.truncate(cid, a, size); | |
11fdf7f2 | 2399 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2400 | ASSERT_EQ(r, 0); |
2401 | } | |
2402 | ||
2403 | // clone | |
2404 | { | |
2405 | ObjectStore::Transaction t; | |
2406 | t.clone(cid, a, b); | |
11fdf7f2 | 2407 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2408 | ASSERT_EQ(r, 0); |
2409 | } | |
2410 | ||
2411 | // append with implicit zeroing | |
2412 | bufferptr bpb(size); | |
2413 | memset(bpb.c_str(), 2, bpb.length()); | |
2414 | bufferlist blb; | |
2415 | blb.append(bpb); | |
2416 | { | |
2417 | ObjectStore::Transaction t; | |
2418 | t.write(cid, a, min_alloc * 3, blb.length(), blb, 0); | |
11fdf7f2 | 2419 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2420 | ASSERT_EQ(r, 0); |
2421 | } | |
2422 | bufferlist final; | |
2423 | final.append(bla); | |
2424 | bufferlist zeros; | |
2425 | zeros.append_zero(min_alloc * 3 - size); | |
2426 | final.append(zeros); | |
2427 | final.append(blb); | |
2428 | bufferlist actual; | |
2429 | { | |
2430 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 2431 | store->read(ch, a, 0, final.length(), actual)); |
7c673cae FG |
2432 | final.hexdump(cout); |
2433 | actual.hexdump(cout); | |
2434 | ASSERT_TRUE(bl_eq(final, actual)); | |
2435 | } | |
2436 | { | |
2437 | ObjectStore::Transaction t; | |
2438 | t.remove(cid, a); | |
2439 | t.remove(cid, b); | |
2440 | t.remove_collection(cid); | |
2441 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2442 | r = store->queue_transaction(ch, std::move(t)); |
7c673cae FG |
2443 | ASSERT_EQ(r, 0); |
2444 | } | |
2445 | } | |
2446 | ||
2447 | TEST_P(StoreTest, SmallSequentialUnaligned) { | |
7c673cae FG |
2448 | int r; |
2449 | coll_t cid; | |
2450 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2451 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2452 | { |
2453 | ObjectStore::Transaction t; | |
2454 | t.create_collection(cid, 0); | |
2455 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2456 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2457 | ASSERT_EQ(r, 0); |
2458 | } | |
2459 | bufferlist bl; | |
2460 | int len = 1000; | |
2461 | bufferptr bp(len); | |
2462 | bp.zero(); | |
2463 | bl.append(bp); | |
2464 | for (int i=0; i<1000; ++i) { | |
2465 | ObjectStore::Transaction t; | |
2466 | t.write(cid, a, i*len, len, bl, 0); | |
11fdf7f2 | 2467 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2468 | ASSERT_EQ(r, 0); |
2469 | } | |
2470 | { | |
2471 | ObjectStore::Transaction t; | |
2472 | t.remove(cid, a); | |
2473 | t.remove_collection(cid); | |
2474 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2475 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2476 | ASSERT_EQ(r, 0); |
2477 | } | |
2478 | } | |
2479 | ||
2480 | TEST_P(StoreTest, ManyBigWrite) { | |
7c673cae FG |
2481 | int r; |
2482 | coll_t cid; | |
2483 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
2484 | ghobject_t b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
11fdf7f2 | 2485 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2486 | { |
2487 | ObjectStore::Transaction t; | |
2488 | t.create_collection(cid, 0); | |
2489 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2490 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2491 | ASSERT_EQ(r, 0); |
2492 | } | |
2493 | bufferlist bl; | |
2494 | bufferptr bp(4 * 1048576); | |
2495 | bp.zero(); | |
2496 | bl.append(bp); | |
2497 | for (int i=0; i<10; ++i) { | |
2498 | ObjectStore::Transaction t; | |
2499 | t.write(cid, a, i*4*1048586, 4*1048576, bl, 0); | |
11fdf7f2 | 2500 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2501 | ASSERT_EQ(r, 0); |
2502 | } | |
2503 | // aligned | |
2504 | for (int i=0; i<10; ++i) { | |
2505 | ObjectStore::Transaction t; | |
2506 | t.write(cid, b, (rand() % 256)*4*1048576, 4*1048576, bl, 0); | |
11fdf7f2 | 2507 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2508 | ASSERT_EQ(r, 0); |
2509 | } | |
2510 | // unaligned | |
2511 | for (int i=0; i<10; ++i) { | |
2512 | ObjectStore::Transaction t; | |
2513 | t.write(cid, b, (rand() % (256*4096))*1024, 4*1048576, bl, 0); | |
11fdf7f2 | 2514 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2515 | ASSERT_EQ(r, 0); |
2516 | } | |
2517 | // do some zeros | |
2518 | for (int i=0; i<10; ++i) { | |
2519 | ObjectStore::Transaction t; | |
2520 | t.zero(cid, b, (rand() % (256*4096))*1024, 16*1048576); | |
11fdf7f2 | 2521 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2522 | ASSERT_EQ(r, 0); |
2523 | } | |
2524 | { | |
2525 | ObjectStore::Transaction t; | |
2526 | t.remove(cid, a); | |
2527 | t.remove(cid, b); | |
2528 | t.remove_collection(cid); | |
2529 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2530 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2531 | ASSERT_EQ(r, 0); |
2532 | } | |
2533 | } | |
2534 | ||
2535 | TEST_P(StoreTest, BigWriteBigZero) { | |
7c673cae FG |
2536 | int r; |
2537 | coll_t cid; | |
2538 | ghobject_t a(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2539 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2540 | { |
2541 | ObjectStore::Transaction t; | |
2542 | t.create_collection(cid, 0); | |
11fdf7f2 | 2543 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2544 | ASSERT_EQ(r, 0); |
2545 | } | |
2546 | bufferlist bl; | |
2547 | bufferptr bp(1048576); | |
2548 | memset(bp.c_str(), 'b', bp.length()); | |
2549 | bl.append(bp); | |
2550 | bufferlist s; | |
2551 | bufferptr sp(4096); | |
2552 | memset(sp.c_str(), 's', sp.length()); | |
2553 | s.append(sp); | |
2554 | { | |
2555 | ObjectStore::Transaction t; | |
2556 | t.write(cid, a, 0, bl.length(), bl); | |
11fdf7f2 | 2557 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2558 | ASSERT_EQ(r, 0); |
2559 | } | |
2560 | { | |
2561 | ObjectStore::Transaction t; | |
2562 | t.zero(cid, a, bl.length() / 4, bl.length() / 2); | |
11fdf7f2 | 2563 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2564 | ASSERT_EQ(r, 0); |
2565 | } | |
2566 | { | |
2567 | ObjectStore::Transaction t; | |
2568 | t.write(cid, a, bl.length() / 2, s.length(), s); | |
11fdf7f2 | 2569 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2570 | ASSERT_EQ(r, 0); |
2571 | } | |
2572 | { | |
2573 | ObjectStore::Transaction t; | |
2574 | t.remove(cid, a); | |
2575 | t.remove_collection(cid); | |
11fdf7f2 | 2576 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2577 | ASSERT_EQ(r, 0); |
2578 | } | |
2579 | } | |
2580 | ||
2581 | TEST_P(StoreTest, MiscFragmentTests) { | |
7c673cae FG |
2582 | int r; |
2583 | coll_t cid; | |
2584 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 2585 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2586 | { |
2587 | ObjectStore::Transaction t; | |
2588 | t.create_collection(cid, 0); | |
2589 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2590 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2591 | ASSERT_EQ(r, 0); |
2592 | } | |
2593 | bufferlist bl; | |
2594 | bufferptr bp(524288); | |
2595 | bp.zero(); | |
2596 | bl.append(bp); | |
2597 | { | |
2598 | ObjectStore::Transaction t; | |
2599 | t.write(cid, a, 0, 524288, bl, 0); | |
11fdf7f2 | 2600 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2601 | ASSERT_EQ(r, 0); |
2602 | } | |
2603 | { | |
2604 | ObjectStore::Transaction t; | |
2605 | t.write(cid, a, 1048576, 524288, bl, 0); | |
11fdf7f2 | 2606 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2607 | ASSERT_EQ(r, 0); |
2608 | } | |
2609 | { | |
2610 | bufferlist inbl; | |
11fdf7f2 | 2611 | int r = store->read(ch, a, 524288 + 131072, 1024, inbl); |
7c673cae FG |
2612 | ASSERT_EQ(r, 1024); |
2613 | ASSERT_EQ(inbl.length(), 1024u); | |
2614 | ASSERT_TRUE(inbl.is_zero()); | |
2615 | } | |
2616 | { | |
2617 | ObjectStore::Transaction t; | |
2618 | t.write(cid, a, 1048576 - 4096, 524288, bl, 0); | |
11fdf7f2 | 2619 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2620 | ASSERT_EQ(r, 0); |
2621 | } | |
2622 | { | |
2623 | ObjectStore::Transaction t; | |
2624 | t.remove(cid, a); | |
2625 | t.remove_collection(cid); | |
2626 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2627 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2628 | ASSERT_EQ(r, 0); |
2629 | } | |
2630 | ||
2631 | } | |
2632 | ||
b32b8144 | 2633 | TEST_P(StoreTest, ZeroVsObjectSize) { |
b32b8144 FG |
2634 | int r; |
2635 | coll_t cid; | |
2636 | struct stat stat; | |
2637 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2638 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2639 | { |
2640 | ObjectStore::Transaction t; | |
2641 | t.create_collection(cid, 0); | |
2642 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2643 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2644 | ASSERT_EQ(r, 0); |
2645 | } | |
2646 | bufferlist a; | |
2647 | a.append("stuff"); | |
2648 | { | |
2649 | ObjectStore::Transaction t; | |
2650 | t.write(cid, hoid, 0, 5, a); | |
11fdf7f2 | 2651 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2652 | ASSERT_EQ(r, 0); |
2653 | } | |
11fdf7f2 | 2654 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2655 | ASSERT_EQ(5, stat.st_size); |
2656 | { | |
2657 | ObjectStore::Transaction t; | |
2658 | t.zero(cid, hoid, 1, 2); | |
11fdf7f2 | 2659 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2660 | ASSERT_EQ(r, 0); |
2661 | } | |
11fdf7f2 | 2662 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2663 | ASSERT_EQ(5, stat.st_size); |
2664 | { | |
2665 | ObjectStore::Transaction t; | |
2666 | t.zero(cid, hoid, 3, 200); | |
11fdf7f2 | 2667 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2668 | ASSERT_EQ(r, 0); |
2669 | } | |
11fdf7f2 | 2670 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2671 | ASSERT_EQ(203, stat.st_size); |
2672 | { | |
2673 | ObjectStore::Transaction t; | |
2674 | t.zero(cid, hoid, 100000, 200); | |
11fdf7f2 | 2675 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2676 | ASSERT_EQ(r, 0); |
2677 | } | |
11fdf7f2 | 2678 | ASSERT_EQ(0, store->stat(ch, hoid, &stat)); |
b32b8144 FG |
2679 | ASSERT_EQ(100200, stat.st_size); |
2680 | } | |
2681 | ||
7c673cae | 2682 | TEST_P(StoreTest, ZeroLengthWrite) { |
7c673cae FG |
2683 | int r; |
2684 | coll_t cid; | |
2685 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2686 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2687 | { |
2688 | ObjectStore::Transaction t; | |
2689 | t.create_collection(cid, 0); | |
2690 | t.touch(cid, hoid); | |
11fdf7f2 | 2691 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2692 | ASSERT_EQ(r, 0); |
2693 | } | |
2694 | { | |
2695 | ObjectStore::Transaction t; | |
2696 | bufferlist empty; | |
2697 | t.write(cid, hoid, 1048576, 0, empty); | |
11fdf7f2 | 2698 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2699 | ASSERT_EQ(r, 0); |
2700 | } | |
2701 | struct stat stat; | |
11fdf7f2 | 2702 | r = store->stat(ch, hoid, &stat); |
7c673cae FG |
2703 | ASSERT_EQ(0, r); |
2704 | ASSERT_EQ(0, stat.st_size); | |
2705 | ||
2706 | bufferlist newdata; | |
11fdf7f2 | 2707 | r = store->read(ch, hoid, 0, 1048576, newdata); |
7c673cae FG |
2708 | ASSERT_EQ(0, r); |
2709 | } | |
2710 | ||
b32b8144 | 2711 | TEST_P(StoreTest, ZeroLengthZero) { |
b32b8144 FG |
2712 | int r; |
2713 | coll_t cid; | |
2714 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); | |
11fdf7f2 | 2715 | auto ch = store->create_new_collection(cid); |
b32b8144 FG |
2716 | { |
2717 | ObjectStore::Transaction t; | |
2718 | t.create_collection(cid, 0); | |
2719 | t.touch(cid, hoid); | |
11fdf7f2 | 2720 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2721 | ASSERT_EQ(0, r); |
2722 | } | |
2723 | { | |
2724 | ObjectStore::Transaction t; | |
2725 | t.zero(cid, hoid, 1048576, 0); | |
11fdf7f2 | 2726 | r = queue_transaction(store, ch, std::move(t)); |
b32b8144 FG |
2727 | ASSERT_EQ(0, r); |
2728 | } | |
2729 | struct stat stat; | |
11fdf7f2 | 2730 | r = store->stat(ch, hoid, &stat); |
b32b8144 FG |
2731 | ASSERT_EQ(0, r); |
2732 | ASSERT_EQ(0, stat.st_size); | |
2733 | ||
2734 | bufferlist newdata; | |
11fdf7f2 | 2735 | r = store->read(ch, hoid, 0, 1048576, newdata); |
b32b8144 FG |
2736 | ASSERT_EQ(0, r); |
2737 | } | |
2738 | ||
7c673cae | 2739 | TEST_P(StoreTest, SimpleAttrTest) { |
7c673cae FG |
2740 | int r; |
2741 | coll_t cid; | |
2742 | ghobject_t hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP))); | |
2743 | bufferlist val, val2; | |
2744 | val.append("value"); | |
2745 | val.append("value2"); | |
2746 | { | |
11fdf7f2 TL |
2747 | auto ch = store->open_collection(cid); |
2748 | ASSERT_FALSE(ch); | |
7c673cae | 2749 | } |
11fdf7f2 | 2750 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2751 | { |
2752 | ObjectStore::Transaction t; | |
2753 | t.create_collection(cid, 0); | |
11fdf7f2 | 2754 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2755 | ASSERT_EQ(r, 0); |
2756 | } | |
2757 | { | |
2758 | bool empty; | |
11fdf7f2 | 2759 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2760 | ASSERT_EQ(0, r); |
2761 | ASSERT_TRUE(empty); | |
2762 | } | |
2763 | { | |
2764 | bufferptr bp; | |
11fdf7f2 | 2765 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2766 | ASSERT_EQ(-ENOENT, r); |
2767 | } | |
2768 | { | |
2769 | ObjectStore::Transaction t; | |
2770 | t.touch(cid, hoid); | |
2771 | t.setattr(cid, hoid, "foo", val); | |
2772 | t.setattr(cid, hoid, "bar", val2); | |
11fdf7f2 | 2773 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2774 | ASSERT_EQ(r, 0); |
2775 | } | |
2776 | { | |
2777 | bool empty; | |
11fdf7f2 | 2778 | int r = store->collection_empty(ch, &empty); |
7c673cae FG |
2779 | ASSERT_EQ(0, r); |
2780 | ASSERT_TRUE(!empty); | |
2781 | } | |
2782 | { | |
2783 | bufferptr bp; | |
11fdf7f2 | 2784 | r = store->getattr(ch, hoid, "nofoo", bp); |
7c673cae FG |
2785 | ASSERT_EQ(-ENODATA, r); |
2786 | ||
11fdf7f2 | 2787 | r = store->getattr(ch, hoid, "foo", bp); |
7c673cae FG |
2788 | ASSERT_EQ(0, r); |
2789 | bufferlist bl; | |
2790 | bl.append(bp); | |
2791 | ASSERT_TRUE(bl_eq(val, bl)); | |
2792 | ||
2793 | map<string,bufferptr> bm; | |
11fdf7f2 | 2794 | r = store->getattrs(ch, hoid, bm); |
7c673cae FG |
2795 | ASSERT_EQ(0, r); |
2796 | ||
2797 | } | |
2798 | { | |
2799 | ObjectStore::Transaction t; | |
2800 | t.remove(cid, hoid); | |
2801 | t.remove_collection(cid); | |
11fdf7f2 | 2802 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2803 | ASSERT_EQ(r, 0); |
2804 | } | |
2805 | } | |
2806 | ||
2807 | TEST_P(StoreTest, SimpleListTest) { | |
7c673cae FG |
2808 | int r; |
2809 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 2810 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2811 | { |
2812 | ObjectStore::Transaction t; | |
2813 | t.create_collection(cid, 0); | |
2814 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2815 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2816 | ASSERT_EQ(r, 0); |
2817 | } | |
2818 | set<ghobject_t> all; | |
2819 | { | |
2820 | ObjectStore::Transaction t; | |
2821 | for (int i=0; i<200; ++i) { | |
2822 | string name("object_"); | |
2823 | name += stringify(i); | |
2824 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2825 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2826 | hoid.hobj.pool = 1; | |
2827 | all.insert(hoid); | |
2828 | t.touch(cid, hoid); | |
2829 | cerr << "Creating object " << hoid << std::endl; | |
2830 | } | |
11fdf7f2 | 2831 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2832 | ASSERT_EQ(r, 0); |
2833 | } | |
2834 | { | |
2835 | set<ghobject_t> saw; | |
2836 | vector<ghobject_t> objects; | |
2837 | ghobject_t next, current; | |
2838 | while (!next.is_max()) { | |
11fdf7f2 | 2839 | int r = store->collection_list(ch, current, ghobject_t::get_max(), |
7c673cae FG |
2840 | 50, |
2841 | &objects, &next); | |
2842 | ASSERT_EQ(r, 0); | |
2843 | ASSERT_TRUE(sorted(objects)); | |
2844 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2845 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2846 | ++p) { | |
2847 | if (saw.count(*p)) { | |
2848 | cout << "got DUP " << *p << std::endl; | |
2849 | } else { | |
2850 | //cout << "got new " << *p << std::endl; | |
2851 | } | |
2852 | saw.insert(*p); | |
2853 | } | |
2854 | objects.clear(); | |
2855 | current = next; | |
2856 | } | |
2857 | ASSERT_EQ(saw.size(), all.size()); | |
2858 | ASSERT_EQ(saw, all); | |
2859 | } | |
2860 | { | |
2861 | ObjectStore::Transaction t; | |
2862 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2863 | t.remove(cid, *p); | |
2864 | t.remove_collection(cid); | |
2865 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2866 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2867 | ASSERT_EQ(r, 0); |
2868 | } | |
2869 | } | |
2870 | ||
2871 | TEST_P(StoreTest, ListEndTest) { | |
7c673cae FG |
2872 | int r; |
2873 | coll_t cid(spg_t(pg_t(0, 1), shard_id_t(1))); | |
11fdf7f2 | 2874 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2875 | { |
2876 | ObjectStore::Transaction t; | |
2877 | t.create_collection(cid, 0); | |
2878 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2879 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2880 | ASSERT_EQ(r, 0); |
2881 | } | |
2882 | set<ghobject_t> all; | |
2883 | { | |
2884 | ObjectStore::Transaction t; | |
2885 | for (int i=0; i<200; ++i) { | |
2886 | string name("object_"); | |
2887 | name += stringify(i); | |
2888 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP)), | |
2889 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2890 | hoid.hobj.pool = 1; | |
2891 | all.insert(hoid); | |
2892 | t.touch(cid, hoid); | |
2893 | cerr << "Creating object " << hoid << std::endl; | |
2894 | } | |
11fdf7f2 | 2895 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2896 | ASSERT_EQ(r, 0); |
2897 | } | |
2898 | { | |
2899 | ghobject_t end(hobject_t(sobject_t("object_100", CEPH_NOSNAP)), | |
2900 | ghobject_t::NO_GEN, shard_id_t(1)); | |
2901 | end.hobj.pool = 1; | |
2902 | vector<ghobject_t> objects; | |
2903 | ghobject_t next; | |
11fdf7f2 | 2904 | int r = store->collection_list(ch, ghobject_t(), end, 500, |
7c673cae FG |
2905 | &objects, &next); |
2906 | ASSERT_EQ(r, 0); | |
2907 | for (auto &p : objects) { | |
2908 | ASSERT_NE(p, end); | |
2909 | } | |
2910 | } | |
2911 | { | |
2912 | ObjectStore::Transaction t; | |
2913 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
2914 | t.remove(cid, *p); | |
2915 | t.remove_collection(cid); | |
2916 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 2917 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2918 | ASSERT_EQ(r, 0); |
2919 | } | |
2920 | } | |
2921 | ||
2922 | TEST_P(StoreTest, Sort) { | |
2923 | { | |
2924 | hobject_t a(sobject_t("a", CEPH_NOSNAP)); | |
2925 | hobject_t b = a; | |
2926 | ASSERT_EQ(a, b); | |
2927 | b.oid.name = "b"; | |
2928 | ASSERT_NE(a, b); | |
2929 | ASSERT_TRUE(a < b); | |
2930 | a.pool = 1; | |
2931 | b.pool = 2; | |
2932 | ASSERT_TRUE(a < b); | |
2933 | a.pool = 3; | |
2934 | ASSERT_TRUE(a > b); | |
2935 | } | |
2936 | { | |
2937 | ghobject_t a(hobject_t(sobject_t("a", CEPH_NOSNAP))); | |
2938 | ghobject_t b(hobject_t(sobject_t("b", CEPH_NOSNAP))); | |
2939 | a.hobj.pool = 1; | |
2940 | b.hobj.pool = 1; | |
2941 | ASSERT_TRUE(a < b); | |
2942 | a.hobj.pool = -3; | |
2943 | ASSERT_TRUE(a < b); | |
2944 | a.hobj.pool = 1; | |
2945 | b.hobj.pool = -3; | |
2946 | ASSERT_TRUE(a > b); | |
2947 | } | |
2948 | } | |
2949 | ||
2950 | TEST_P(StoreTest, MultipoolListTest) { | |
7c673cae FG |
2951 | int r; |
2952 | int poolid = 4373; | |
2953 | coll_t cid = coll_t(spg_t(pg_t(0, poolid), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 2954 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
2955 | { |
2956 | ObjectStore::Transaction t; | |
2957 | t.create_collection(cid, 0); | |
2958 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 2959 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2960 | ASSERT_EQ(r, 0); |
2961 | } | |
2962 | set<ghobject_t> all, saw; | |
2963 | { | |
2964 | ObjectStore::Transaction t; | |
2965 | for (int i=0; i<200; ++i) { | |
2966 | string name("object_"); | |
2967 | name += stringify(i); | |
2968 | ghobject_t hoid(hobject_t(sobject_t(name, CEPH_NOSNAP))); | |
2969 | if (rand() & 1) | |
2970 | hoid.hobj.pool = -2 - poolid; | |
2971 | else | |
2972 | hoid.hobj.pool = poolid; | |
2973 | all.insert(hoid); | |
2974 | t.touch(cid, hoid); | |
2975 | cerr << "Creating object " << hoid << std::endl; | |
2976 | } | |
11fdf7f2 | 2977 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
2978 | ASSERT_EQ(r, 0); |
2979 | } | |
2980 | { | |
2981 | vector<ghobject_t> objects; | |
2982 | ghobject_t next, current; | |
2983 | while (!next.is_max()) { | |
11fdf7f2 | 2984 | int r = store->collection_list(ch, current, ghobject_t::get_max(), 50, |
7c673cae FG |
2985 | &objects, &next); |
2986 | ASSERT_EQ(r, 0); | |
2987 | cout << " got " << objects.size() << " next " << next << std::endl; | |
2988 | for (vector<ghobject_t>::iterator p = objects.begin(); p != objects.end(); | |
2989 | ++p) { | |
2990 | saw.insert(*p); | |
2991 | } | |
2992 | objects.clear(); | |
2993 | current = next; | |
2994 | } | |
2995 | ASSERT_EQ(saw, all); | |
2996 | } | |
2997 | { | |
2998 | ObjectStore::Transaction t; | |
2999 | for (set<ghobject_t>::iterator p = all.begin(); p != all.end(); ++p) | |
3000 | t.remove(cid, *p); | |
3001 | t.remove_collection(cid); | |
3002 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3003 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3004 | ASSERT_EQ(r, 0); |
3005 | } | |
3006 | } | |
3007 | ||
3008 | TEST_P(StoreTest, SimpleCloneTest) { | |
7c673cae FG |
3009 | int r; |
3010 | coll_t cid; | |
11fdf7f2 | 3011 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3012 | { |
3013 | ObjectStore::Transaction t; | |
3014 | t.create_collection(cid, 0); | |
3015 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3016 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3017 | ASSERT_EQ(r, 0); |
3018 | } | |
3019 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3020 | "key", 123, -1, "")); | |
3021 | bufferlist small, large, xlarge, newdata, attr; | |
3022 | small.append("small"); | |
3023 | large.append("large"); | |
3024 | xlarge.append("xlarge"); | |
3025 | { | |
3026 | ObjectStore::Transaction t; | |
3027 | t.touch(cid, hoid); | |
3028 | t.setattr(cid, hoid, "attr1", small); | |
3029 | t.setattr(cid, hoid, "attr2", large); | |
3030 | t.setattr(cid, hoid, "attr3", xlarge); | |
3031 | t.write(cid, hoid, 0, small.length(), small); | |
3032 | t.write(cid, hoid, 10, small.length(), small); | |
3033 | cerr << "Creating object and set attr " << hoid << std::endl; | |
11fdf7f2 | 3034 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3035 | ASSERT_EQ(r, 0); |
3036 | } | |
3037 | ||
3038 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3039 | "key", 123, -1, "")); | |
3040 | ghobject_t hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP))); | |
3041 | { | |
3042 | ObjectStore::Transaction t; | |
3043 | t.clone(cid, hoid, hoid2); | |
3044 | t.setattr(cid, hoid2, "attr2", small); | |
3045 | t.rmattr(cid, hoid2, "attr1"); | |
3046 | t.write(cid, hoid, 10, large.length(), large); | |
3047 | t.setattr(cid, hoid, "attr1", large); | |
3048 | t.setattr(cid, hoid, "attr2", small); | |
3049 | cerr << "Clone object and rm attr" << std::endl; | |
11fdf7f2 | 3050 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3051 | ASSERT_EQ(r, 0); |
3052 | ||
11fdf7f2 | 3053 | r = store->read(ch, hoid, 10, 5, newdata); |
7c673cae FG |
3054 | ASSERT_EQ(r, 5); |
3055 | ASSERT_TRUE(bl_eq(large, newdata)); | |
3056 | ||
3057 | newdata.clear(); | |
11fdf7f2 | 3058 | r = store->read(ch, hoid, 0, 5, newdata); |
7c673cae FG |
3059 | ASSERT_EQ(r, 5); |
3060 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3061 | ||
3062 | newdata.clear(); | |
11fdf7f2 | 3063 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3064 | ASSERT_EQ(r, 5); |
3065 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3066 | ||
11fdf7f2 | 3067 | r = store->getattr(ch, hoid2, "attr2", attr); |
7c673cae FG |
3068 | ASSERT_EQ(r, 0); |
3069 | ASSERT_TRUE(bl_eq(small, attr)); | |
3070 | ||
3071 | attr.clear(); | |
11fdf7f2 | 3072 | r = store->getattr(ch, hoid2, "attr3", attr); |
7c673cae FG |
3073 | ASSERT_EQ(r, 0); |
3074 | ASSERT_TRUE(bl_eq(xlarge, attr)); | |
3075 | ||
3076 | attr.clear(); | |
11fdf7f2 | 3077 | r = store->getattr(ch, hoid, "attr1", attr); |
7c673cae FG |
3078 | ASSERT_EQ(r, 0); |
3079 | ASSERT_TRUE(bl_eq(large, attr)); | |
3080 | } | |
3081 | { | |
3082 | ObjectStore::Transaction t; | |
3083 | t.remove(cid, hoid); | |
3084 | t.remove(cid, hoid2); | |
11fdf7f2 | 3085 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3086 | } |
3087 | { | |
3088 | bufferlist final; | |
3089 | bufferptr p(16384); | |
3090 | memset(p.c_str(), 1, p.length()); | |
3091 | bufferlist pl; | |
3092 | pl.append(p); | |
3093 | final.append(p); | |
3094 | ObjectStore::Transaction t; | |
3095 | t.write(cid, hoid, 0, pl.length(), pl); | |
3096 | t.clone(cid, hoid, hoid2); | |
3097 | bufferptr a(4096); | |
3098 | memset(a.c_str(), 2, a.length()); | |
3099 | bufferlist al; | |
3100 | al.append(a); | |
3101 | final.append(a); | |
3102 | t.write(cid, hoid, pl.length(), a.length(), al); | |
11fdf7f2 | 3103 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3104 | ASSERT_EQ(r, 0); |
3105 | bufferlist rl; | |
3106 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3107 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3108 | ASSERT_TRUE(bl_eq(rl, final)); |
3109 | } | |
3110 | { | |
3111 | ObjectStore::Transaction t; | |
3112 | t.remove(cid, hoid); | |
3113 | t.remove(cid, hoid2); | |
11fdf7f2 | 3114 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3115 | } |
3116 | { | |
3117 | bufferlist final; | |
3118 | bufferptr p(16384); | |
3119 | memset(p.c_str(), 111, p.length()); | |
3120 | bufferlist pl; | |
3121 | pl.append(p); | |
3122 | final.append(p); | |
3123 | ObjectStore::Transaction t; | |
3124 | t.write(cid, hoid, 0, pl.length(), pl); | |
3125 | t.clone(cid, hoid, hoid2); | |
3126 | bufferptr z(4096); | |
3127 | z.zero(); | |
3128 | final.append(z); | |
3129 | bufferptr a(4096); | |
3130 | memset(a.c_str(), 112, a.length()); | |
3131 | bufferlist al; | |
3132 | al.append(a); | |
3133 | final.append(a); | |
3134 | t.write(cid, hoid, pl.length() + z.length(), a.length(), al); | |
11fdf7f2 | 3135 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3136 | ASSERT_EQ(r, 0); |
3137 | bufferlist rl; | |
3138 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3139 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3140 | ASSERT_TRUE(bl_eq(rl, final)); |
3141 | } | |
3142 | { | |
3143 | ObjectStore::Transaction t; | |
3144 | t.remove(cid, hoid); | |
3145 | t.remove(cid, hoid2); | |
11fdf7f2 | 3146 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3147 | } |
3148 | { | |
3149 | bufferlist final; | |
3150 | bufferptr p(16000); | |
3151 | memset(p.c_str(), 5, p.length()); | |
3152 | bufferlist pl; | |
3153 | pl.append(p); | |
3154 | final.append(p); | |
3155 | ObjectStore::Transaction t; | |
3156 | t.write(cid, hoid, 0, pl.length(), pl); | |
3157 | t.clone(cid, hoid, hoid2); | |
3158 | bufferptr z(1000); | |
3159 | z.zero(); | |
3160 | final.append(z); | |
3161 | bufferptr a(8000); | |
3162 | memset(a.c_str(), 6, a.length()); | |
3163 | bufferlist al; | |
3164 | al.append(a); | |
3165 | final.append(a); | |
3166 | t.write(cid, hoid, 17000, a.length(), al); | |
11fdf7f2 | 3167 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3168 | bufferlist rl; |
3169 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3170 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3171 | /*cout << "expected:\n"; |
3172 | final.hexdump(cout); | |
3173 | cout << "got:\n"; | |
3174 | rl.hexdump(cout);*/ | |
3175 | ASSERT_TRUE(bl_eq(rl, final)); | |
3176 | } | |
3177 | { | |
3178 | ObjectStore::Transaction t; | |
3179 | t.remove(cid, hoid); | |
3180 | t.remove(cid, hoid2); | |
11fdf7f2 | 3181 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3182 | } |
3183 | { | |
3184 | bufferptr p(1048576); | |
3185 | memset(p.c_str(), 3, p.length()); | |
3186 | bufferlist pl; | |
3187 | pl.append(p); | |
3188 | ObjectStore::Transaction t; | |
3189 | t.write(cid, hoid, 0, pl.length(), pl); | |
3190 | t.clone(cid, hoid, hoid2); | |
3191 | bufferptr a(65536); | |
3192 | memset(a.c_str(), 4, a.length()); | |
3193 | bufferlist al; | |
3194 | al.append(a); | |
3195 | t.write(cid, hoid, a.length(), a.length(), al); | |
11fdf7f2 | 3196 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3197 | bufferlist rl; |
3198 | bufferlist final; | |
3199 | final.substr_of(pl, 0, al.length()); | |
3200 | final.append(al); | |
3201 | bufferlist end; | |
3202 | end.substr_of(pl, al.length()*2, pl.length() - al.length()*2); | |
3203 | final.append(end); | |
3204 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3205 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3206 | /*cout << "expected:\n"; |
3207 | final.hexdump(cout); | |
3208 | cout << "got:\n"; | |
3209 | rl.hexdump(cout);*/ | |
3210 | ASSERT_TRUE(bl_eq(rl, final)); | |
3211 | } | |
3212 | { | |
3213 | ObjectStore::Transaction t; | |
3214 | t.remove(cid, hoid); | |
3215 | t.remove(cid, hoid2); | |
11fdf7f2 | 3216 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3217 | } |
3218 | { | |
3219 | bufferptr p(65536); | |
3220 | memset(p.c_str(), 7, p.length()); | |
3221 | bufferlist pl; | |
3222 | pl.append(p); | |
3223 | ObjectStore::Transaction t; | |
3224 | t.write(cid, hoid, 0, pl.length(), pl); | |
3225 | t.clone(cid, hoid, hoid2); | |
3226 | bufferptr a(4096); | |
3227 | memset(a.c_str(), 8, a.length()); | |
3228 | bufferlist al; | |
3229 | al.append(a); | |
3230 | t.write(cid, hoid, 32768, a.length(), al); | |
11fdf7f2 | 3231 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3232 | bufferlist rl; |
3233 | bufferlist final; | |
3234 | final.substr_of(pl, 0, 32768); | |
3235 | final.append(al); | |
3236 | bufferlist end; | |
3237 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3238 | final.append(end); | |
3239 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3240 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3241 | /*cout << "expected:\n"; |
3242 | final.hexdump(cout); | |
3243 | cout << "got:\n"; | |
3244 | rl.hexdump(cout);*/ | |
3245 | ASSERT_TRUE(bl_eq(rl, final)); | |
3246 | } | |
3247 | { | |
3248 | ObjectStore::Transaction t; | |
3249 | t.remove(cid, hoid); | |
3250 | t.remove(cid, hoid2); | |
11fdf7f2 | 3251 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3252 | } |
3253 | { | |
3254 | bufferptr p(65536); | |
3255 | memset(p.c_str(), 9, p.length()); | |
3256 | bufferlist pl; | |
3257 | pl.append(p); | |
3258 | ObjectStore::Transaction t; | |
3259 | t.write(cid, hoid, 0, pl.length(), pl); | |
3260 | t.clone(cid, hoid, hoid2); | |
3261 | bufferptr a(4096); | |
3262 | memset(a.c_str(), 10, a.length()); | |
3263 | bufferlist al; | |
3264 | al.append(a); | |
3265 | t.write(cid, hoid, 33768, a.length(), al); | |
11fdf7f2 | 3266 | ASSERT_EQ(0, queue_transaction(store, ch, std::move(t))); |
7c673cae FG |
3267 | bufferlist rl; |
3268 | bufferlist final; | |
3269 | final.substr_of(pl, 0, 33768); | |
3270 | final.append(al); | |
3271 | bufferlist end; | |
3272 | end.substr_of(pl, final.length(), pl.length() - final.length()); | |
3273 | final.append(end); | |
3274 | ASSERT_EQ((int)final.length(), | |
11fdf7f2 | 3275 | store->read(ch, hoid, 0, final.length(), rl)); |
7c673cae FG |
3276 | /*cout << "expected:\n"; |
3277 | final.hexdump(cout); | |
3278 | cout << "got:\n"; | |
3279 | rl.hexdump(cout);*/ | |
3280 | ASSERT_TRUE(bl_eq(rl, final)); | |
3281 | } | |
3282 | ||
3283 | //Unfortunately we need a workaround for filestore since EXPECT_DEATH | |
3284 | // macro has potential issues when using /in multithread environments. | |
3285 | //It works well for all stores but filestore for now. | |
3286 | //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well - | |
3287 | // test app clone asserts on store folder presence. | |
3288 | // | |
3289 | if (string(GetParam()) != "filestore") { | |
3290 | //verify if non-empty collection is properly handled after store reload | |
11fdf7f2 | 3291 | ch.reset(); |
7c673cae FG |
3292 | r = store->umount(); |
3293 | ASSERT_EQ(r, 0); | |
3294 | r = store->mount(); | |
3295 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3296 | ch = store->open_collection(cid); |
7c673cae FG |
3297 | |
3298 | ObjectStore::Transaction t; | |
3299 | t.remove_collection(cid); | |
3300 | cerr << "Invalid rm coll" << std::endl; | |
3301 | PrCtl unset_dumpable; | |
11fdf7f2 | 3302 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3303 | } |
3304 | { | |
3305 | ObjectStore::Transaction t; | |
3306 | t.touch(cid, hoid3); //new record in db | |
11fdf7f2 | 3307 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3308 | ASSERT_EQ(r, 0); |
3309 | } | |
3310 | //See comment above for "filestore" check explanation. | |
3311 | if (string(GetParam()) != "filestore") { | |
3312 | ObjectStore::Transaction t; | |
3313 | //verify if non-empty collection is properly handled when there are some pending removes and live records in db | |
3314 | cerr << "Invalid rm coll again" << std::endl; | |
11fdf7f2 | 3315 | ch.reset(); |
7c673cae FG |
3316 | r = store->umount(); |
3317 | ASSERT_EQ(r, 0); | |
3318 | r = store->mount(); | |
3319 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 3320 | ch = store->open_collection(cid); |
7c673cae FG |
3321 | |
3322 | t.remove(cid, hoid); | |
3323 | t.remove(cid, hoid2); | |
3324 | t.remove_collection(cid); | |
3325 | PrCtl unset_dumpable; | |
11fdf7f2 | 3326 | EXPECT_DEATH(queue_transaction(store, ch, std::move(t)), ""); |
7c673cae FG |
3327 | } |
3328 | { | |
3329 | ObjectStore::Transaction t; | |
3330 | t.remove(cid, hoid); | |
3331 | t.remove(cid, hoid2); | |
3332 | t.remove(cid, hoid3); | |
3333 | t.remove_collection(cid); | |
3334 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3335 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3336 | ASSERT_EQ(r, 0); |
3337 | } | |
3338 | } | |
3339 | ||
3340 | TEST_P(StoreTest, OmapSimple) { | |
7c673cae FG |
3341 | int r; |
3342 | coll_t cid; | |
11fdf7f2 | 3343 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3344 | { |
3345 | ObjectStore::Transaction t; | |
3346 | t.create_collection(cid, 0); | |
3347 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3348 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3349 | ASSERT_EQ(r, 0); |
3350 | } | |
3351 | ghobject_t hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP), | |
3352 | "key", 123, -1, "")); | |
3353 | bufferlist small; | |
3354 | small.append("small"); | |
3355 | map<string,bufferlist> km; | |
3356 | km["foo"] = small; | |
3357 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3358 | bufferlist header; | |
3359 | header.append("this is a header"); | |
3360 | { | |
3361 | ObjectStore::Transaction t; | |
3362 | t.touch(cid, hoid); | |
3363 | t.omap_setkeys(cid, hoid, km); | |
3364 | t.omap_setheader(cid, hoid, header); | |
3365 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3366 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3367 | ASSERT_EQ(r, 0); |
3368 | } | |
3369 | // get header, keys | |
3370 | { | |
3371 | bufferlist h; | |
3372 | map<string,bufferlist> r; | |
11fdf7f2 | 3373 | store->omap_get(ch, hoid, &h, &r); |
7c673cae FG |
3374 | ASSERT_TRUE(bl_eq(header, h)); |
3375 | ASSERT_EQ(r.size(), km.size()); | |
3376 | cout << "r: " << r << std::endl; | |
3377 | } | |
3378 | // test iterator with seek_to_first | |
3379 | { | |
3380 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3381 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3382 | for (iter->seek_to_first(); iter->valid(); iter->next()) { | |
7c673cae FG |
3383 | r[iter->key()] = iter->value(); |
3384 | } | |
3385 | cout << "r: " << r << std::endl; | |
3386 | ASSERT_EQ(r.size(), km.size()); | |
3387 | } | |
3388 | // test iterator with initial lower_bound | |
3389 | { | |
3390 | map<string,bufferlist> r; | |
11fdf7f2 TL |
3391 | ObjectMap::ObjectMapIterator iter = store->get_omap_iterator(ch, hoid); |
3392 | for (iter->lower_bound(string()); iter->valid(); iter->next()) { | |
7c673cae FG |
3393 | r[iter->key()] = iter->value(); |
3394 | } | |
3395 | cout << "r: " << r << std::endl; | |
3396 | ASSERT_EQ(r.size(), km.size()); | |
3397 | } | |
3398 | { | |
3399 | ObjectStore::Transaction t; | |
3400 | t.remove(cid, hoid); | |
3401 | t.remove_collection(cid); | |
3402 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3403 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3404 | ASSERT_EQ(r, 0); |
3405 | } | |
3406 | } | |
3407 | ||
3408 | TEST_P(StoreTest, OmapCloneTest) { | |
7c673cae FG |
3409 | int r; |
3410 | coll_t cid; | |
11fdf7f2 | 3411 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3412 | { |
3413 | ObjectStore::Transaction t; | |
3414 | t.create_collection(cid, 0); | |
3415 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3416 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3417 | ASSERT_EQ(r, 0); |
3418 | } | |
3419 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP), | |
3420 | "key", 123, -1, "")); | |
3421 | bufferlist small; | |
3422 | small.append("small"); | |
3423 | map<string,bufferlist> km; | |
3424 | km["foo"] = small; | |
3425 | km["bar"].append("asdfjkasdkjdfsjkafskjsfdj"); | |
3426 | bufferlist header; | |
3427 | header.append("this is a header"); | |
3428 | { | |
3429 | ObjectStore::Transaction t; | |
3430 | t.touch(cid, hoid); | |
3431 | t.omap_setkeys(cid, hoid, km); | |
3432 | t.omap_setheader(cid, hoid, header); | |
3433 | cerr << "Creating object and set omap " << hoid << std::endl; | |
11fdf7f2 | 3434 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3435 | ASSERT_EQ(r, 0); |
3436 | } | |
3437 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP), | |
3438 | "key", 123, -1, "")); | |
3439 | { | |
3440 | ObjectStore::Transaction t; | |
3441 | t.clone(cid, hoid, hoid2); | |
3442 | cerr << "Clone object" << std::endl; | |
11fdf7f2 | 3443 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3444 | ASSERT_EQ(r, 0); |
3445 | } | |
3446 | { | |
3447 | map<string,bufferlist> r; | |
3448 | bufferlist h; | |
11fdf7f2 | 3449 | store->omap_get(ch, hoid2, &h, &r); |
7c673cae FG |
3450 | ASSERT_TRUE(bl_eq(header, h)); |
3451 | ASSERT_EQ(r.size(), km.size()); | |
3452 | } | |
3453 | { | |
3454 | ObjectStore::Transaction t; | |
3455 | t.remove(cid, hoid); | |
3456 | t.remove(cid, hoid2); | |
3457 | t.remove_collection(cid); | |
3458 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3459 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3460 | ASSERT_EQ(r, 0); |
3461 | } | |
3462 | } | |
3463 | ||
3464 | TEST_P(StoreTest, SimpleCloneRangeTest) { | |
7c673cae FG |
3465 | int r; |
3466 | coll_t cid; | |
11fdf7f2 | 3467 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3468 | { |
3469 | ObjectStore::Transaction t; | |
3470 | t.create_collection(cid, 0); | |
3471 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3472 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3473 | ASSERT_EQ(r, 0); |
3474 | } | |
3475 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
3476 | hoid.hobj.pool = -1; | |
3477 | bufferlist small, newdata; | |
3478 | small.append("small"); | |
3479 | { | |
3480 | ObjectStore::Transaction t; | |
3481 | t.write(cid, hoid, 10, 5, small); | |
3482 | cerr << "Creating object and write bl " << hoid << std::endl; | |
11fdf7f2 | 3483 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3484 | ASSERT_EQ(r, 0); |
3485 | } | |
3486 | ghobject_t hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP))); | |
3487 | hoid2.hobj.pool = -1; | |
3488 | { | |
3489 | ObjectStore::Transaction t; | |
3490 | t.clone_range(cid, hoid, hoid2, 10, 5, 10); | |
3491 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3492 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 3493 | ASSERT_EQ(r, 0); |
11fdf7f2 | 3494 | r = store->read(ch, hoid2, 10, 5, newdata); |
7c673cae FG |
3495 | ASSERT_EQ(r, 5); |
3496 | ASSERT_TRUE(bl_eq(small, newdata)); | |
3497 | } | |
3498 | { | |
3499 | ObjectStore::Transaction t; | |
3500 | t.truncate(cid, hoid, 1024*1024); | |
3501 | t.clone_range(cid, hoid, hoid2, 0, 1024*1024, 0); | |
3502 | cerr << "Clone range object" << std::endl; | |
11fdf7f2 | 3503 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3504 | ASSERT_EQ(r, 0); |
3505 | struct stat stat, stat2; | |
11fdf7f2 TL |
3506 | r = store->stat(ch, hoid, &stat); |
3507 | r = store->stat(ch, hoid2, &stat2); | |
7c673cae FG |
3508 | ASSERT_EQ(stat.st_size, stat2.st_size); |
3509 | ASSERT_EQ(1024*1024, stat2.st_size); | |
3510 | } | |
3511 | { | |
3512 | ObjectStore::Transaction t; | |
3513 | t.remove(cid, hoid); | |
3514 | t.remove(cid, hoid2); | |
3515 | t.remove_collection(cid); | |
3516 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3517 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3518 | ASSERT_EQ(r, 0); |
3519 | } | |
3520 | } | |
3521 | ||
3522 | ||
3523 | TEST_P(StoreTest, SimpleObjectLongnameTest) { | |
7c673cae FG |
3524 | int r; |
3525 | coll_t cid; | |
11fdf7f2 | 3526 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3527 | { |
3528 | ObjectStore::Transaction t; | |
3529 | t.create_collection(cid, 0); | |
3530 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3531 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3532 | ASSERT_EQ(r, 0); |
3533 | } | |
3534 | ghobject_t hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP))); | |
3535 | { | |
3536 | ObjectStore::Transaction t; | |
3537 | t.touch(cid, hoid); | |
3538 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3539 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3540 | ASSERT_EQ(r, 0); |
3541 | } | |
3542 | { | |
3543 | ObjectStore::Transaction t; | |
3544 | t.remove(cid, hoid); | |
3545 | t.remove_collection(cid); | |
3546 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3547 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3548 | ASSERT_EQ(r, 0); |
3549 | } | |
3550 | } | |
3551 | ||
3552 | ghobject_t generate_long_name(unsigned i) | |
3553 | { | |
3554 | stringstream name; | |
3555 | name << "object id " << i << " "; | |
3556 | for (unsigned j = 0; j < 500; ++j) name << 'a'; | |
3557 | ghobject_t hoid(hobject_t(sobject_t(name.str(), CEPH_NOSNAP))); | |
3558 | hoid.hobj.set_hash(i % 2); | |
3559 | return hoid; | |
3560 | } | |
3561 | ||
3562 | TEST_P(StoreTest, LongnameSplitTest) { | |
7c673cae FG |
3563 | int r; |
3564 | coll_t cid; | |
11fdf7f2 | 3565 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3566 | { |
3567 | ObjectStore::Transaction t; | |
3568 | t.create_collection(cid, 0); | |
3569 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 3570 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3571 | ASSERT_EQ(0, r); |
3572 | } | |
3573 | for (unsigned i = 0; i < 320; ++i) { | |
3574 | ObjectStore::Transaction t; | |
3575 | ghobject_t hoid = generate_long_name(i); | |
3576 | t.touch(cid, hoid); | |
3577 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 3578 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3579 | ASSERT_EQ(0, r); |
3580 | } | |
3581 | ||
3582 | ghobject_t test_obj = generate_long_name(319); | |
3583 | ghobject_t test_obj_2 = test_obj; | |
3584 | test_obj_2.generation = 0; | |
3585 | { | |
3586 | ObjectStore::Transaction t; | |
3587 | // should cause a split | |
3588 | t.collection_move_rename( | |
3589 | cid, test_obj, | |
3590 | cid, test_obj_2); | |
11fdf7f2 | 3591 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3592 | ASSERT_EQ(0, r); |
3593 | } | |
3594 | ||
3595 | for (unsigned i = 0; i < 319; ++i) { | |
3596 | ObjectStore::Transaction t; | |
3597 | ghobject_t hoid = generate_long_name(i); | |
3598 | t.remove(cid, hoid); | |
3599 | cerr << "Removing object " << hoid << std::endl; | |
11fdf7f2 | 3600 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3601 | ASSERT_EQ(0, r); |
3602 | } | |
3603 | { | |
3604 | ObjectStore::Transaction t; | |
3605 | t.remove(cid, test_obj_2); | |
3606 | t.remove_collection(cid); | |
3607 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 3608 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3609 | ASSERT_EQ(0, r); |
3610 | } | |
3611 | ||
3612 | } | |
3613 | ||
3614 | TEST_P(StoreTest, ManyObjectTest) { | |
7c673cae FG |
3615 | int NUM_OBJS = 2000; |
3616 | int r = 0; | |
3617 | coll_t cid; | |
3618 | string base = ""; | |
3619 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
3620 | set<ghobject_t> created; | |
11fdf7f2 | 3621 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
3622 | { |
3623 | ObjectStore::Transaction t; | |
3624 | t.create_collection(cid, 0); | |
11fdf7f2 | 3625 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3626 | ASSERT_EQ(r, 0); |
3627 | } | |
3628 | for (int i = 0; i < NUM_OBJS; ++i) { | |
3629 | if (!(i % 5)) { | |
3630 | cerr << "Object " << i << std::endl; | |
3631 | } | |
3632 | ObjectStore::Transaction t; | |
3633 | char buf[100]; | |
3634 | snprintf(buf, sizeof(buf), "%d", i); | |
3635 | ghobject_t hoid(hobject_t(sobject_t(string(buf) + base, CEPH_NOSNAP))); | |
3636 | t.touch(cid, hoid); | |
3637 | created.insert(hoid); | |
11fdf7f2 | 3638 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3639 | ASSERT_EQ(r, 0); |
3640 | } | |
3641 | ||
3642 | for (set<ghobject_t>::iterator i = created.begin(); | |
3643 | i != created.end(); | |
3644 | ++i) { | |
3645 | struct stat buf; | |
11fdf7f2 | 3646 | ASSERT_TRUE(!store->stat(ch, *i, &buf)); |
7c673cae FG |
3647 | } |
3648 | ||
3649 | set<ghobject_t> listed, listed2; | |
3650 | vector<ghobject_t> objects; | |
11fdf7f2 | 3651 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0); |
7c673cae FG |
3652 | ASSERT_EQ(r, 0); |
3653 | ||
3654 | cerr << "objects.size() is " << objects.size() << std::endl; | |
3655 | for (vector<ghobject_t> ::iterator i = objects.begin(); | |
3656 | i != objects.end(); | |
3657 | ++i) { | |
3658 | listed.insert(*i); | |
3659 | ASSERT_TRUE(created.count(*i)); | |
3660 | } | |
3661 | ASSERT_TRUE(listed.size() == created.size()); | |
3662 | ||
3663 | ghobject_t start, next; | |
3664 | objects.clear(); | |
3665 | r = store->collection_list( | |
11fdf7f2 | 3666 | ch, |
7c673cae FG |
3667 | ghobject_t::get_max(), |
3668 | ghobject_t::get_max(), | |
3669 | 50, | |
3670 | &objects, | |
3671 | &next | |
3672 | ); | |
3673 | ASSERT_EQ(r, 0); | |
3674 | ASSERT_TRUE(objects.empty()); | |
3675 | ||
3676 | objects.clear(); | |
3677 | listed.clear(); | |
3678 | ghobject_t start2, next2; | |
3679 | while (1) { | |
11fdf7f2 | 3680 | r = store->collection_list(ch, start, ghobject_t::get_max(), |
7c673cae FG |
3681 | 50, |
3682 | &objects, | |
3683 | &next); | |
3684 | ASSERT_TRUE(sorted(objects)); | |
3685 | ASSERT_EQ(r, 0); | |
3686 | listed.insert(objects.begin(), objects.end()); | |
3687 | if (objects.size() < 50) { | |
3688 | ASSERT_TRUE(next.is_max()); | |
3689 | break; | |
3690 | } | |
3691 | objects.clear(); | |
3692 | ||
3693 | start = next; | |
3694 | } | |
3695 | cerr << "listed.size() is " << listed.size() << std::endl; | |
3696 | ASSERT_TRUE(listed.size() == created.size()); | |
3697 | if (listed2.size()) { | |
3698 | ASSERT_EQ(listed.size(), listed2.size()); | |
3699 | } | |
3700 | for (set<ghobject_t>::iterator i = listed.begin(); | |
3701 | i != listed.end(); | |
3702 | ++i) { | |
3703 | ASSERT_TRUE(created.count(*i)); | |
3704 | } | |
3705 | ||
3706 | for (set<ghobject_t>::iterator i = created.begin(); | |
3707 | i != created.end(); | |
3708 | ++i) { | |
3709 | ObjectStore::Transaction t; | |
3710 | t.remove(cid, *i); | |
11fdf7f2 | 3711 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3712 | ASSERT_EQ(r, 0); |
3713 | } | |
3714 | cerr << "cleaning up" << std::endl; | |
3715 | { | |
3716 | ObjectStore::Transaction t; | |
3717 | t.remove_collection(cid); | |
11fdf7f2 | 3718 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3719 | ASSERT_EQ(r, 0); |
3720 | } | |
3721 | } | |
3722 | ||
3723 | ||
3724 | class ObjectGenerator { | |
3725 | public: | |
3726 | virtual ghobject_t create_object(gen_type *gen) = 0; | |
3727 | virtual ~ObjectGenerator() {} | |
3728 | }; | |
3729 | ||
3730 | class MixedGenerator : public ObjectGenerator { | |
3731 | public: | |
3732 | unsigned seq; | |
3733 | int64_t poolid; | |
3734 | explicit MixedGenerator(int64_t p) : seq(0), poolid(p) {} | |
3735 | ghobject_t create_object(gen_type *gen) override { | |
3736 | char buf[100]; | |
3737 | snprintf(buf, sizeof(buf), "OBJ_%u", seq); | |
3738 | string name(buf); | |
3739 | if (seq % 2) { | |
3740 | for (unsigned i = 0; i < 300; ++i) { | |
3741 | name.push_back('a'); | |
3742 | } | |
3743 | } | |
3744 | ++seq; | |
3745 | return ghobject_t( | |
3746 | hobject_t( | |
3747 | name, string(), rand() & 2 ? CEPH_NOSNAP : rand(), | |
3748 | (((seq / 1024) % 2) * 0xF00 ) + | |
3749 | (seq & 0xFF), | |
3750 | poolid, "")); | |
3751 | } | |
3752 | }; | |
3753 | ||
3754 | class SyntheticWorkloadState { | |
3755 | struct Object { | |
3756 | bufferlist data; | |
3757 | map<string, bufferlist> attrs; | |
3758 | }; | |
3759 | public: | |
3760 | static const unsigned max_in_flight = 16; | |
3761 | static const unsigned max_objects = 3000; | |
3762 | static const unsigned max_attr_size = 5; | |
3763 | static const unsigned max_attr_name_len = 100; | |
3764 | static const unsigned max_attr_value_len = 1024 * 64; | |
3765 | coll_t cid; | |
3766 | unsigned write_alignment; | |
3767 | unsigned max_object_len, max_write_len; | |
3768 | unsigned in_flight; | |
3769 | map<ghobject_t, Object> contents; | |
3770 | set<ghobject_t> available_objects; | |
f6b5b4d7 | 3771 | set<ghobject_t>::iterator next_available_object; |
7c673cae FG |
3772 | set<ghobject_t> in_flight_objects; |
3773 | ObjectGenerator *object_gen; | |
3774 | gen_type *rng; | |
3775 | ObjectStore *store; | |
11fdf7f2 | 3776 | ObjectStore::CollectionHandle ch; |
7c673cae | 3777 | |
9f95a23c TL |
3778 | ceph::mutex lock = ceph::make_mutex("State lock"); |
3779 | ceph::condition_variable cond; | |
7c673cae FG |
3780 | |
3781 | struct EnterExit { | |
3782 | const char *msg; | |
3783 | explicit EnterExit(const char *m) : msg(m) { | |
3784 | //cout << pthread_self() << " enter " << msg << std::endl; | |
3785 | } | |
3786 | ~EnterExit() { | |
3787 | //cout << pthread_self() << " exit " << msg << std::endl; | |
3788 | } | |
3789 | }; | |
3790 | ||
3791 | class C_SyntheticOnReadable : public Context { | |
3792 | public: | |
3793 | SyntheticWorkloadState *state; | |
3794 | ghobject_t hoid; | |
3795 | C_SyntheticOnReadable(SyntheticWorkloadState *state, ghobject_t hoid) | |
3796 | : state(state), hoid(hoid) {} | |
3797 | ||
3798 | void finish(int r) override { | |
9f95a23c | 3799 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3800 | EnterExit ee("onreadable finish"); |
3801 | ASSERT_TRUE(state->in_flight_objects.count(hoid)); | |
3802 | ASSERT_EQ(r, 0); | |
3803 | state->in_flight_objects.erase(hoid); | |
3804 | if (state->contents.count(hoid)) | |
3805 | state->available_objects.insert(hoid); | |
3806 | --(state->in_flight); | |
9f95a23c | 3807 | state->cond.notify_all(); |
7c673cae FG |
3808 | |
3809 | bufferlist r2; | |
11fdf7f2 TL |
3810 | r = state->store->read(state->ch, hoid, 0, state->contents[hoid].data.length(), r2); |
3811 | ceph_assert(bl_eq(state->contents[hoid].data, r2)); | |
9f95a23c | 3812 | state->cond.notify_all(); |
7c673cae FG |
3813 | } |
3814 | }; | |
3815 | ||
3816 | class C_SyntheticOnStash : public Context { | |
3817 | public: | |
3818 | SyntheticWorkloadState *state; | |
3819 | ghobject_t oid, noid; | |
3820 | ||
3821 | C_SyntheticOnStash(SyntheticWorkloadState *state, | |
3822 | ghobject_t oid, ghobject_t noid) | |
3823 | : state(state), oid(oid), noid(noid) {} | |
3824 | ||
3825 | void finish(int r) override { | |
9f95a23c | 3826 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3827 | EnterExit ee("stash finish"); |
3828 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3829 | ASSERT_EQ(r, 0); | |
3830 | state->in_flight_objects.erase(oid); | |
3831 | if (state->contents.count(noid)) | |
3832 | state->available_objects.insert(noid); | |
3833 | --(state->in_flight); | |
3834 | bufferlist r2; | |
3835 | r = state->store->read( | |
11fdf7f2 | 3836 | state->ch, noid, 0, |
7c673cae | 3837 | state->contents[noid].data.length(), r2); |
11fdf7f2 | 3838 | ceph_assert(bl_eq(state->contents[noid].data, r2)); |
9f95a23c | 3839 | state->cond.notify_all(); |
7c673cae FG |
3840 | } |
3841 | }; | |
3842 | ||
3843 | class C_SyntheticOnClone : public Context { | |
3844 | public: | |
3845 | SyntheticWorkloadState *state; | |
3846 | ghobject_t oid, noid; | |
3847 | ||
3848 | C_SyntheticOnClone(SyntheticWorkloadState *state, | |
3849 | ghobject_t oid, ghobject_t noid) | |
3850 | : state(state), oid(oid), noid(noid) {} | |
3851 | ||
3852 | void finish(int r) override { | |
9f95a23c | 3853 | std::lock_guard locker{state->lock}; |
7c673cae FG |
3854 | EnterExit ee("clone finish"); |
3855 | ASSERT_TRUE(state->in_flight_objects.count(oid)); | |
3856 | ASSERT_EQ(r, 0); | |
3857 | state->in_flight_objects.erase(oid); | |
3858 | if (state->contents.count(oid)) | |
3859 | state->available_objects.insert(oid); | |
3860 | if (state->contents.count(noid)) | |
3861 | state->available_objects.insert(noid); | |
3862 | --(state->in_flight); | |
3863 | bufferlist r2; | |
11fdf7f2 TL |
3864 | r = state->store->read(state->ch, noid, 0, state->contents[noid].data.length(), r2); |
3865 | ceph_assert(bl_eq(state->contents[noid].data, r2)); | |
9f95a23c | 3866 | state->cond.notify_all(); |
7c673cae FG |
3867 | } |
3868 | }; | |
3869 | ||
3870 | static void filled_byte_array(bufferlist& bl, size_t size) | |
3871 | { | |
3872 | static const char alphanum[] = "0123456789" | |
3873 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ" | |
3874 | "abcdefghijklmnopqrstuvwxyz"; | |
3875 | if (!size) { | |
3876 | return; | |
3877 | } | |
3878 | bufferptr bp(size); | |
3879 | for (unsigned int i = 0; i < size - 1; i++) { | |
3880 | // severely limit entropy so we can compress... | |
3881 | bp[i] = alphanum[rand() % 10]; //(sizeof(alphanum) - 1)]; | |
3882 | } | |
3883 | bp[size - 1] = '\0'; | |
3884 | ||
3885 | bl.append(bp); | |
3886 | } | |
3887 | ||
3888 | SyntheticWorkloadState(ObjectStore *store, | |
3889 | ObjectGenerator *gen, | |
3890 | gen_type *rng, | |
7c673cae FG |
3891 | coll_t cid, |
3892 | unsigned max_size, | |
3893 | unsigned max_write, | |
3894 | unsigned alignment) | |
3895 | : cid(cid), write_alignment(alignment), max_object_len(max_size), | |
f6b5b4d7 TL |
3896 | max_write_len(max_write), in_flight(0), |
3897 | next_available_object(available_objects.end()), | |
3898 | object_gen(gen), rng(rng), store(store) {} | |
7c673cae FG |
3899 | |
3900 | int init() { | |
3901 | ObjectStore::Transaction t; | |
11fdf7f2 | 3902 | ch = store->create_new_collection(cid); |
7c673cae | 3903 | t.create_collection(cid, 0); |
11fdf7f2 | 3904 | return queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3905 | } |
3906 | void shutdown() { | |
f6b5b4d7 | 3907 | ghobject_t next; |
7c673cae FG |
3908 | while (1) { |
3909 | vector<ghobject_t> objects; | |
f6b5b4d7 TL |
3910 | int r = store->collection_list(ch, next, ghobject_t::get_max(), |
3911 | 10, &objects, &next); | |
11fdf7f2 | 3912 | ceph_assert(r >= 0); |
f6b5b4d7 TL |
3913 | if (objects.size() == 0) |
3914 | break; | |
7c673cae | 3915 | ObjectStore::Transaction t; |
f6b5b4d7 | 3916 | std::map<std::string, ceph::buffer::list> attrset; |
7c673cae | 3917 | for (vector<ghobject_t>::iterator p = objects.begin(); |
f6b5b4d7 TL |
3918 | p != objects.end(); ++p) { |
3919 | t.remove(cid, *p); | |
7c673cae | 3920 | } |
11fdf7f2 | 3921 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3922 | } |
3923 | ObjectStore::Transaction t; | |
3924 | t.remove_collection(cid); | |
11fdf7f2 | 3925 | queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
3926 | } |
3927 | void statfs(store_statfs_t& stat) { | |
3928 | store->statfs(&stat); | |
3929 | } | |
3930 | ||
9f95a23c TL |
3931 | ghobject_t get_uniform_random_object(std::unique_lock<ceph::mutex>& locker) { |
3932 | cond.wait(locker, [this] { | |
3933 | return in_flight < max_in_flight && !available_objects.empty(); | |
3934 | }); | |
7c673cae FG |
3935 | boost::uniform_int<> choose(0, available_objects.size() - 1); |
3936 | int index = choose(*rng); | |
3937 | set<ghobject_t>::iterator i = available_objects.begin(); | |
3938 | for ( ; index > 0; --index, ++i) ; | |
3939 | ghobject_t ret = *i; | |
3940 | return ret; | |
3941 | } | |
3942 | ||
f6b5b4d7 TL |
3943 | ghobject_t get_next_object(std::unique_lock<ceph::mutex>& locker) { |
3944 | cond.wait(locker, [this] { | |
3945 | return in_flight < max_in_flight && !available_objects.empty(); | |
3946 | }); | |
3947 | ||
3948 | if (next_available_object == available_objects.end()) { | |
3949 | next_available_object = available_objects.begin(); | |
3950 | } | |
3951 | ||
3952 | ghobject_t ret = *next_available_object; | |
3953 | ++next_available_object; | |
3954 | return ret; | |
3955 | } | |
3956 | ||
9f95a23c TL |
3957 | void wait_for_ready(std::unique_lock<ceph::mutex>& locker) { |
3958 | cond.wait(locker, [this] { return in_flight < max_in_flight; }); | |
7c673cae FG |
3959 | } |
3960 | ||
3961 | void wait_for_done() { | |
9f95a23c TL |
3962 | std::unique_lock locker{lock}; |
3963 | cond.wait(locker, [this] { return in_flight == 0; }); | |
7c673cae FG |
3964 | } |
3965 | ||
3966 | bool can_create() { | |
3967 | return (available_objects.size() + in_flight_objects.size()) < max_objects; | |
3968 | } | |
3969 | ||
3970 | bool can_unlink() { | |
3971 | return (available_objects.size() + in_flight_objects.size()) > 0; | |
3972 | } | |
3973 | ||
3974 | unsigned get_random_alloc_hints() { | |
3975 | unsigned f = 0; | |
3976 | { | |
3977 | boost::uniform_int<> u(0, 3); | |
3978 | switch (u(*rng)) { | |
3979 | case 1: | |
3980 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE; | |
3981 | break; | |
3982 | case 2: | |
3983 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE; | |
3984 | break; | |
3985 | } | |
3986 | } | |
3987 | { | |
3988 | boost::uniform_int<> u(0, 3); | |
3989 | switch (u(*rng)) { | |
3990 | case 1: | |
3991 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ; | |
3992 | break; | |
3993 | case 2: | |
3994 | f |= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ; | |
3995 | break; | |
3996 | } | |
3997 | } | |
3998 | { | |
3999 | // append_only, immutable | |
4000 | boost::uniform_int<> u(0, 4); | |
4001 | f |= u(*rng) << 4; | |
4002 | } | |
4003 | { | |
4004 | boost::uniform_int<> u(0, 3); | |
4005 | switch (u(*rng)) { | |
4006 | case 1: | |
4007 | f |= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED; | |
4008 | break; | |
4009 | case 2: | |
4010 | f |= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED; | |
4011 | break; | |
4012 | } | |
4013 | } | |
4014 | { | |
4015 | boost::uniform_int<> u(0, 3); | |
4016 | switch (u(*rng)) { | |
4017 | case 1: | |
4018 | f |= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE; | |
4019 | break; | |
4020 | case 2: | |
4021 | f |= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE; | |
4022 | break; | |
4023 | } | |
4024 | } | |
4025 | return f; | |
4026 | } | |
4027 | ||
4028 | int touch() { | |
9f95a23c | 4029 | std::unique_lock locker{lock}; |
7c673cae FG |
4030 | EnterExit ee("touch"); |
4031 | if (!can_create()) | |
4032 | return -ENOSPC; | |
9f95a23c | 4033 | wait_for_ready(locker); |
7c673cae FG |
4034 | ghobject_t new_obj = object_gen->create_object(rng); |
4035 | available_objects.erase(new_obj); | |
4036 | ObjectStore::Transaction t; | |
4037 | t.touch(cid, new_obj); | |
4038 | boost::uniform_int<> u(17, 22); | |
4039 | boost::uniform_int<> v(12, 17); | |
4040 | t.set_alloc_hint(cid, new_obj, | |
4041 | 1ull << u(*rng), | |
4042 | 1ull << v(*rng), | |
4043 | get_random_alloc_hints()); | |
4044 | ++in_flight; | |
4045 | in_flight_objects.insert(new_obj); | |
4046 | if (!contents.count(new_obj)) | |
4047 | contents[new_obj] = Object(); | |
11fdf7f2 TL |
4048 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4049 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4050 | return status; |
4051 | } | |
4052 | ||
4053 | int stash() { | |
9f95a23c | 4054 | std::unique_lock locker{lock}; |
7c673cae FG |
4055 | EnterExit ee("stash"); |
4056 | if (!can_unlink()) | |
4057 | return -ENOENT; | |
4058 | if (!can_create()) | |
4059 | return -ENOSPC; | |
9f95a23c | 4060 | wait_for_ready(locker); |
7c673cae FG |
4061 | |
4062 | ghobject_t old_obj; | |
4063 | int max = 20; | |
4064 | do { | |
9f95a23c | 4065 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4066 | } while (--max && !contents[old_obj].data.length()); |
4067 | available_objects.erase(old_obj); | |
4068 | ghobject_t new_obj = old_obj; | |
4069 | new_obj.generation++; | |
4070 | available_objects.erase(new_obj); | |
4071 | ||
4072 | ObjectStore::Transaction t; | |
4073 | t.collection_move_rename(cid, old_obj, cid, new_obj); | |
4074 | ++in_flight; | |
4075 | in_flight_objects.insert(old_obj); | |
4076 | ||
4077 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4078 | contents[new_obj].data = contents[old_obj].data; | |
4079 | contents.erase(old_obj); | |
11fdf7f2 TL |
4080 | t.register_on_applied(new C_SyntheticOnStash(this, old_obj, new_obj)); |
4081 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4082 | return status; |
4083 | } | |
4084 | ||
4085 | int clone() { | |
9f95a23c | 4086 | std::unique_lock locker{lock}; |
7c673cae FG |
4087 | EnterExit ee("clone"); |
4088 | if (!can_unlink()) | |
4089 | return -ENOENT; | |
4090 | if (!can_create()) | |
4091 | return -ENOSPC; | |
9f95a23c | 4092 | wait_for_ready(locker); |
7c673cae FG |
4093 | |
4094 | ghobject_t old_obj; | |
4095 | int max = 20; | |
4096 | do { | |
9f95a23c | 4097 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4098 | } while (--max && !contents[old_obj].data.length()); |
4099 | available_objects.erase(old_obj); | |
4100 | ghobject_t new_obj = object_gen->create_object(rng); | |
4101 | // make the hash match | |
4102 | new_obj.hobj.set_hash(old_obj.hobj.get_hash()); | |
4103 | available_objects.erase(new_obj); | |
4104 | ||
4105 | ObjectStore::Transaction t; | |
4106 | t.clone(cid, old_obj, new_obj); | |
4107 | ++in_flight; | |
4108 | in_flight_objects.insert(old_obj); | |
4109 | ||
4110 | contents[new_obj].attrs = contents[old_obj].attrs; | |
4111 | contents[new_obj].data = contents[old_obj].data; | |
4112 | ||
11fdf7f2 TL |
4113 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4114 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4115 | return status; |
4116 | } | |
4117 | ||
4118 | int clone_range() { | |
9f95a23c | 4119 | std::unique_lock locker{lock}; |
7c673cae FG |
4120 | EnterExit ee("clone_range"); |
4121 | if (!can_unlink()) | |
4122 | return -ENOENT; | |
4123 | if (!can_create()) | |
4124 | return -ENOSPC; | |
9f95a23c | 4125 | wait_for_ready(locker); |
7c673cae FG |
4126 | |
4127 | ghobject_t old_obj; | |
4128 | int max = 20; | |
4129 | do { | |
9f95a23c | 4130 | old_obj = get_uniform_random_object(locker); |
7c673cae FG |
4131 | } while (--max && !contents[old_obj].data.length()); |
4132 | bufferlist &srcdata = contents[old_obj].data; | |
4133 | if (srcdata.length() == 0) { | |
4134 | return 0; | |
4135 | } | |
4136 | available_objects.erase(old_obj); | |
9f95a23c | 4137 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4138 | available_objects.erase(new_obj); |
4139 | ||
4140 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4141 | boost::uniform_int<> u2(0, max_write_len); | |
4142 | uint64_t srcoff = u1(*rng); | |
4143 | // make src and dst offsets match, since that's what the osd does | |
4144 | uint64_t dstoff = srcoff; //u1(*rng); | |
4145 | uint64_t len = u2(*rng); | |
4146 | if (write_alignment) { | |
11fdf7f2 TL |
4147 | srcoff = round_up_to(srcoff, write_alignment); |
4148 | dstoff = round_up_to(dstoff, write_alignment); | |
4149 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4150 | } |
4151 | ||
4152 | if (srcoff > srcdata.length() - 1) { | |
4153 | srcoff = srcdata.length() - 1; | |
4154 | } | |
4155 | if (srcoff + len > srcdata.length()) { | |
4156 | len = srcdata.length() - srcoff; | |
4157 | } | |
4158 | if (0) | |
4159 | cout << __func__ << " from " << srcoff << "~" << len | |
4160 | << " (size " << srcdata.length() << ") to " | |
4161 | << dstoff << "~" << len << std::endl; | |
4162 | ||
4163 | ObjectStore::Transaction t; | |
4164 | t.clone_range(cid, old_obj, new_obj, srcoff, len, dstoff); | |
4165 | ++in_flight; | |
4166 | in_flight_objects.insert(old_obj); | |
4167 | ||
4168 | bufferlist bl; | |
4169 | if (srcoff < srcdata.length()) { | |
4170 | if (srcoff + len > srcdata.length()) { | |
4171 | bl.substr_of(srcdata, srcoff, srcdata.length() - srcoff); | |
4172 | } else { | |
4173 | bl.substr_of(srcdata, srcoff, len); | |
4174 | } | |
4175 | } | |
4176 | ||
4177 | bufferlist& dstdata = contents[new_obj].data; | |
4178 | if (dstdata.length() <= dstoff) { | |
4179 | if (bl.length() > 0) { | |
4180 | dstdata.append_zero(dstoff - dstdata.length()); | |
4181 | dstdata.append(bl); | |
4182 | } | |
4183 | } else { | |
4184 | bufferlist value; | |
11fdf7f2 | 4185 | ceph_assert(dstdata.length() > dstoff); |
9f95a23c | 4186 | dstdata.cbegin().copy(dstoff, value); |
7c673cae FG |
4187 | value.append(bl); |
4188 | if (value.length() < dstdata.length()) | |
9f95a23c TL |
4189 | dstdata.cbegin(value.length()).copy( |
4190 | dstdata.length() - value.length(), value); | |
7c673cae FG |
4191 | value.swap(dstdata); |
4192 | } | |
4193 | ||
11fdf7f2 TL |
4194 | t.register_on_applied(new C_SyntheticOnClone(this, old_obj, new_obj)); |
4195 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4196 | return status; |
4197 | } | |
4198 | ||
4199 | ||
4200 | int write() { | |
9f95a23c | 4201 | std::unique_lock locker{lock}; |
7c673cae FG |
4202 | EnterExit ee("write"); |
4203 | if (!can_unlink()) | |
4204 | return -ENOENT; | |
9f95a23c | 4205 | wait_for_ready(locker); |
7c673cae | 4206 | |
9f95a23c | 4207 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4208 | available_objects.erase(new_obj); |
4209 | ObjectStore::Transaction t; | |
4210 | ||
4211 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4212 | boost::uniform_int<> u2(0, max_write_len); | |
4213 | uint64_t offset = u1(*rng); | |
4214 | uint64_t len = u2(*rng); | |
4215 | bufferlist bl; | |
4216 | if (write_alignment) { | |
11fdf7f2 TL |
4217 | offset = round_up_to(offset, write_alignment); |
4218 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4219 | } |
4220 | ||
4221 | filled_byte_array(bl, len); | |
4222 | ||
4223 | bufferlist& data = contents[new_obj].data; | |
4224 | if (data.length() <= offset) { | |
4225 | if (len > 0) { | |
4226 | data.append_zero(offset-data.length()); | |
4227 | data.append(bl); | |
4228 | } | |
4229 | } else { | |
4230 | bufferlist value; | |
11fdf7f2 | 4231 | ceph_assert(data.length() > offset); |
9f95a23c | 4232 | data.cbegin().copy(offset, value); |
7c673cae FG |
4233 | value.append(bl); |
4234 | if (value.length() < data.length()) | |
9f95a23c TL |
4235 | data.cbegin(value.length()).copy( |
4236 | data.length()-value.length(), value); | |
7c673cae FG |
4237 | value.swap(data); |
4238 | } | |
4239 | ||
4240 | t.write(cid, new_obj, offset, len, bl); | |
4241 | ++in_flight; | |
4242 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4243 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4244 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4245 | return status; |
4246 | } | |
4247 | ||
4248 | int truncate() { | |
9f95a23c | 4249 | std::unique_lock locker{lock}; |
7c673cae FG |
4250 | EnterExit ee("truncate"); |
4251 | if (!can_unlink()) | |
4252 | return -ENOENT; | |
9f95a23c | 4253 | wait_for_ready(locker); |
7c673cae | 4254 | |
9f95a23c | 4255 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4256 | available_objects.erase(obj); |
4257 | ObjectStore::Transaction t; | |
4258 | ||
4259 | boost::uniform_int<> choose(0, max_object_len); | |
4260 | size_t len = choose(*rng); | |
4261 | if (write_alignment) { | |
11fdf7f2 | 4262 | len = round_up_to(len, write_alignment); |
7c673cae FG |
4263 | } |
4264 | ||
4265 | t.truncate(cid, obj, len); | |
4266 | ++in_flight; | |
4267 | in_flight_objects.insert(obj); | |
4268 | bufferlist& data = contents[obj].data; | |
4269 | if (data.length() <= len) { | |
4270 | data.append_zero(len - data.length()); | |
4271 | } else { | |
4272 | bufferlist bl; | |
9f95a23c | 4273 | data.cbegin().copy(len, bl); |
7c673cae FG |
4274 | bl.swap(data); |
4275 | } | |
4276 | ||
11fdf7f2 TL |
4277 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4278 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4279 | return status; |
4280 | } | |
4281 | ||
4282 | int zero() { | |
9f95a23c | 4283 | std::unique_lock locker{lock}; |
7c673cae FG |
4284 | EnterExit ee("zero"); |
4285 | if (!can_unlink()) | |
4286 | return -ENOENT; | |
9f95a23c | 4287 | wait_for_ready(locker); |
7c673cae | 4288 | |
9f95a23c | 4289 | ghobject_t new_obj = get_uniform_random_object(locker); |
7c673cae FG |
4290 | available_objects.erase(new_obj); |
4291 | ObjectStore::Transaction t; | |
4292 | ||
4293 | boost::uniform_int<> u1(0, max_object_len - max_write_len); | |
4294 | boost::uniform_int<> u2(0, max_write_len); | |
4295 | uint64_t offset = u1(*rng); | |
4296 | uint64_t len = u2(*rng); | |
4297 | if (write_alignment) { | |
11fdf7f2 TL |
4298 | offset = round_up_to(offset, write_alignment); |
4299 | len = round_up_to(len, write_alignment); | |
7c673cae FG |
4300 | } |
4301 | ||
b32b8144 FG |
4302 | if (len > 0) { |
4303 | auto& data = contents[new_obj].data; | |
4304 | if (data.length() < offset + len) { | |
4305 | data.append_zero(offset+len-data.length()); | |
4306 | } | |
4307 | bufferlist n; | |
4308 | n.substr_of(data, 0, offset); | |
4309 | n.append_zero(len); | |
4310 | if (data.length() > offset + len) | |
9f95a23c | 4311 | data.cbegin(offset + len).copy(data.length() - offset - len, n); |
b32b8144 | 4312 | data.swap(n); |
7c673cae | 4313 | } |
7c673cae FG |
4314 | |
4315 | t.zero(cid, new_obj, offset, len); | |
4316 | ++in_flight; | |
4317 | in_flight_objects.insert(new_obj); | |
11fdf7f2 TL |
4318 | t.register_on_applied(new C_SyntheticOnReadable(this, new_obj)); |
4319 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4320 | return status; |
4321 | } | |
4322 | ||
4323 | void read() { | |
4324 | EnterExit ee("read"); | |
4325 | boost::uniform_int<> u1(0, max_object_len/2); | |
4326 | boost::uniform_int<> u2(0, max_object_len); | |
4327 | uint64_t offset = u1(*rng); | |
4328 | uint64_t len = u2(*rng); | |
4329 | if (offset > len) | |
4330 | swap(offset, len); | |
4331 | ||
4332 | ghobject_t obj; | |
4333 | bufferlist expected; | |
4334 | int r; | |
4335 | { | |
9f95a23c | 4336 | std::unique_lock locker{lock}; |
7c673cae FG |
4337 | EnterExit ee("read locked"); |
4338 | if (!can_unlink()) | |
4339 | return ; | |
9f95a23c | 4340 | wait_for_ready(locker); |
7c673cae | 4341 | |
9f95a23c | 4342 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4343 | expected = contents[obj].data; |
4344 | } | |
4345 | bufferlist bl, result; | |
4346 | if (0) cout << " obj " << obj | |
4347 | << " size " << expected.length() | |
4348 | << " offset " << offset | |
4349 | << " len " << len << std::endl; | |
11fdf7f2 | 4350 | r = store->read(ch, obj, offset, len, result); |
7c673cae FG |
4351 | if (offset >= expected.length()) { |
4352 | ASSERT_EQ(r, 0); | |
4353 | } else { | |
4354 | size_t max_len = expected.length() - offset; | |
4355 | if (len > max_len) | |
4356 | len = max_len; | |
11fdf7f2 | 4357 | ceph_assert(len == result.length()); |
7c673cae | 4358 | ASSERT_EQ(len, result.length()); |
9f95a23c | 4359 | expected.cbegin(offset).copy(len, bl); |
7c673cae FG |
4360 | ASSERT_EQ(r, (int)len); |
4361 | ASSERT_TRUE(bl_eq(bl, result)); | |
4362 | } | |
4363 | } | |
4364 | ||
4365 | int setattrs() { | |
9f95a23c | 4366 | std::unique_lock locker{lock}; |
7c673cae FG |
4367 | EnterExit ee("setattrs"); |
4368 | if (!can_unlink()) | |
4369 | return -ENOENT; | |
9f95a23c | 4370 | wait_for_ready(locker); |
7c673cae | 4371 | |
9f95a23c | 4372 | ghobject_t obj = get_uniform_random_object(locker); |
7c673cae FG |
4373 | available_objects.erase(obj); |
4374 | ObjectStore::Transaction t; | |
4375 | ||
4376 | boost::uniform_int<> u0(1, max_attr_size); | |
4377 | boost::uniform_int<> u1(4, max_attr_name_len); | |
4378 | boost::uniform_int<> u2(4, max_attr_value_len); | |
4379 | boost::uniform_int<> u3(0, 100); | |
4380 | uint64_t size = u0(*rng); | |
4381 | uint64_t name_len; | |
4382 | map<string, bufferlist> attrs; | |
4383 | set<string> keys; | |
4384 | for (map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4385 | it != contents[obj].attrs.end(); ++it) | |
4386 | keys.insert(it->first); | |
4387 | ||
4388 | while (size--) { | |
4389 | bufferlist name, value; | |
4390 | uint64_t get_exist = u3(*rng); | |
4391 | uint64_t value_len = u2(*rng); | |
4392 | filled_byte_array(value, value_len); | |
4393 | if (get_exist < 50 && keys.size()) { | |
4394 | set<string>::iterator k = keys.begin(); | |
4395 | attrs[*k] = value; | |
4396 | contents[obj].attrs[*k] = value; | |
4397 | keys.erase(k); | |
4398 | } else { | |
4399 | name_len = u1(*rng); | |
4400 | filled_byte_array(name, name_len); | |
4401 | attrs[name.c_str()] = value; | |
4402 | contents[obj].attrs[name.c_str()] = value; | |
4403 | } | |
4404 | } | |
4405 | t.setattrs(cid, obj, attrs); | |
4406 | ++in_flight; | |
4407 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4408 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4409 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4410 | return status; |
4411 | } | |
4412 | ||
f6b5b4d7 TL |
4413 | int set_fixed_attrs(size_t entries, size_t key_size, size_t val_size) { |
4414 | std::unique_lock locker{ lock }; | |
4415 | EnterExit ee("setattrs"); | |
4416 | if (!can_unlink()) | |
4417 | return -ENOENT; | |
4418 | wait_for_ready(locker); | |
4419 | ||
4420 | ghobject_t obj = get_next_object(locker); | |
4421 | available_objects.erase(obj); | |
4422 | ObjectStore::Transaction t; | |
4423 | ||
4424 | map<string, bufferlist> attrs; | |
4425 | set<string> keys; | |
4426 | ||
4427 | while (entries--) { | |
4428 | bufferlist name, value; | |
4429 | filled_byte_array(value, val_size); | |
4430 | filled_byte_array(name, key_size); | |
4431 | attrs[name.c_str()] = value; | |
4432 | contents[obj].attrs[name.c_str()] = value; | |
4433 | } | |
4434 | t.setattrs(cid, obj, attrs); | |
4435 | ++in_flight; | |
4436 | in_flight_objects.insert(obj); | |
4437 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); | |
4438 | int status = store->queue_transaction(ch, std::move(t)); | |
4439 | return status; | |
4440 | } | |
4441 | ||
7c673cae FG |
4442 | void getattrs() { |
4443 | EnterExit ee("getattrs"); | |
4444 | ghobject_t obj; | |
4445 | map<string, bufferlist> expected; | |
4446 | { | |
9f95a23c | 4447 | std::unique_lock locker{lock}; |
7c673cae FG |
4448 | EnterExit ee("getattrs locked"); |
4449 | if (!can_unlink()) | |
4450 | return ; | |
9f95a23c | 4451 | wait_for_ready(locker); |
7c673cae FG |
4452 | |
4453 | int retry = 10; | |
4454 | do { | |
9f95a23c | 4455 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4456 | if (!--retry) |
4457 | return ; | |
4458 | } while (contents[obj].attrs.empty()); | |
4459 | expected = contents[obj].attrs; | |
4460 | } | |
4461 | map<string, bufferlist> attrs; | |
11fdf7f2 | 4462 | int r = store->getattrs(ch, obj, attrs); |
7c673cae FG |
4463 | ASSERT_TRUE(r == 0); |
4464 | ASSERT_TRUE(attrs.size() == expected.size()); | |
4465 | for (map<string, bufferlist>::iterator it = expected.begin(); | |
4466 | it != expected.end(); ++it) { | |
4467 | ASSERT_TRUE(bl_eq(attrs[it->first], it->second)); | |
4468 | } | |
4469 | } | |
4470 | ||
4471 | void getattr() { | |
4472 | EnterExit ee("getattr"); | |
4473 | ghobject_t obj; | |
4474 | int r; | |
4475 | int retry; | |
4476 | map<string, bufferlist> expected; | |
4477 | { | |
9f95a23c | 4478 | std::unique_lock locker{lock}; |
7c673cae FG |
4479 | EnterExit ee("getattr locked"); |
4480 | if (!can_unlink()) | |
4481 | return ; | |
9f95a23c | 4482 | wait_for_ready(locker); |
7c673cae FG |
4483 | |
4484 | retry = 10; | |
4485 | do { | |
9f95a23c | 4486 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4487 | if (!--retry) |
4488 | return ; | |
4489 | } while (contents[obj].attrs.empty()); | |
4490 | expected = contents[obj].attrs; | |
4491 | } | |
4492 | boost::uniform_int<> u(0, expected.size()-1); | |
4493 | retry = u(*rng); | |
4494 | map<string, bufferlist>::iterator it = expected.begin(); | |
4495 | while (retry) { | |
4496 | retry--; | |
4497 | ++it; | |
4498 | } | |
4499 | ||
4500 | bufferlist bl; | |
11fdf7f2 | 4501 | r = store->getattr(ch, obj, it->first, bl); |
7c673cae FG |
4502 | ASSERT_EQ(r, 0); |
4503 | ASSERT_TRUE(bl_eq(it->second, bl)); | |
4504 | } | |
4505 | ||
4506 | int rmattr() { | |
9f95a23c | 4507 | std::unique_lock locker{lock}; |
7c673cae FG |
4508 | EnterExit ee("rmattr"); |
4509 | if (!can_unlink()) | |
4510 | return -ENOENT; | |
9f95a23c | 4511 | wait_for_ready(locker); |
7c673cae FG |
4512 | |
4513 | ghobject_t obj; | |
4514 | int retry = 10; | |
4515 | do { | |
9f95a23c | 4516 | obj = get_uniform_random_object(locker); |
7c673cae FG |
4517 | if (!--retry) |
4518 | return 0; | |
4519 | } while (contents[obj].attrs.empty()); | |
4520 | ||
4521 | boost::uniform_int<> u(0, contents[obj].attrs.size()-1); | |
4522 | retry = u(*rng); | |
4523 | map<string, bufferlist>::iterator it = contents[obj].attrs.begin(); | |
4524 | while (retry) { | |
4525 | retry--; | |
4526 | ++it; | |
4527 | } | |
4528 | ||
4529 | available_objects.erase(obj); | |
4530 | ObjectStore::Transaction t; | |
4531 | t.rmattr(cid, obj, it->first); | |
4532 | ||
4533 | contents[obj].attrs.erase(it->first); | |
4534 | ++in_flight; | |
4535 | in_flight_objects.insert(obj); | |
11fdf7f2 TL |
4536 | t.register_on_applied(new C_SyntheticOnReadable(this, obj)); |
4537 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4538 | return status; |
4539 | } | |
4540 | ||
4541 | void fsck(bool deep) { | |
9f95a23c | 4542 | std::unique_lock locker{lock}; |
7c673cae | 4543 | EnterExit ee("fsck"); |
9f95a23c | 4544 | cond.wait(locker, [this] { return in_flight == 0; }); |
11fdf7f2 | 4545 | ch.reset(); |
7c673cae FG |
4546 | store->umount(); |
4547 | int r = store->fsck(deep); | |
11fdf7f2 | 4548 | ceph_assert(r == 0 || r == -EOPNOTSUPP); |
7c673cae | 4549 | store->mount(); |
11fdf7f2 | 4550 | ch = store->open_collection(cid); |
7c673cae FG |
4551 | } |
4552 | ||
4553 | void scan() { | |
9f95a23c | 4554 | std::unique_lock locker{lock}; |
7c673cae | 4555 | EnterExit ee("scan"); |
9f95a23c | 4556 | cond.wait(locker, [this] { return in_flight == 0; }); |
7c673cae FG |
4557 | vector<ghobject_t> objects; |
4558 | set<ghobject_t> objects_set, objects_set2; | |
4559 | ghobject_t next, current; | |
4560 | while (1) { | |
4561 | //cerr << "scanning..." << std::endl; | |
11fdf7f2 | 4562 | int r = store->collection_list(ch, current, ghobject_t::get_max(), 100, |
7c673cae FG |
4563 | &objects, &next); |
4564 | ASSERT_EQ(r, 0); | |
4565 | ASSERT_TRUE(sorted(objects)); | |
4566 | objects_set.insert(objects.begin(), objects.end()); | |
4567 | objects.clear(); | |
4568 | if (next.is_max()) break; | |
4569 | current = next; | |
4570 | } | |
4571 | if (objects_set.size() != available_objects.size()) { | |
4572 | for (set<ghobject_t>::iterator p = objects_set.begin(); | |
4573 | p != objects_set.end(); | |
4574 | ++p) | |
4575 | if (available_objects.count(*p) == 0) { | |
4576 | cerr << "+ " << *p << std::endl; | |
4577 | ceph_abort(); | |
4578 | } | |
4579 | for (set<ghobject_t>::iterator p = available_objects.begin(); | |
4580 | p != available_objects.end(); | |
4581 | ++p) | |
4582 | if (objects_set.count(*p) == 0) | |
4583 | cerr << "- " << *p << std::endl; | |
4584 | //cerr << " objects_set: " << objects_set << std::endl; | |
4585 | //cerr << " available_set: " << available_objects << std::endl; | |
11fdf7f2 | 4586 | ceph_abort_msg("badness"); |
7c673cae FG |
4587 | } |
4588 | ||
4589 | ASSERT_EQ(objects_set.size(), available_objects.size()); | |
4590 | for (set<ghobject_t>::iterator i = objects_set.begin(); | |
4591 | i != objects_set.end(); | |
4592 | ++i) { | |
4593 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4594 | } | |
4595 | ||
11fdf7f2 | 4596 | int r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), |
7c673cae FG |
4597 | INT_MAX, &objects, 0); |
4598 | ASSERT_EQ(r, 0); | |
4599 | objects_set2.insert(objects.begin(), objects.end()); | |
4600 | ASSERT_EQ(objects_set2.size(), available_objects.size()); | |
4601 | for (set<ghobject_t>::iterator i = objects_set2.begin(); | |
4602 | i != objects_set2.end(); | |
4603 | ++i) { | |
4604 | ASSERT_GT(available_objects.count(*i), (unsigned)0); | |
4605 | if (available_objects.count(*i) == 0) { | |
4606 | cerr << "+ " << *i << std::endl; | |
4607 | } | |
4608 | } | |
4609 | } | |
4610 | ||
4611 | void stat() { | |
4612 | EnterExit ee("stat"); | |
4613 | ghobject_t hoid; | |
4614 | uint64_t expected; | |
4615 | { | |
9f95a23c | 4616 | std::unique_lock locker{lock}; |
7c673cae FG |
4617 | EnterExit ee("stat lock1"); |
4618 | if (!can_unlink()) | |
4619 | return ; | |
9f95a23c | 4620 | hoid = get_uniform_random_object(locker); |
7c673cae FG |
4621 | in_flight_objects.insert(hoid); |
4622 | available_objects.erase(hoid); | |
4623 | ++in_flight; | |
4624 | expected = contents[hoid].data.length(); | |
4625 | } | |
4626 | struct stat buf; | |
11fdf7f2 | 4627 | int r = store->stat(ch, hoid, &buf); |
7c673cae | 4628 | ASSERT_EQ(0, r); |
11fdf7f2 | 4629 | ceph_assert((uint64_t)buf.st_size == expected); |
7c673cae FG |
4630 | ASSERT_TRUE((uint64_t)buf.st_size == expected); |
4631 | { | |
9f95a23c | 4632 | std::lock_guard locker{lock}; |
7c673cae FG |
4633 | EnterExit ee("stat lock2"); |
4634 | --in_flight; | |
9f95a23c | 4635 | cond.notify_all(); |
7c673cae FG |
4636 | in_flight_objects.erase(hoid); |
4637 | available_objects.insert(hoid); | |
4638 | } | |
4639 | } | |
4640 | ||
4641 | int unlink() { | |
9f95a23c | 4642 | std::unique_lock locker{lock}; |
7c673cae FG |
4643 | EnterExit ee("unlink"); |
4644 | if (!can_unlink()) | |
4645 | return -ENOENT; | |
9f95a23c | 4646 | ghobject_t to_remove = get_uniform_random_object(locker); |
7c673cae FG |
4647 | ObjectStore::Transaction t; |
4648 | t.remove(cid, to_remove); | |
4649 | ++in_flight; | |
4650 | available_objects.erase(to_remove); | |
4651 | in_flight_objects.insert(to_remove); | |
4652 | contents.erase(to_remove); | |
11fdf7f2 TL |
4653 | t.register_on_applied(new C_SyntheticOnReadable(this, to_remove)); |
4654 | int status = store->queue_transaction(ch, std::move(t)); | |
7c673cae FG |
4655 | return status; |
4656 | } | |
4657 | ||
4658 | void print_internal_state() { | |
9f95a23c | 4659 | std::lock_guard locker{lock}; |
7c673cae FG |
4660 | cerr << "available_objects: " << available_objects.size() |
4661 | << " in_flight_objects: " << in_flight_objects.size() | |
4662 | << " total objects: " << in_flight_objects.size() + available_objects.size() | |
4663 | << " in_flight " << in_flight << std::endl; | |
4664 | } | |
4665 | }; | |
4666 | ||
4667 | ||
11fdf7f2 | 4668 | void StoreTest::doSyntheticTest( |
7c673cae FG |
4669 | int num_ops, |
4670 | uint64_t max_obj, uint64_t max_wr, uint64_t align) | |
4671 | { | |
7c673cae FG |
4672 | MixedGenerator gen(555); |
4673 | gen_type rng(time(NULL)); | |
4674 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
4675 | ||
11fdf7f2 TL |
4676 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); |
4677 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
4678 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae | 4679 | |
11fdf7f2 | 4680 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, |
7c673cae FG |
4681 | max_obj, max_wr, align); |
4682 | test_obj.init(); | |
4683 | for (int i = 0; i < num_ops/10; ++i) { | |
4684 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
4685 | test_obj.touch(); | |
4686 | } | |
4687 | for (int i = 0; i < num_ops; ++i) { | |
4688 | if (!(i % 1000)) { | |
4689 | cerr << "Op " << i << std::endl; | |
4690 | test_obj.print_internal_state(); | |
4691 | } | |
4692 | boost::uniform_int<> true_false(0, 999); | |
4693 | int val = true_false(rng); | |
4694 | if (val > 998) { | |
4695 | test_obj.fsck(true); | |
4696 | } else if (val > 997) { | |
4697 | test_obj.fsck(false); | |
4698 | } else if (val > 970) { | |
4699 | test_obj.scan(); | |
4700 | } else if (val > 950) { | |
4701 | test_obj.stat(); | |
4702 | } else if (val > 850) { | |
4703 | test_obj.zero(); | |
4704 | } else if (val > 800) { | |
4705 | test_obj.unlink(); | |
4706 | } else if (val > 550) { | |
4707 | test_obj.write(); | |
4708 | } else if (val > 500) { | |
4709 | test_obj.clone(); | |
4710 | } else if (val > 450) { | |
4711 | test_obj.clone_range(); | |
4712 | } else if (val > 300) { | |
4713 | test_obj.stash(); | |
4714 | } else if (val > 100) { | |
4715 | test_obj.read(); | |
4716 | } else { | |
4717 | test_obj.truncate(); | |
4718 | } | |
4719 | } | |
4720 | test_obj.wait_for_done(); | |
4721 | test_obj.shutdown(); | |
7c673cae FG |
4722 | } |
4723 | ||
4724 | TEST_P(StoreTest, Synthetic) { | |
11fdf7f2 | 4725 | doSyntheticTest(10000, 400*1024, 40*1024, 0); |
7c673cae FG |
4726 | } |
4727 | ||
9f95a23c | 4728 | #if defined(WITH_BLUESTORE) |
11fdf7f2 TL |
4729 | TEST_P(StoreTestSpecificAUSize, BlueFSExtenderTest) { |
4730 | if(string(GetParam()) != "bluestore") | |
4731 | return; | |
4732 | ||
4733 | SetVal(g_conf(), "bluestore_block_db_size", "0"); | |
4734 | SetVal(g_conf(), "bluestore_block_wal_size", "0"); | |
4735 | SetVal(g_conf(), "bluestore_bluefs_min", "12582912"); | |
4736 | SetVal(g_conf(), "bluestore_bluefs_min_free", "4194304"); | |
4737 | SetVal(g_conf(), "bluestore_bluefs_gift_ratio", "0"); | |
4738 | SetVal(g_conf(), "bluestore_bluefs_min_ratio", "0"); | |
4739 | SetVal(g_conf(), "bluestore_bluefs_balance_interval", "100000"); | |
4740 | SetVal(g_conf(), "bluestore_bluefs_db_compatibility", "false"); | |
4741 | ||
4742 | g_conf().apply_changes(nullptr); | |
4743 | ||
4744 | StartDeferred(4096); | |
7c673cae | 4745 | |
11fdf7f2 TL |
4746 | doSyntheticTest(10000, 400*1024, 40*1024, 0); |
4747 | ||
4748 | BlueStore* bstore = NULL; | |
4749 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
4750 | ||
4751 | // verify downgrades are broken and repair that | |
4752 | bstore->umount(); | |
4753 | ASSERT_EQ(bstore->fsck(false), 0); | |
4754 | ||
4755 | SetVal(g_conf(), "bluestore_bluefs_db_compatibility", "true"); | |
4756 | g_conf().apply_changes(nullptr); | |
4757 | ||
4758 | ASSERT_EQ(bstore->fsck(false), 1); | |
4759 | ASSERT_EQ(bstore->repair(false), 0); | |
4760 | ASSERT_EQ(bstore->fsck(false), 0); | |
4761 | bstore->mount(); | |
4762 | } | |
4763 | ||
7c673cae FG |
4764 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixSharding) { |
4765 | if (string(GetParam()) != "bluestore") | |
4766 | return; | |
4767 | ||
4768 | const char *m[][10] = { | |
4769 | { "bluestore_min_alloc_size", "4096", 0 }, // must be the first! | |
4770 | { "num_ops", "50000", 0 }, | |
4771 | { "max_write", "65536", 0 }, | |
4772 | { "max_size", "262144", 0 }, | |
4773 | { "alignment", "4096", 0 }, | |
4774 | { "bluestore_max_blob_size", "65536", 0 }, | |
4775 | { "bluestore_extent_map_shard_min_size", "60", 0 }, | |
4776 | { "bluestore_extent_map_shard_max_size", "300", 0 }, | |
4777 | { "bluestore_extent_map_shard_target_size", "150", 0 }, | |
4778 | { "bluestore_default_buffered_read", "true", 0 }, | |
4779 | { "bluestore_default_buffered_write", "true", 0 }, | |
4780 | { 0 }, | |
4781 | }; | |
11fdf7f2 | 4782 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4783 | } |
4784 | ||
4785 | TEST_P(StoreTestSpecificAUSize, ZipperPatternSharded) { | |
4786 | if(string(GetParam()) != "bluestore") | |
4787 | return; | |
4788 | StartDeferred(4096); | |
4789 | ||
4790 | int r; | |
7c673cae FG |
4791 | coll_t cid; |
4792 | ghobject_t a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 4793 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
4794 | { |
4795 | ObjectStore::Transaction t; | |
4796 | t.create_collection(cid, 0); | |
4797 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 4798 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4799 | ASSERT_EQ(r, 0); |
4800 | } | |
4801 | bufferlist bl; | |
4802 | int len = 4096; | |
4803 | bufferptr bp(len); | |
4804 | bp.zero(); | |
4805 | bl.append(bp); | |
4806 | for (int i=0; i<1000; ++i) { | |
4807 | ObjectStore::Transaction t; | |
4808 | t.write(cid, a, i*2*len, len, bl, 0); | |
11fdf7f2 | 4809 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4810 | ASSERT_EQ(r, 0); |
4811 | } | |
4812 | for (int i=0; i<1000; ++i) { | |
4813 | ObjectStore::Transaction t; | |
4814 | t.write(cid, a, i*2*len + 1, len, bl, 0); | |
11fdf7f2 | 4815 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4816 | ASSERT_EQ(r, 0); |
4817 | } | |
4818 | { | |
4819 | ObjectStore::Transaction t; | |
4820 | t.remove(cid, a); | |
4821 | t.remove_collection(cid); | |
4822 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 4823 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4824 | ASSERT_EQ(r, 0); |
4825 | } | |
4826 | } | |
4827 | ||
4828 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumAlgorithm) { | |
4829 | if (string(GetParam()) != "bluestore") | |
4830 | return; | |
4831 | ||
4832 | const char *m[][10] = { | |
4833 | { "bluestore_min_alloc_size", "65536", 0 }, // must be the first! | |
4834 | { "max_write", "65536", 0 }, | |
4835 | { "max_size", "1048576", 0 }, | |
4836 | { "alignment", "16", 0 }, | |
4837 | { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32", | |
4838 | "xxhash64", "none", 0 }, | |
4839 | { "bluestore_default_buffered_write", "false", 0 }, | |
4840 | { 0 }, | |
4841 | }; | |
11fdf7f2 | 4842 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4843 | } |
4844 | ||
4845 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCsumVsCompression) { | |
4846 | if (string(GetParam()) != "bluestore") | |
4847 | return; | |
4848 | ||
4849 | const char *m[][10] = { | |
4850 | { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first! | |
4851 | { "max_write", "131072", 0 }, | |
4852 | { "max_size", "262144", 0 }, | |
4853 | { "alignment", "512", 0 }, | |
4854 | { "bluestore_compression_mode", "force", 0}, | |
4855 | { "bluestore_compression_algorithm", "snappy", "zlib", 0 }, | |
4856 | { "bluestore_csum_type", "crc32c", 0 }, | |
4857 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4858 | { "bluestore_default_buffered_write", "true", "false", 0 }, | |
4859 | { "bluestore_sync_submit_transaction", "false", 0 }, | |
4860 | { 0 }, | |
4861 | }; | |
11fdf7f2 | 4862 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4863 | } |
4864 | ||
4865 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompression) { | |
4866 | if (string(GetParam()) != "bluestore") | |
4867 | return; | |
4868 | ||
4869 | const char *m[][10] = { | |
4870 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4871 | { "max_write", "1048576", 0 }, | |
4872 | { "max_size", "4194304", 0 }, | |
4873 | { "alignment", "65536", 0 }, | |
4874 | { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0}, | |
4875 | { "bluestore_default_buffered_write", "false", 0 }, | |
4876 | { "bluestore_sync_submit_transaction", "true", 0 }, | |
4877 | { 0 }, | |
4878 | }; | |
11fdf7f2 | 4879 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4880 | } |
4881 | ||
4882 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixCompressionAlgorithm) { | |
4883 | if (string(GetParam()) != "bluestore") | |
4884 | return; | |
4885 | ||
4886 | const char *m[][10] = { | |
4887 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4888 | { "max_write", "1048576", 0 }, | |
4889 | { "max_size", "4194304", 0 }, | |
4890 | { "alignment", "65536", 0 }, | |
4891 | { "bluestore_compression_algorithm", "zlib", "snappy", 0 }, | |
4892 | { "bluestore_compression_mode", "force", 0 }, | |
4893 | { "bluestore_default_buffered_write", "false", 0 }, | |
4894 | { 0 }, | |
4895 | }; | |
11fdf7f2 | 4896 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4897 | } |
4898 | ||
4899 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixNoCsum) { | |
4900 | if (string(GetParam()) != "bluestore") | |
4901 | return; | |
4902 | ||
4903 | const char *m[][10] = { | |
4904 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4905 | { "max_write", "65536", 0 }, | |
4906 | { "max_size", "1048576", 0 }, | |
4907 | { "alignment", "512", 0 }, | |
4908 | { "bluestore_max_blob_size", "262144", 0 }, | |
4909 | { "bluestore_compression_mode", "force", "none", 0}, | |
4910 | { "bluestore_csum_type", "none", 0}, | |
4911 | { "bluestore_default_buffered_read", "true", "false", 0 }, | |
4912 | { "bluestore_default_buffered_write", "true", 0 }, | |
4913 | { "bluestore_sync_submit_transaction", "true", "false", 0 }, | |
4914 | { 0 }, | |
4915 | }; | |
11fdf7f2 | 4916 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae FG |
4917 | } |
4918 | ||
4919 | TEST_P(StoreTestSpecificAUSize, SyntheticMatrixPreferDeferred) { | |
4920 | if (string(GetParam()) != "bluestore") | |
4921 | return; | |
4922 | ||
4923 | const char *m[][10] = { | |
4924 | { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first! | |
4925 | { "max_write", "65536", 0 }, | |
4926 | { "max_size", "1048576", 0 }, | |
4927 | { "alignment", "512", 0 }, | |
4928 | { "bluestore_max_blob_size", "262144", 0 }, | |
4929 | { "bluestore_compression_mode", "force", "none", 0}, | |
4930 | { "bluestore_prefer_deferred_size", "32768", "0", 0}, | |
4931 | { 0 }, | |
4932 | }; | |
11fdf7f2 | 4933 | do_matrix(m, std::bind(&StoreTest::doSyntheticTest, this, _1, _2, _3, _4)); |
7c673cae | 4934 | } |
11fdf7f2 | 4935 | #endif // WITH_BLUESTORE |
7c673cae FG |
4936 | |
4937 | TEST_P(StoreTest, AttrSynthetic) { | |
7c673cae FG |
4938 | MixedGenerator gen(447); |
4939 | gen_type rng(time(NULL)); | |
4940 | coll_t cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD)); | |
4941 | ||
11fdf7f2 | 4942 | SyntheticWorkloadState test_obj(store.get(), &gen, &rng, cid, 40*1024, 4*1024, 0); |
7c673cae FG |
4943 | test_obj.init(); |
4944 | for (int i = 0; i < 500; ++i) { | |
4945 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; | |
4946 | test_obj.touch(); | |
4947 | } | |
4948 | for (int i = 0; i < 1000; ++i) { | |
4949 | if (!(i % 100)) { | |
4950 | cerr << "Op " << i << std::endl; | |
4951 | test_obj.print_internal_state(); | |
4952 | } | |
4953 | boost::uniform_int<> true_false(0, 99); | |
4954 | int val = true_false(rng); | |
4955 | if (val > 97) { | |
4956 | test_obj.scan(); | |
4957 | } else if (val > 93) { | |
4958 | test_obj.stat(); | |
4959 | } else if (val > 75) { | |
4960 | test_obj.rmattr(); | |
4961 | } else if (val > 47) { | |
4962 | test_obj.setattrs(); | |
4963 | } else if (val > 45) { | |
4964 | test_obj.clone(); | |
4965 | } else if (val > 37) { | |
4966 | test_obj.stash(); | |
4967 | } else if (val > 30) { | |
4968 | test_obj.getattrs(); | |
4969 | } else { | |
4970 | test_obj.getattr(); | |
4971 | } | |
4972 | } | |
4973 | test_obj.wait_for_done(); | |
4974 | test_obj.shutdown(); | |
4975 | } | |
4976 | ||
4977 | TEST_P(StoreTest, HashCollisionTest) { | |
7c673cae FG |
4978 | int64_t poolid = 11; |
4979 | coll_t cid(spg_t(pg_t(0,poolid),shard_id_t::NO_SHARD)); | |
4980 | int r; | |
11fdf7f2 | 4981 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
4982 | { |
4983 | ObjectStore::Transaction t; | |
4984 | t.create_collection(cid, 0); | |
11fdf7f2 | 4985 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
4986 | ASSERT_EQ(r, 0); |
4987 | } | |
4988 | string base = ""; | |
4989 | for (int i = 0; i < 100; ++i) base.append("aaaaa"); | |
4990 | set<ghobject_t> created; | |
4991 | for (int n = 0; n < 10; ++n) { | |
4992 | char nbuf[100]; | |
4993 | sprintf(nbuf, "n%d", n); | |
4994 | for (int i = 0; i < 1000; ++i) { | |
4995 | char buf[100]; | |
4996 | sprintf(buf, "%d", i); | |
4997 | if (!(i % 100)) { | |
4998 | cerr << "Object n" << n << " "<< i << std::endl; | |
4999 | } | |
5000 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, 0, poolid, string(nbuf))); | |
5001 | { | |
5002 | ObjectStore::Transaction t; | |
5003 | t.touch(cid, hoid); | |
11fdf7f2 | 5004 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5005 | ASSERT_EQ(r, 0); |
5006 | } | |
5007 | created.insert(hoid); | |
5008 | } | |
5009 | } | |
5010 | vector<ghobject_t> objects; | |
11fdf7f2 | 5011 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), INT_MAX, &objects, 0); |
7c673cae FG |
5012 | ASSERT_EQ(r, 0); |
5013 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5014 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5015 | ASSERT_TRUE(listed.size() == created.size()); | |
5016 | objects.clear(); | |
5017 | listed.clear(); | |
5018 | ghobject_t current, next; | |
5019 | while (1) { | |
11fdf7f2 | 5020 | r = store->collection_list(ch, current, ghobject_t::get_max(), 60, |
7c673cae FG |
5021 | &objects, &next); |
5022 | ASSERT_EQ(r, 0); | |
5023 | ASSERT_TRUE(sorted(objects)); | |
5024 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5025 | i != objects.end(); | |
5026 | ++i) { | |
5027 | if (listed.count(*i)) | |
5028 | cerr << *i << " repeated" << std::endl; | |
5029 | listed.insert(*i); | |
5030 | } | |
5031 | if (objects.size() < 50) { | |
5032 | ASSERT_TRUE(next.is_max()); | |
5033 | break; | |
5034 | } | |
5035 | objects.clear(); | |
5036 | current = next; | |
5037 | } | |
5038 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5039 | ASSERT_TRUE(listed.size() == created.size()); | |
5040 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5041 | i != listed.end(); | |
5042 | ++i) { | |
5043 | ASSERT_TRUE(created.count(*i)); | |
5044 | } | |
5045 | ||
5046 | for (set<ghobject_t>::iterator i = created.begin(); | |
5047 | i != created.end(); | |
5048 | ++i) { | |
5049 | ObjectStore::Transaction t; | |
5050 | t.remove(cid, *i); | |
11fdf7f2 | 5051 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5052 | ASSERT_EQ(r, 0); |
5053 | } | |
5054 | ObjectStore::Transaction t; | |
5055 | t.remove_collection(cid); | |
11fdf7f2 | 5056 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5057 | ASSERT_EQ(r, 0); |
5058 | } | |
5059 | ||
5060 | TEST_P(StoreTest, ScrubTest) { | |
7c673cae FG |
5061 | int64_t poolid = 111; |
5062 | coll_t cid(spg_t(pg_t(0, poolid),shard_id_t(1))); | |
5063 | int r; | |
11fdf7f2 | 5064 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5065 | { |
5066 | ObjectStore::Transaction t; | |
5067 | t.create_collection(cid, 0); | |
11fdf7f2 | 5068 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5069 | ASSERT_EQ(r, 0); |
5070 | } | |
5071 | string base = "aaaaa"; | |
5072 | set<ghobject_t> created; | |
5073 | for (int i = 0; i < 1000; ++i) { | |
5074 | char buf[100]; | |
5075 | sprintf(buf, "%d", i); | |
5076 | if (!(i % 5)) { | |
5077 | cerr << "Object " << i << std::endl; | |
5078 | } | |
5079 | ghobject_t hoid(hobject_t(string(buf) + base, string(), CEPH_NOSNAP, i, | |
5080 | poolid, ""), | |
5081 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5082 | { | |
5083 | ObjectStore::Transaction t; | |
5084 | t.touch(cid, hoid); | |
11fdf7f2 | 5085 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5086 | ASSERT_EQ(r, 0); |
5087 | } | |
5088 | created.insert(hoid); | |
5089 | } | |
5090 | ||
5091 | // Add same hobject_t but different generation | |
5092 | { | |
5093 | ghobject_t hoid1(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), | |
5094 | ghobject_t::NO_GEN, shard_id_t(1)); | |
5095 | ghobject_t hoid2(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)1, shard_id_t(1)); | |
5096 | ghobject_t hoid3(hobject_t("same-object", string(), CEPH_NOSNAP, 0, poolid, ""), (gen_t)2, shard_id_t(1)); | |
5097 | ObjectStore::Transaction t; | |
5098 | t.touch(cid, hoid1); | |
5099 | t.touch(cid, hoid2); | |
5100 | t.touch(cid, hoid3); | |
11fdf7f2 | 5101 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5102 | created.insert(hoid1); |
5103 | created.insert(hoid2); | |
5104 | created.insert(hoid3); | |
5105 | ASSERT_EQ(r, 0); | |
5106 | } | |
5107 | ||
5108 | vector<ghobject_t> objects; | |
11fdf7f2 | 5109 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), |
7c673cae FG |
5110 | INT_MAX, &objects, 0); |
5111 | ASSERT_EQ(r, 0); | |
5112 | set<ghobject_t> listed(objects.begin(), objects.end()); | |
5113 | cerr << "listed.size() is " << listed.size() << " and created.size() is " << created.size() << std::endl; | |
5114 | ASSERT_TRUE(listed.size() == created.size()); | |
5115 | objects.clear(); | |
5116 | listed.clear(); | |
5117 | ghobject_t current, next; | |
5118 | while (1) { | |
11fdf7f2 | 5119 | r = store->collection_list(ch, current, ghobject_t::get_max(), 60, |
7c673cae FG |
5120 | &objects, &next); |
5121 | ASSERT_EQ(r, 0); | |
5122 | ASSERT_TRUE(sorted(objects)); | |
5123 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5124 | i != objects.end(); ++i) { | |
5125 | if (listed.count(*i)) | |
5126 | cerr << *i << " repeated" << std::endl; | |
5127 | listed.insert(*i); | |
5128 | } | |
5129 | if (objects.size() < 50) { | |
5130 | ASSERT_TRUE(next.is_max()); | |
5131 | break; | |
5132 | } | |
5133 | objects.clear(); | |
5134 | current = next.get_boundary(); | |
5135 | } | |
5136 | cerr << "listed.size() is " << listed.size() << std::endl; | |
5137 | ASSERT_TRUE(listed.size() == created.size()); | |
5138 | for (set<ghobject_t>::iterator i = listed.begin(); | |
5139 | i != listed.end(); | |
5140 | ++i) { | |
5141 | ASSERT_TRUE(created.count(*i)); | |
5142 | } | |
5143 | ||
5144 | for (set<ghobject_t>::iterator i = created.begin(); | |
5145 | i != created.end(); | |
5146 | ++i) { | |
5147 | ObjectStore::Transaction t; | |
5148 | t.remove(cid, *i); | |
11fdf7f2 | 5149 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5150 | ASSERT_EQ(r, 0); |
5151 | } | |
5152 | ObjectStore::Transaction t; | |
5153 | t.remove_collection(cid); | |
11fdf7f2 | 5154 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5155 | ASSERT_EQ(r, 0); |
5156 | } | |
5157 | ||
5158 | ||
5159 | TEST_P(StoreTest, OMapTest) { | |
7c673cae FG |
5160 | coll_t cid; |
5161 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 5162 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5163 | int r; |
5164 | { | |
5165 | ObjectStore::Transaction t; | |
5166 | t.create_collection(cid, 0); | |
11fdf7f2 | 5167 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5168 | ASSERT_EQ(r, 0); |
5169 | } | |
5170 | ||
5171 | map<string, bufferlist> attrs; | |
5172 | { | |
5173 | ObjectStore::Transaction t; | |
5174 | t.touch(cid, hoid); | |
5175 | t.omap_clear(cid, hoid); | |
5176 | map<string, bufferlist> start_set; | |
5177 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5178 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5179 | ASSERT_EQ(r, 0); |
7c673cae FG |
5180 | } |
5181 | ||
5182 | for (int i = 0; i < 100; i++) { | |
5183 | if (!(i%5)) { | |
5184 | std::cout << "On iteration " << i << std::endl; | |
5185 | } | |
5186 | ObjectStore::Transaction t; | |
5187 | bufferlist bl; | |
5188 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5189 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5190 | ASSERT_EQ(r, 0); |
5191 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5192 | j != attrs.end(); | |
5193 | ++j) { | |
5194 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5195 | if (!correct) { | |
5196 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5197 | if (cur_attrs.count(j->first) > 0) { | |
5198 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5199 | } | |
5200 | } | |
5201 | ASSERT_EQ(correct, true); | |
5202 | } | |
5203 | ASSERT_EQ(attrs.size(), cur_attrs.size()); | |
5204 | ||
5205 | char buf[100]; | |
5206 | snprintf(buf, sizeof(buf), "%d", i); | |
5207 | bl.clear(); | |
5208 | bufferptr bp(buf, strlen(buf) + 1); | |
5209 | bl.append(bp); | |
5210 | map<string, bufferlist> to_add; | |
5211 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5212 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5213 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5214 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5215 | ASSERT_EQ(r, 0); |
7c673cae FG |
5216 | } |
5217 | ||
5218 | int i = 0; | |
5219 | while (attrs.size()) { | |
5220 | if (!(i%5)) { | |
5221 | std::cout << "removal: On iteration " << i << std::endl; | |
5222 | } | |
5223 | ObjectStore::Transaction t; | |
5224 | bufferlist bl; | |
5225 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5226 | r = store->omap_get(ch, hoid, &bl, &cur_attrs); |
7c673cae FG |
5227 | ASSERT_EQ(r, 0); |
5228 | for (map<string, bufferlist>::iterator j = attrs.begin(); | |
5229 | j != attrs.end(); | |
5230 | ++j) { | |
5231 | bool correct = cur_attrs.count(j->first) && string(cur_attrs[j->first].c_str()) == string(j->second.c_str()); | |
5232 | if (!correct) { | |
5233 | std::cout << j->first << " is present in cur_attrs " << cur_attrs.count(j->first) << " times " << std::endl; | |
5234 | if (cur_attrs.count(j->first) > 0) { | |
5235 | std::cout << j->second.c_str() << " : " << cur_attrs[j->first].c_str() << std::endl; | |
5236 | } | |
5237 | } | |
5238 | ASSERT_EQ(correct, true); | |
5239 | } | |
5240 | ||
5241 | string to_remove = attrs.begin()->first; | |
9f95a23c | 5242 | t.omap_rmkey(cid, hoid, to_remove); |
11fdf7f2 | 5243 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5244 | ASSERT_EQ(r, 0); |
7c673cae FG |
5245 | |
5246 | attrs.erase(to_remove); | |
5247 | ||
5248 | ++i; | |
5249 | } | |
5250 | ||
5251 | { | |
5252 | bufferlist bl1; | |
5253 | bl1.append("omap_header"); | |
5254 | ObjectStore::Transaction t; | |
5255 | t.omap_setheader(cid, hoid, bl1); | |
11fdf7f2 | 5256 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5257 | ASSERT_EQ(r, 0); |
7c673cae FG |
5258 | t = ObjectStore::Transaction(); |
5259 | ||
5260 | bufferlist bl2; | |
5261 | bl2.append("value"); | |
5262 | map<string, bufferlist> to_add; | |
5263 | to_add.insert(pair<string, bufferlist>("key", bl2)); | |
5264 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5265 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5266 | ASSERT_EQ(r, 0); |
7c673cae FG |
5267 | |
5268 | bufferlist bl3; | |
5269 | map<string, bufferlist> cur_attrs; | |
11fdf7f2 | 5270 | r = store->omap_get(ch, hoid, &bl3, &cur_attrs); |
7c673cae FG |
5271 | ASSERT_EQ(r, 0); |
5272 | ASSERT_EQ(cur_attrs.size(), size_t(1)); | |
5273 | ASSERT_TRUE(bl_eq(bl1, bl3)); | |
5274 | ||
5275 | set<string> keys; | |
11fdf7f2 | 5276 | r = store->omap_get_keys(ch, hoid, &keys); |
7c673cae FG |
5277 | ASSERT_EQ(r, 0); |
5278 | ASSERT_EQ(keys.size(), size_t(1)); | |
5279 | } | |
5280 | ||
5281 | // test omap_clear, omap_rmkey_range | |
5282 | { | |
5283 | { | |
5284 | map<string,bufferlist> to_set; | |
5285 | for (int n=0; n<10; ++n) { | |
5286 | to_set[stringify(n)].append("foo"); | |
5287 | } | |
5288 | bufferlist h; | |
5289 | h.append("header"); | |
5290 | ObjectStore::Transaction t; | |
5291 | t.remove(cid, hoid); | |
5292 | t.touch(cid, hoid); | |
5293 | t.omap_setheader(cid, hoid, h); | |
5294 | t.omap_setkeys(cid, hoid, to_set); | |
11fdf7f2 | 5295 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5296 | ASSERT_EQ(r, 0); |
7c673cae FG |
5297 | } |
5298 | { | |
5299 | ObjectStore::Transaction t; | |
5300 | t.omap_rmkeyrange(cid, hoid, "3", "7"); | |
11fdf7f2 | 5301 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5302 | ASSERT_EQ(r, 0); |
7c673cae FG |
5303 | } |
5304 | { | |
5305 | bufferlist hdr; | |
5306 | map<string,bufferlist> m; | |
11fdf7f2 | 5307 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5308 | ASSERT_EQ(6u, hdr.length()); |
5309 | ASSERT_TRUE(m.count("2")); | |
5310 | ASSERT_TRUE(!m.count("3")); | |
5311 | ASSERT_TRUE(!m.count("6")); | |
5312 | ASSERT_TRUE(m.count("7")); | |
5313 | ASSERT_TRUE(m.count("8")); | |
5314 | //cout << m << std::endl; | |
5315 | ASSERT_EQ(6u, m.size()); | |
5316 | } | |
5317 | { | |
5318 | ObjectStore::Transaction t; | |
5319 | t.omap_clear(cid, hoid); | |
11fdf7f2 | 5320 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5321 | ASSERT_EQ(r, 0); |
7c673cae FG |
5322 | } |
5323 | { | |
5324 | bufferlist hdr; | |
5325 | map<string,bufferlist> m; | |
11fdf7f2 | 5326 | store->omap_get(ch, hoid, &hdr, &m); |
7c673cae FG |
5327 | ASSERT_EQ(0u, hdr.length()); |
5328 | ASSERT_EQ(0u, m.size()); | |
5329 | } | |
5330 | } | |
5331 | ||
5332 | ObjectStore::Transaction t; | |
5333 | t.remove(cid, hoid); | |
5334 | t.remove_collection(cid); | |
11fdf7f2 | 5335 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5336 | ASSERT_EQ(r, 0); |
5337 | } | |
5338 | ||
5339 | TEST_P(StoreTest, OMapIterator) { | |
7c673cae FG |
5340 | coll_t cid; |
5341 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5342 | int count = 0; | |
11fdf7f2 | 5343 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5344 | int r; |
5345 | { | |
5346 | ObjectStore::Transaction t; | |
5347 | t.create_collection(cid, 0); | |
11fdf7f2 | 5348 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5349 | ASSERT_EQ(r, 0); |
5350 | } | |
5351 | ||
5352 | map<string, bufferlist> attrs; | |
5353 | { | |
5354 | ObjectStore::Transaction t; | |
5355 | t.touch(cid, hoid); | |
5356 | t.omap_clear(cid, hoid); | |
5357 | map<string, bufferlist> start_set; | |
5358 | t.omap_setkeys(cid, hoid, start_set); | |
11fdf7f2 | 5359 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5360 | ASSERT_EQ(r, 0); |
7c673cae FG |
5361 | } |
5362 | ObjectMap::ObjectMapIterator iter; | |
5363 | bool correct; | |
5364 | //basic iteration | |
5365 | for (int i = 0; i < 100; i++) { | |
5366 | if (!(i%5)) { | |
5367 | std::cout << "On iteration " << i << std::endl; | |
5368 | } | |
5369 | bufferlist bl; | |
5370 | ||
5371 | // FileStore may deadlock two active iterators over the same data | |
5372 | iter = ObjectMap::ObjectMapIterator(); | |
5373 | ||
11fdf7f2 | 5374 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5375 | for (iter->seek_to_first(), count=0; iter->valid(); iter->next(), count++) { |
5376 | string key = iter->key(); | |
5377 | bufferlist value = iter->value(); | |
5378 | correct = attrs.count(key) && (string(value.c_str()) == string(attrs[key].c_str())); | |
5379 | if (!correct) { | |
5380 | if (attrs.count(key) > 0) { | |
5381 | std::cout << "key " << key << "in omap , " << value.c_str() << " : " << attrs[key].c_str() << std::endl; | |
5382 | } | |
5383 | else | |
5384 | std::cout << "key " << key << "should not exists in omap" << std::endl; | |
5385 | } | |
5386 | ASSERT_EQ(correct, true); | |
5387 | } | |
5388 | ASSERT_EQ((int)attrs.size(), count); | |
5389 | ||
11fdf7f2 | 5390 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5391 | iter = ObjectMap::ObjectMapIterator(); |
5392 | ||
5393 | char buf[100]; | |
5394 | snprintf(buf, sizeof(buf), "%d", i); | |
5395 | bl.clear(); | |
5396 | bufferptr bp(buf, strlen(buf) + 1); | |
5397 | bl.append(bp); | |
5398 | map<string, bufferlist> to_add; | |
5399 | to_add.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5400 | attrs.insert(pair<string, bufferlist>("key-" + string(buf), bl)); | |
5401 | ObjectStore::Transaction t; | |
5402 | t.omap_setkeys(cid, hoid, to_add); | |
11fdf7f2 | 5403 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 5404 | ASSERT_EQ(r, 0); |
7c673cae FG |
5405 | } |
5406 | ||
11fdf7f2 | 5407 | iter = store->get_omap_iterator(ch, hoid); |
7c673cae FG |
5408 | //lower bound |
5409 | string bound_key = "key-5"; | |
5410 | iter->lower_bound(bound_key); | |
5411 | correct = bound_key <= iter->key(); | |
5412 | if (!correct) { | |
5413 | std::cout << "lower bound, bound key is " << bound_key << " < iter key is " << iter->key() << std::endl; | |
5414 | } | |
5415 | ASSERT_EQ(correct, true); | |
5416 | //upper bound | |
5417 | iter->upper_bound(bound_key); | |
5418 | correct = iter->key() > bound_key; | |
5419 | if (!correct) { | |
5420 | std::cout << "upper bound, bound key is " << bound_key << " >= iter key is " << iter->key() << std::endl; | |
5421 | } | |
5422 | ASSERT_EQ(correct, true); | |
5423 | ||
11fdf7f2 | 5424 | // FileStore may deadlock an active iterator vs queue_transaction |
7c673cae FG |
5425 | iter = ObjectMap::ObjectMapIterator(); |
5426 | { | |
5427 | ObjectStore::Transaction t; | |
5428 | t.remove(cid, hoid); | |
5429 | t.remove_collection(cid); | |
11fdf7f2 | 5430 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5431 | ASSERT_EQ(r, 0); |
5432 | } | |
5433 | } | |
5434 | ||
5435 | TEST_P(StoreTest, XattrTest) { | |
7c673cae FG |
5436 | coll_t cid; |
5437 | ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, 0, "")); | |
5438 | bufferlist big; | |
5439 | for (unsigned i = 0; i < 10000; ++i) { | |
5440 | big.append('\0'); | |
5441 | } | |
5442 | bufferlist small; | |
5443 | for (unsigned i = 0; i < 10; ++i) { | |
5444 | small.append('\0'); | |
5445 | } | |
5446 | int r; | |
11fdf7f2 | 5447 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5448 | { |
5449 | ObjectStore::Transaction t; | |
5450 | t.create_collection(cid, 0); | |
5451 | t.touch(cid, hoid); | |
11fdf7f2 | 5452 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5453 | ASSERT_EQ(r, 0); |
5454 | } | |
5455 | ||
5456 | map<string, bufferlist> attrs; | |
5457 | { | |
5458 | ObjectStore::Transaction t; | |
5459 | t.setattr(cid, hoid, "attr1", small); | |
5460 | attrs["attr1"] = small; | |
5461 | t.setattr(cid, hoid, "attr2", big); | |
5462 | attrs["attr2"] = big; | |
5463 | t.setattr(cid, hoid, "attr3", small); | |
5464 | attrs["attr3"] = small; | |
5465 | t.setattr(cid, hoid, "attr1", small); | |
5466 | attrs["attr1"] = small; | |
5467 | t.setattr(cid, hoid, "attr4", big); | |
5468 | attrs["attr4"] = big; | |
5469 | t.setattr(cid, hoid, "attr3", big); | |
5470 | attrs["attr3"] = big; | |
11fdf7f2 | 5471 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5472 | ASSERT_EQ(r, 0); |
5473 | } | |
5474 | ||
5475 | map<string, bufferptr> aset; | |
11fdf7f2 | 5476 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5477 | ASSERT_EQ(aset.size(), attrs.size()); |
5478 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5479 | i != aset.end(); | |
5480 | ++i) { | |
5481 | bufferlist bl; | |
5482 | bl.push_back(i->second); | |
5483 | ASSERT_TRUE(attrs[i->first] == bl); | |
5484 | } | |
5485 | ||
5486 | { | |
5487 | ObjectStore::Transaction t; | |
5488 | t.rmattr(cid, hoid, "attr2"); | |
5489 | attrs.erase("attr2"); | |
11fdf7f2 | 5490 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5491 | ASSERT_EQ(r, 0); |
5492 | } | |
5493 | ||
5494 | aset.clear(); | |
11fdf7f2 | 5495 | store->getattrs(ch, hoid, aset); |
7c673cae FG |
5496 | ASSERT_EQ(aset.size(), attrs.size()); |
5497 | for (map<string, bufferptr>::iterator i = aset.begin(); | |
5498 | i != aset.end(); | |
5499 | ++i) { | |
5500 | bufferlist bl; | |
5501 | bl.push_back(i->second); | |
5502 | ASSERT_TRUE(attrs[i->first] == bl); | |
5503 | } | |
5504 | ||
5505 | bufferptr bp; | |
11fdf7f2 | 5506 | r = store->getattr(ch, hoid, "attr2", bp); |
7c673cae FG |
5507 | ASSERT_EQ(r, -ENODATA); |
5508 | ||
11fdf7f2 | 5509 | r = store->getattr(ch, hoid, "attr3", bp); |
7c673cae FG |
5510 | ASSERT_EQ(r, 0); |
5511 | bufferlist bl2; | |
5512 | bl2.push_back(bp); | |
5513 | ASSERT_TRUE(bl2 == attrs["attr3"]); | |
5514 | ||
5515 | ObjectStore::Transaction t; | |
5516 | t.remove(cid, hoid); | |
5517 | t.remove_collection(cid); | |
11fdf7f2 | 5518 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5519 | ASSERT_EQ(r, 0); |
5520 | } | |
5521 | ||
5522 | void colsplittest( | |
5523 | ObjectStore *store, | |
5524 | unsigned num_objects, | |
5525 | unsigned common_suffix_size, | |
5526 | bool clones | |
5527 | ) { | |
7c673cae FG |
5528 | coll_t cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD)); |
5529 | coll_t tid(spg_t(pg_t(1<<common_suffix_size,52),shard_id_t::NO_SHARD)); | |
11fdf7f2 TL |
5530 | auto ch = store->create_new_collection(cid); |
5531 | auto tch = store->create_new_collection(tid); | |
7c673cae FG |
5532 | int r = 0; |
5533 | { | |
5534 | ObjectStore::Transaction t; | |
5535 | t.create_collection(cid, common_suffix_size); | |
11fdf7f2 | 5536 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5537 | ASSERT_EQ(r, 0); |
5538 | } | |
5539 | bufferlist small; | |
5540 | small.append("small"); | |
5541 | { | |
5542 | ObjectStore::Transaction t; | |
5543 | for (uint32_t i = 0; i < (2 - (int)clones)*num_objects; ++i) { | |
5544 | stringstream objname; | |
5545 | objname << "obj" << i; | |
5546 | ghobject_t a(hobject_t( | |
5547 | objname.str(), | |
5548 | "", | |
5549 | CEPH_NOSNAP, | |
5550 | i<<common_suffix_size, | |
5551 | 52, "")); | |
5552 | t.write(cid, a, 0, small.length(), small, | |
5553 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
5554 | if (clones) { | |
5555 | objname << "-clone"; | |
5556 | ghobject_t b(hobject_t( | |
5557 | objname.str(), | |
5558 | "", | |
5559 | CEPH_NOSNAP, | |
5560 | i<<common_suffix_size, | |
5561 | 52, "")); | |
5562 | t.clone(cid, a, b); | |
5563 | } | |
5564 | if (i % 100) { | |
11fdf7f2 | 5565 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5566 | ASSERT_EQ(r, 0); |
5567 | t = ObjectStore::Transaction(); | |
5568 | } | |
5569 | } | |
11fdf7f2 | 5570 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5571 | ASSERT_EQ(r, 0); |
5572 | } | |
5573 | { | |
5574 | ObjectStore::Transaction t; | |
5575 | t.create_collection(tid, common_suffix_size + 1); | |
5576 | t.split_collection(cid, common_suffix_size+1, 1<<common_suffix_size, tid); | |
11fdf7f2 | 5577 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5578 | ASSERT_EQ(r, 0); |
5579 | } | |
11fdf7f2 | 5580 | ch->flush(); |
7c673cae | 5581 | |
11fdf7f2 | 5582 | // check |
7c673cae | 5583 | vector<ghobject_t> objects; |
11fdf7f2 | 5584 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), |
7c673cae FG |
5585 | INT_MAX, &objects, 0); |
5586 | ASSERT_EQ(r, 0); | |
5587 | ASSERT_EQ(objects.size(), num_objects); | |
7c673cae FG |
5588 | for (vector<ghobject_t>::iterator i = objects.begin(); |
5589 | i != objects.end(); | |
5590 | ++i) { | |
5591 | ASSERT_EQ(!!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
5592 | } |
5593 | ||
5594 | objects.clear(); | |
11fdf7f2 | 5595 | r = store->collection_list(tch, ghobject_t(), ghobject_t::get_max(), |
7c673cae FG |
5596 | INT_MAX, &objects, 0); |
5597 | ASSERT_EQ(r, 0); | |
5598 | ASSERT_EQ(objects.size(), num_objects); | |
5599 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5600 | i != objects.end(); | |
5601 | ++i) { | |
5602 | ASSERT_EQ(!(i->hobj.get_hash() & (1<<common_suffix_size)), 0u); | |
7c673cae FG |
5603 | } |
5604 | ||
11fdf7f2 TL |
5605 | // merge them again! |
5606 | { | |
5607 | ObjectStore::Transaction t; | |
5608 | t.merge_collection(tid, cid, common_suffix_size); | |
5609 | r = queue_transaction(store, ch, std::move(t)); | |
5610 | ASSERT_EQ(r, 0); | |
5611 | } | |
5612 | ||
5613 | // check and clean up | |
5614 | ObjectStore::Transaction t; | |
5615 | { | |
5616 | vector<ghobject_t> objects; | |
5617 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), | |
5618 | INT_MAX, &objects, 0); | |
5619 | ASSERT_EQ(r, 0); | |
5620 | ASSERT_EQ(objects.size(), num_objects * 2); // both halves | |
5621 | unsigned size = 0; | |
5622 | for (vector<ghobject_t>::iterator i = objects.begin(); | |
5623 | i != objects.end(); | |
5624 | ++i) { | |
5625 | t.remove(cid, *i); | |
5626 | if (++size > 100) { | |
5627 | size = 0; | |
5628 | r = queue_transaction(store, ch, std::move(t)); | |
5629 | ASSERT_EQ(r, 0); | |
5630 | t = ObjectStore::Transaction(); | |
5631 | } | |
5632 | } | |
5633 | } | |
7c673cae | 5634 | t.remove_collection(cid); |
11fdf7f2 | 5635 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae | 5636 | ASSERT_EQ(r, 0); |
11fdf7f2 TL |
5637 | |
5638 | ch->flush(); | |
5639 | ASSERT_TRUE(!store->collection_exists(tid)); | |
7c673cae FG |
5640 | } |
5641 | ||
11fdf7f2 TL |
5642 | TEST_P(StoreTest, ColSplitTest0) { |
5643 | colsplittest(store.get(), 10, 5, false); | |
5644 | } | |
7c673cae FG |
5645 | TEST_P(StoreTest, ColSplitTest1) { |
5646 | colsplittest(store.get(), 10000, 11, false); | |
5647 | } | |
5648 | TEST_P(StoreTest, ColSplitTest1Clones) { | |
5649 | colsplittest(store.get(), 10000, 11, true); | |
5650 | } | |
5651 | TEST_P(StoreTest, ColSplitTest2) { | |
5652 | colsplittest(store.get(), 100, 7, false); | |
5653 | } | |
5654 | TEST_P(StoreTest, ColSplitTest2Clones) { | |
5655 | colsplittest(store.get(), 100, 7, true); | |
5656 | } | |
5657 | ||
5658 | #if 0 | |
5659 | TEST_P(StoreTest, ColSplitTest3) { | |
5660 | colsplittest(store.get(), 100000, 25); | |
5661 | } | |
5662 | #endif | |
5663 | ||
11fdf7f2 TL |
5664 | void test_merge_skewed(ObjectStore *store, |
5665 | unsigned base, unsigned bits, | |
5666 | unsigned anum, unsigned bnum) | |
5667 | { | |
5668 | cout << __func__ << " 0x" << std::hex << base << std::dec | |
5669 | << " bits " << bits | |
5670 | << " anum " << anum << " bnum " << bnum << std::endl; | |
5671 | /* | |
5672 | make merge source pgs have radically different # of objects in them, | |
5673 | which should trigger different splitting in filestore, and verify that | |
5674 | post-merge all objects are accessible. | |
5675 | */ | |
5676 | int r; | |
5677 | coll_t a(spg_t(pg_t(base, 0), shard_id_t::NO_SHARD)); | |
5678 | coll_t b(spg_t(pg_t(base | (1<<bits), 0), shard_id_t::NO_SHARD)); | |
5679 | ||
5680 | auto cha = store->create_new_collection(a); | |
5681 | auto chb = store->create_new_collection(b); | |
5682 | { | |
5683 | ObjectStore::Transaction t; | |
5684 | t.create_collection(a, bits + 1); | |
5685 | r = queue_transaction(store, cha, std::move(t)); | |
5686 | ASSERT_EQ(r, 0); | |
5687 | } | |
5688 | { | |
5689 | ObjectStore::Transaction t; | |
5690 | t.create_collection(b, bits + 1); | |
5691 | r = queue_transaction(store, chb, std::move(t)); | |
5692 | ASSERT_EQ(r, 0); | |
5693 | } | |
5694 | ||
5695 | bufferlist small; | |
5696 | small.append("small"); | |
5697 | string suffix = "ooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooaaaaaaaaaa"; | |
5698 | set<ghobject_t> aobjects, bobjects; | |
5699 | { | |
5700 | // fill a | |
5701 | ObjectStore::Transaction t; | |
5702 | for (unsigned i = 0; i < 1000; ++i) { | |
5703 | string objname = "a" + stringify(i) + suffix; | |
5704 | ghobject_t o(hobject_t( | |
5705 | objname, | |
5706 | "", | |
5707 | CEPH_NOSNAP, | |
5708 | i<<(bits+1) | base, | |
5709 | 52, "")); | |
5710 | aobjects.insert(o); | |
5711 | t.write(a, o, 0, small.length(), small, 0); | |
5712 | if (i % 100) { | |
5713 | r = queue_transaction(store, cha, std::move(t)); | |
5714 | ASSERT_EQ(r, 0); | |
5715 | t = ObjectStore::Transaction(); | |
5716 | } | |
5717 | } | |
5718 | r = queue_transaction(store, cha, std::move(t)); | |
5719 | ASSERT_EQ(r, 0); | |
5720 | } | |
5721 | { | |
5722 | // fill b | |
5723 | ObjectStore::Transaction t; | |
5724 | for (unsigned i = 0; i < 10; ++i) { | |
5725 | string objname = "b" + stringify(i) + suffix; | |
5726 | ghobject_t o(hobject_t( | |
5727 | objname, | |
5728 | "", | |
5729 | CEPH_NOSNAP, | |
5730 | (i<<(base+1)) | base | (1<<bits), | |
5731 | 52, "")); | |
5732 | bobjects.insert(o); | |
5733 | t.write(b, o, 0, small.length(), small, 0); | |
5734 | if (i % 100) { | |
5735 | r = queue_transaction(store, chb, std::move(t)); | |
5736 | ASSERT_EQ(r, 0); | |
5737 | t = ObjectStore::Transaction(); | |
5738 | } | |
5739 | } | |
5740 | r = queue_transaction(store, chb, std::move(t)); | |
5741 | ASSERT_EQ(r, 0); | |
5742 | } | |
5743 | ||
5744 | // merge b->a | |
5745 | { | |
5746 | ObjectStore::Transaction t; | |
5747 | t.merge_collection(b, a, bits); | |
5748 | r = queue_transaction(store, cha, std::move(t)); | |
5749 | ASSERT_EQ(r, 0); | |
5750 | } | |
5751 | ||
5752 | // verify | |
5753 | { | |
5754 | vector<ghobject_t> got; | |
5755 | store->collection_list(cha, ghobject_t(), ghobject_t::get_max(), INT_MAX, | |
5756 | &got, 0); | |
5757 | set<ghobject_t> gotset; | |
5758 | for (auto& o : got) { | |
5759 | ASSERT_TRUE(aobjects.count(o) || bobjects.count(o)); | |
5760 | gotset.insert(o); | |
5761 | } | |
5762 | // check both listing and stat-ability (different code paths!) | |
5763 | struct stat st; | |
5764 | for (auto& o : aobjects) { | |
5765 | ASSERT_TRUE(gotset.count(o)); | |
5766 | int r = store->stat(cha, o, &st, false); | |
5767 | ASSERT_EQ(r, 0); | |
5768 | } | |
5769 | for (auto& o : bobjects) { | |
5770 | ASSERT_TRUE(gotset.count(o)); | |
5771 | int r = store->stat(cha, o, &st, false); | |
5772 | ASSERT_EQ(r, 0); | |
5773 | } | |
5774 | } | |
5775 | ||
5776 | // clean up | |
5777 | { | |
5778 | ObjectStore::Transaction t; | |
5779 | for (auto &o : aobjects) { | |
5780 | t.remove(a, o); | |
5781 | } | |
5782 | r = queue_transaction(store, cha, std::move(t)); | |
5783 | ASSERT_EQ(r, 0); | |
5784 | } | |
5785 | { | |
5786 | ObjectStore::Transaction t; | |
5787 | for (auto &o : bobjects) { | |
5788 | t.remove(a, o); | |
5789 | } | |
5790 | t.remove_collection(a); | |
5791 | r = queue_transaction(store, cha, std::move(t)); | |
5792 | ASSERT_EQ(r, 0); | |
5793 | } | |
5794 | } | |
5795 | ||
5796 | TEST_P(StoreTest, MergeSkewed) { | |
5797 | if (string(GetParam()) != "filestore") | |
5798 | return; | |
5799 | ||
5800 | // this is sufficient to exercise merges with different hashing levels | |
5801 | test_merge_skewed(store.get(), 0xf, 4, 10, 10000); | |
5802 | test_merge_skewed(store.get(), 0xf, 4, 10000, 10); | |
5803 | ||
5804 | /* | |
5805 | // this covers a zillion variations that all boil down to the same thing | |
5806 | for (unsigned base = 3; base < 0x1000; base *= 5) { | |
5807 | unsigned bits; | |
5808 | unsigned t = base; | |
5809 | for (bits = 0; t; t >>= 1) { | |
5810 | ++bits; | |
5811 | } | |
5812 | for (unsigned b = bits; b < bits + 10; b += 3) { | |
5813 | for (auto anum : { 10, 1000, 10000 }) { | |
5814 | for (auto bnum : { 10, 1000, 10000 }) { | |
5815 | if (anum == bnum) { | |
5816 | continue; | |
5817 | } | |
5818 | test_merge_skewed(store.get(), base, b, anum, bnum); | |
5819 | } | |
5820 | } | |
5821 | } | |
5822 | } | |
5823 | */ | |
5824 | } | |
5825 | ||
5826 | ||
7c673cae FG |
5827 | /** |
5828 | * This test tests adding two different groups | |
5829 | * of objects, each with 1 common prefix and 1 | |
5830 | * different prefix. We then remove half | |
5831 | * in order to verify that the merging correctly | |
5832 | * stops at the common prefix subdir. See bug | |
5833 | * #5273 */ | |
5834 | TEST_P(StoreTest, TwoHash) { | |
7c673cae FG |
5835 | coll_t cid; |
5836 | int r; | |
11fdf7f2 | 5837 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5838 | { |
5839 | ObjectStore::Transaction t; | |
5840 | t.create_collection(cid, 0); | |
11fdf7f2 | 5841 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5842 | ASSERT_EQ(r, 0); |
5843 | } | |
5844 | std::cout << "Making objects" << std::endl; | |
5845 | for (int i = 0; i < 360; ++i) { | |
5846 | ObjectStore::Transaction t; | |
5847 | ghobject_t o; | |
5848 | o.hobj.pool = -1; | |
5849 | if (i < 8) { | |
5850 | o.hobj.set_hash((i << 16) | 0xA1); | |
5851 | t.touch(cid, o); | |
5852 | } | |
5853 | o.hobj.set_hash((i << 16) | 0xB1); | |
5854 | t.touch(cid, o); | |
11fdf7f2 | 5855 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5856 | ASSERT_EQ(r, 0); |
5857 | } | |
5858 | std::cout << "Removing half" << std::endl; | |
5859 | for (int i = 1; i < 8; ++i) { | |
5860 | ObjectStore::Transaction t; | |
5861 | ghobject_t o; | |
5862 | o.hobj.pool = -1; | |
5863 | o.hobj.set_hash((i << 16) | 0xA1); | |
5864 | t.remove(cid, o); | |
11fdf7f2 | 5865 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5866 | ASSERT_EQ(r, 0); |
5867 | } | |
5868 | std::cout << "Checking" << std::endl; | |
5869 | for (int i = 1; i < 8; ++i) { | |
5870 | ObjectStore::Transaction t; | |
5871 | ghobject_t o; | |
5872 | o.hobj.set_hash((i << 16) | 0xA1); | |
5873 | o.hobj.pool = -1; | |
11fdf7f2 | 5874 | bool exists = store->exists(ch, o); |
7c673cae FG |
5875 | ASSERT_EQ(exists, false); |
5876 | } | |
5877 | { | |
5878 | ghobject_t o; | |
5879 | o.hobj.set_hash(0xA1); | |
5880 | o.hobj.pool = -1; | |
11fdf7f2 | 5881 | bool exists = store->exists(ch, o); |
7c673cae FG |
5882 | ASSERT_EQ(exists, true); |
5883 | } | |
5884 | std::cout << "Cleanup" << std::endl; | |
5885 | for (int i = 0; i < 360; ++i) { | |
5886 | ObjectStore::Transaction t; | |
5887 | ghobject_t o; | |
5888 | o.hobj.set_hash((i << 16) | 0xA1); | |
5889 | o.hobj.pool = -1; | |
5890 | t.remove(cid, o); | |
5891 | o.hobj.set_hash((i << 16) | 0xB1); | |
5892 | t.remove(cid, o); | |
11fdf7f2 | 5893 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5894 | ASSERT_EQ(r, 0); |
5895 | } | |
5896 | ObjectStore::Transaction t; | |
5897 | t.remove_collection(cid); | |
11fdf7f2 | 5898 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5899 | ASSERT_EQ(r, 0); |
5900 | } | |
5901 | ||
5902 | TEST_P(StoreTest, Rename) { | |
7c673cae FG |
5903 | coll_t cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD)); |
5904 | ghobject_t srcoid(hobject_t("src_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5905 | ghobject_t dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5906 | bufferlist a, b; | |
5907 | a.append("foo"); | |
5908 | b.append("bar"); | |
11fdf7f2 | 5909 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5910 | int r; |
5911 | { | |
5912 | ObjectStore::Transaction t; | |
5913 | t.create_collection(cid, 0); | |
5914 | t.write(cid, srcoid, 0, a.length(), a); | |
11fdf7f2 | 5915 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5916 | ASSERT_EQ(r, 0); |
5917 | } | |
11fdf7f2 | 5918 | ASSERT_TRUE(store->exists(ch, srcoid)); |
7c673cae FG |
5919 | { |
5920 | ObjectStore::Transaction t; | |
5921 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
5922 | t.write(cid, srcoid, 0, b.length(), b); | |
5923 | t.setattr(cid, srcoid, "attr", b); | |
11fdf7f2 | 5924 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5925 | ASSERT_EQ(r, 0); |
5926 | } | |
11fdf7f2 TL |
5927 | ASSERT_TRUE(store->exists(ch, srcoid)); |
5928 | ASSERT_TRUE(store->exists(ch, dstoid)); | |
7c673cae FG |
5929 | { |
5930 | bufferlist bl; | |
11fdf7f2 | 5931 | store->read(ch, srcoid, 0, 3, bl); |
7c673cae | 5932 | ASSERT_TRUE(bl_eq(b, bl)); |
11fdf7f2 | 5933 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
5934 | ASSERT_TRUE(bl_eq(a, bl)); |
5935 | } | |
5936 | { | |
5937 | ObjectStore::Transaction t; | |
5938 | t.remove(cid, dstoid); | |
5939 | t.collection_move_rename(cid, srcoid, cid, dstoid); | |
11fdf7f2 | 5940 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5941 | ASSERT_EQ(r, 0); |
5942 | } | |
11fdf7f2 TL |
5943 | ASSERT_TRUE(store->exists(ch, dstoid)); |
5944 | ASSERT_FALSE(store->exists(ch, srcoid)); | |
7c673cae FG |
5945 | { |
5946 | bufferlist bl; | |
11fdf7f2 | 5947 | store->read(ch, dstoid, 0, 3, bl); |
7c673cae FG |
5948 | ASSERT_TRUE(bl_eq(b, bl)); |
5949 | } | |
5950 | { | |
5951 | ObjectStore::Transaction t; | |
5952 | t.remove(cid, dstoid); | |
5953 | t.remove_collection(cid); | |
11fdf7f2 | 5954 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5955 | ASSERT_EQ(r, 0); |
5956 | } | |
5957 | } | |
5958 | ||
5959 | TEST_P(StoreTest, MoveRename) { | |
7c673cae FG |
5960 | coll_t cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD)); |
5961 | ghobject_t temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
5962 | ghobject_t oid(hobject_t("dest_oid", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 5963 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
5964 | int r; |
5965 | { | |
5966 | ObjectStore::Transaction t; | |
5967 | t.create_collection(cid, 0); | |
5968 | t.touch(cid, oid); | |
11fdf7f2 | 5969 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5970 | ASSERT_EQ(r, 0); |
5971 | } | |
11fdf7f2 | 5972 | ASSERT_TRUE(store->exists(ch, oid)); |
7c673cae FG |
5973 | bufferlist data, attr; |
5974 | map<string, bufferlist> omap; | |
5975 | data.append("data payload"); | |
5976 | attr.append("attr value"); | |
5977 | omap["omap_key"].append("omap value"); | |
5978 | { | |
5979 | ObjectStore::Transaction t; | |
5980 | t.touch(cid, temp_oid); | |
5981 | t.write(cid, temp_oid, 0, data.length(), data); | |
5982 | t.setattr(cid, temp_oid, "attr", attr); | |
5983 | t.omap_setkeys(cid, temp_oid, omap); | |
11fdf7f2 | 5984 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5985 | ASSERT_EQ(r, 0); |
5986 | } | |
11fdf7f2 | 5987 | ASSERT_TRUE(store->exists(ch, temp_oid)); |
7c673cae FG |
5988 | { |
5989 | ObjectStore::Transaction t; | |
5990 | t.remove(cid, oid); | |
5991 | t.collection_move_rename(cid, temp_oid, cid, oid); | |
11fdf7f2 | 5992 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
5993 | ASSERT_EQ(r, 0); |
5994 | } | |
11fdf7f2 TL |
5995 | ASSERT_TRUE(store->exists(ch, oid)); |
5996 | ASSERT_FALSE(store->exists(ch, temp_oid)); | |
7c673cae FG |
5997 | { |
5998 | bufferlist newdata; | |
11fdf7f2 | 5999 | r = store->read(ch, oid, 0, 1000, newdata); |
7c673cae FG |
6000 | ASSERT_GE(r, 0); |
6001 | ASSERT_TRUE(bl_eq(data, newdata)); | |
6002 | bufferlist newattr; | |
11fdf7f2 | 6003 | r = store->getattr(ch, oid, "attr", newattr); |
7c673cae FG |
6004 | ASSERT_EQ(r, 0); |
6005 | ASSERT_TRUE(bl_eq(attr, newattr)); | |
6006 | set<string> keys; | |
6007 | keys.insert("omap_key"); | |
6008 | map<string, bufferlist> newomap; | |
11fdf7f2 | 6009 | r = store->omap_get_values(ch, oid, keys, &newomap); |
7c673cae FG |
6010 | ASSERT_GE(r, 0); |
6011 | ASSERT_EQ(1u, newomap.size()); | |
6012 | ASSERT_TRUE(newomap.count("omap_key")); | |
6013 | ASSERT_TRUE(bl_eq(omap["omap_key"], newomap["omap_key"])); | |
6014 | } | |
6015 | { | |
6016 | ObjectStore::Transaction t; | |
6017 | t.remove(cid, oid); | |
6018 | t.remove_collection(cid); | |
11fdf7f2 | 6019 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6020 | ASSERT_EQ(r, 0); |
6021 | } | |
6022 | } | |
6023 | ||
6024 | TEST_P(StoreTest, BigRGWObjectName) { | |
7c673cae FG |
6025 | coll_t cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD)); |
6026 | ghobject_t oid( | |
6027 | hobject_t( | |
6028 | "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", | |
6029 | "", | |
6030 | CEPH_NOSNAP, | |
6031 | 0x81920472, | |
6032 | 12, | |
6033 | ""), | |
6034 | 15, | |
6035 | shard_id_t::NO_SHARD); | |
6036 | ghobject_t oid2(oid); | |
6037 | oid2.generation = 17; | |
6038 | ghobject_t oidhead(oid); | |
6039 | oidhead.generation = ghobject_t::NO_GEN; | |
6040 | ||
11fdf7f2 TL |
6041 | auto ch = store->create_new_collection(cid); |
6042 | ||
7c673cae FG |
6043 | int r; |
6044 | { | |
6045 | ObjectStore::Transaction t; | |
6046 | t.create_collection(cid, 0); | |
6047 | t.touch(cid, oidhead); | |
6048 | t.collection_move_rename(cid, oidhead, cid, oid); | |
6049 | t.touch(cid, oidhead); | |
6050 | t.collection_move_rename(cid, oidhead, cid, oid2); | |
11fdf7f2 | 6051 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6052 | ASSERT_EQ(r, 0); |
6053 | } | |
6054 | ||
6055 | { | |
6056 | ObjectStore::Transaction t; | |
6057 | t.remove(cid, oid); | |
11fdf7f2 | 6058 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6059 | ASSERT_EQ(r, 0); |
6060 | } | |
6061 | ||
6062 | { | |
6063 | vector<ghobject_t> objects; | |
11fdf7f2 | 6064 | r = store->collection_list(ch, ghobject_t(), ghobject_t::get_max(), |
7c673cae FG |
6065 | INT_MAX, &objects, 0); |
6066 | ASSERT_EQ(r, 0); | |
6067 | ASSERT_EQ(objects.size(), 1u); | |
6068 | ASSERT_EQ(objects[0], oid2); | |
6069 | } | |
6070 | ||
11fdf7f2 | 6071 | ASSERT_FALSE(store->exists(ch, oid)); |
7c673cae FG |
6072 | |
6073 | { | |
6074 | ObjectStore::Transaction t; | |
6075 | t.remove(cid, oid2); | |
6076 | t.remove_collection(cid); | |
11fdf7f2 | 6077 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6078 | ASSERT_EQ(r, 0); |
6079 | ||
6080 | } | |
6081 | } | |
6082 | ||
6083 | TEST_P(StoreTest, SetAllocHint) { | |
7c673cae FG |
6084 | coll_t cid; |
6085 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, 0, "")); | |
11fdf7f2 | 6086 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6087 | int r; |
6088 | { | |
6089 | ObjectStore::Transaction t; | |
6090 | t.create_collection(cid, 0); | |
6091 | t.touch(cid, hoid); | |
11fdf7f2 | 6092 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6093 | ASSERT_EQ(r, 0); |
6094 | } | |
6095 | { | |
6096 | ObjectStore::Transaction t; | |
6097 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6098 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6099 | ASSERT_EQ(r, 0); |
6100 | } | |
6101 | { | |
6102 | ObjectStore::Transaction t; | |
6103 | t.remove(cid, hoid); | |
11fdf7f2 | 6104 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6105 | ASSERT_EQ(r, 0); |
6106 | } | |
6107 | { | |
6108 | ObjectStore::Transaction t; | |
6109 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*4, 0); | |
11fdf7f2 | 6110 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6111 | ASSERT_EQ(r, 0); |
6112 | } | |
6113 | { | |
6114 | ObjectStore::Transaction t; | |
6115 | t.remove_collection(cid); | |
11fdf7f2 | 6116 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6117 | ASSERT_EQ(r, 0); |
6118 | } | |
6119 | } | |
6120 | ||
6121 | TEST_P(StoreTest, TryMoveRename) { | |
7c673cae FG |
6122 | coll_t cid; |
6123 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6124 | ghobject_t hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP, 0, -1, "")); | |
11fdf7f2 | 6125 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6126 | int r; |
6127 | { | |
6128 | ObjectStore::Transaction t; | |
6129 | t.create_collection(cid, 0); | |
11fdf7f2 | 6130 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6131 | ASSERT_EQ(r, 0); |
6132 | } | |
6133 | { | |
6134 | ObjectStore::Transaction t; | |
6135 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6136 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6137 | ASSERT_EQ(r, 0); |
6138 | } | |
6139 | { | |
6140 | ObjectStore::Transaction t; | |
6141 | t.touch(cid, hoid); | |
11fdf7f2 | 6142 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6143 | ASSERT_EQ(r, 0); |
6144 | } | |
6145 | { | |
6146 | ObjectStore::Transaction t; | |
6147 | t.try_rename(cid, hoid, hoid2); | |
11fdf7f2 | 6148 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6149 | ASSERT_EQ(r, 0); |
6150 | } | |
6151 | struct stat st; | |
11fdf7f2 TL |
6152 | ASSERT_EQ(store->stat(ch, hoid, &st), -ENOENT); |
6153 | ASSERT_EQ(store->stat(ch, hoid2, &st), 0); | |
7c673cae FG |
6154 | } |
6155 | ||
11fdf7f2 | 6156 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6157 | TEST_P(StoreTest, BluestoreOnOffCSumTest) { |
6158 | if (string(GetParam()) != "bluestore") | |
6159 | return; | |
11fdf7f2 TL |
6160 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6161 | g_conf().apply_changes(nullptr); | |
7c673cae | 6162 | |
7c673cae FG |
6163 | int r; |
6164 | coll_t cid; | |
6165 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
6166 | { | |
11fdf7f2 TL |
6167 | auto ch = store->open_collection(cid); |
6168 | ASSERT_FALSE(ch); | |
7c673cae | 6169 | } |
11fdf7f2 | 6170 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6171 | { |
6172 | ObjectStore::Transaction t; | |
6173 | t.create_collection(cid, 0); | |
6174 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 6175 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6176 | ASSERT_EQ(r, 0); |
6177 | } | |
6178 | { | |
6179 | //write with csum enabled followed by read with csum disabled | |
6180 | size_t block_size = 64*1024; | |
6181 | ObjectStore::Transaction t; | |
6182 | bufferlist bl, orig; | |
6183 | bl.append(std::string(block_size, 'a')); | |
6184 | orig = bl; | |
6185 | t.remove(cid, hoid); | |
6186 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6187 | t.write(cid, hoid, 0, bl.length(), bl); | |
6188 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6189 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6190 | ASSERT_EQ(r, 0); |
6191 | ||
11fdf7f2 TL |
6192 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6193 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6194 | |
6195 | bufferlist in; | |
11fdf7f2 | 6196 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6197 | ASSERT_EQ((int)block_size, r); |
6198 | ASSERT_TRUE(bl_eq(orig, in)); | |
6199 | ||
6200 | } | |
6201 | { | |
6202 | //write with csum disabled followed by read with csum enabled | |
6203 | ||
6204 | size_t block_size = 64*1024; | |
6205 | ObjectStore::Transaction t; | |
6206 | bufferlist bl, orig; | |
6207 | bl.append(std::string(block_size, 'a')); | |
6208 | orig = bl; | |
6209 | t.remove(cid, hoid); | |
6210 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6211 | t.write(cid, hoid, 0, bl.length(), bl); | |
6212 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6213 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6214 | ASSERT_EQ(r, 0); |
6215 | ||
11fdf7f2 TL |
6216 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6217 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6218 | |
6219 | bufferlist in; | |
11fdf7f2 | 6220 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6221 | ASSERT_EQ((int)block_size, r); |
6222 | ASSERT_TRUE(bl_eq(orig, in)); | |
6223 | } | |
6224 | { | |
6225 | //'mixed' non-overlapping writes to the same blob | |
6226 | ||
6227 | ObjectStore::Transaction t; | |
6228 | bufferlist bl, orig; | |
6229 | size_t block_size = 8000; | |
6230 | bl.append(std::string(block_size, 'a')); | |
6231 | orig = bl; | |
6232 | t.remove(cid, hoid); | |
6233 | t.write(cid, hoid, 0, bl.length(), bl); | |
6234 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6235 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6236 | ASSERT_EQ(r, 0); |
6237 | ||
11fdf7f2 TL |
6238 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6239 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6240 | |
6241 | ObjectStore::Transaction t2; | |
6242 | t2.write(cid, hoid, block_size*2, bl.length(), bl); | |
6243 | cerr << "Append 'unprotected'" << std::endl; | |
11fdf7f2 | 6244 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6245 | ASSERT_EQ(r, 0); |
6246 | ||
6247 | bufferlist in; | |
11fdf7f2 | 6248 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6249 | ASSERT_EQ((int)block_size, r); |
6250 | ASSERT_TRUE(bl_eq(orig, in)); | |
6251 | in.clear(); | |
11fdf7f2 | 6252 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6253 | ASSERT_EQ((int)block_size, r); |
6254 | ASSERT_TRUE(bl_eq(orig, in)); | |
6255 | ||
11fdf7f2 TL |
6256 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6257 | g_conf().apply_changes(nullptr); | |
7c673cae | 6258 | in.clear(); |
11fdf7f2 | 6259 | r = store->read(ch, hoid, 0, block_size, in); |
7c673cae FG |
6260 | ASSERT_EQ((int)block_size, r); |
6261 | ASSERT_TRUE(bl_eq(orig, in)); | |
6262 | in.clear(); | |
11fdf7f2 | 6263 | r = store->read(ch, hoid, block_size*2, block_size, in); |
7c673cae FG |
6264 | ASSERT_EQ((int)block_size, r); |
6265 | ASSERT_TRUE(bl_eq(orig, in)); | |
6266 | } | |
6267 | { | |
6268 | //partially blob overwrite under a different csum enablement mode | |
6269 | ||
6270 | ObjectStore::Transaction t; | |
6271 | bufferlist bl, orig, orig2; | |
6272 | size_t block_size0 = 0x10000; | |
6273 | size_t block_size = 9000; | |
6274 | size_t block_size2 = 5000; | |
6275 | bl.append(std::string(block_size0, 'a')); | |
6276 | t.remove(cid, hoid); | |
6277 | t.set_alloc_hint(cid, hoid, 4*1024*1024, 1024*8, 0); | |
6278 | t.write(cid, hoid, 0, bl.length(), bl); | |
6279 | cerr << "Remove then create" << std::endl; | |
11fdf7f2 | 6280 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6281 | ASSERT_EQ(r, 0); |
6282 | ||
11fdf7f2 TL |
6283 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6284 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6285 | |
6286 | ObjectStore::Transaction t2; | |
6287 | bl.clear(); | |
6288 | bl.append(std::string(block_size, 'b')); | |
6289 | t2.write(cid, hoid, 0, bl.length(), bl); | |
6290 | t2.write(cid, hoid, block_size0, bl.length(), bl); | |
6291 | cerr << "Overwrite with unprotected data" << std::endl; | |
11fdf7f2 | 6292 | r = queue_transaction(store, ch, std::move(t2)); |
7c673cae FG |
6293 | ASSERT_EQ(r, 0); |
6294 | ||
6295 | orig = bl; | |
6296 | orig2 = bl; | |
6297 | orig.append( std::string(block_size0 - block_size, 'a')); | |
6298 | ||
6299 | bufferlist in; | |
11fdf7f2 | 6300 | r = store->read(ch, hoid, 0, block_size0, in); |
7c673cae FG |
6301 | ASSERT_EQ((int)block_size0, r); |
6302 | ASSERT_TRUE(bl_eq(orig, in)); | |
6303 | ||
11fdf7f2 | 6304 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6305 | ASSERT_EQ((int)block_size, r); |
6306 | ASSERT_TRUE(bl_eq(orig2, in)); | |
6307 | ||
11fdf7f2 TL |
6308 | SetVal(g_conf(), "bluestore_csum_type", "crc32c"); |
6309 | g_conf().apply_changes(nullptr); | |
7c673cae FG |
6310 | |
6311 | ObjectStore::Transaction t3; | |
6312 | bl.clear(); | |
6313 | bl.append(std::string(block_size2, 'c')); | |
6314 | t3.write(cid, hoid, block_size0, bl.length(), bl); | |
6315 | cerr << "Overwrite with protected data" << std::endl; | |
11fdf7f2 | 6316 | r = queue_transaction(store, ch, std::move(t3)); |
7c673cae FG |
6317 | ASSERT_EQ(r, 0); |
6318 | ||
6319 | in.clear(); | |
6320 | orig = bl; | |
6321 | orig.append( std::string(block_size - block_size2, 'b')); | |
11fdf7f2 | 6322 | r = store->read(ch, hoid, block_size0, block_size, in); |
7c673cae FG |
6323 | ASSERT_EQ((int)block_size, r); |
6324 | ASSERT_TRUE(bl_eq(orig, in)); | |
6325 | } | |
6326 | ||
6327 | { | |
6328 | ObjectStore::Transaction t; | |
6329 | t.remove(cid, hoid); | |
6330 | t.remove_collection(cid); | |
6331 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6332 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6333 | ASSERT_EQ(r, 0); |
6334 | } | |
6335 | } | |
6336 | #endif | |
6337 | ||
9f95a23c | 6338 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6339 | ObjectStore, |
6340 | StoreTest, | |
6341 | ::testing::Values( | |
6342 | "memstore", | |
6343 | "filestore", | |
11fdf7f2 | 6344 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6345 | "bluestore", |
6346 | #endif | |
6347 | "kstore")); | |
6348 | ||
6349 | // Note: instantiate all stores to preserve store numbering order only | |
9f95a23c | 6350 | INSTANTIATE_TEST_SUITE_P( |
7c673cae FG |
6351 | ObjectStore, |
6352 | StoreTestSpecificAUSize, | |
6353 | ::testing::Values( | |
6354 | "memstore", | |
6355 | "filestore", | |
11fdf7f2 | 6356 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6357 | "bluestore", |
6358 | #endif | |
6359 | "kstore")); | |
6360 | ||
7c673cae FG |
6361 | void doMany4KWritesTest(boost::scoped_ptr<ObjectStore>& store, |
6362 | unsigned max_objects, | |
6363 | unsigned max_ops, | |
6364 | unsigned max_object_size, | |
6365 | unsigned max_write_size, | |
11fdf7f2 | 6366 | unsigned write_alignment) |
7c673cae | 6367 | { |
7c673cae FG |
6368 | MixedGenerator gen(555); |
6369 | gen_type rng(time(NULL)); | |
6370 | coll_t cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD)); | |
11fdf7f2 | 6371 | store_statfs_t res_stat; |
7c673cae FG |
6372 | |
6373 | SyntheticWorkloadState test_obj(store.get(), | |
6374 | &gen, | |
6375 | &rng, | |
7c673cae FG |
6376 | cid, |
6377 | max_object_size, | |
6378 | max_write_size, | |
6379 | write_alignment); | |
6380 | test_obj.init(); | |
6381 | for (unsigned i = 0; i < max_objects; ++i) { | |
6382 | if (!(i % 500)) cerr << "seeding object " << i << std::endl; | |
6383 | test_obj.touch(); | |
6384 | } | |
6385 | for (unsigned i = 0; i < max_ops; ++i) { | |
6386 | if (!(i % 200)) { | |
6387 | cerr << "Op " << i << std::endl; | |
6388 | test_obj.print_internal_state(); | |
6389 | } | |
6390 | test_obj.write(); | |
6391 | } | |
6392 | test_obj.wait_for_done(); | |
11fdf7f2 TL |
6393 | test_obj.statfs(res_stat); |
6394 | if (!(res_stat.data_stored <= max_object_size) || | |
6395 | !(res_stat.allocated <= max_object_size)) { | |
6396 | // this will provide more insight on the mismatch and | |
6397 | // helps to avoid any races during stats collection | |
6398 | test_obj.fsck(false); | |
6399 | // retrieving stats once again and assert if still broken | |
6400 | test_obj.statfs(res_stat); | |
6401 | ASSERT_LE(res_stat.data_stored, max_object_size); | |
6402 | ASSERT_LE(res_stat.allocated, max_object_size); | |
7c673cae FG |
6403 | } |
6404 | test_obj.shutdown(); | |
6405 | } | |
6406 | ||
6407 | TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { | |
6408 | if (string(GetParam()) != "bluestore") | |
6409 | return; | |
6410 | ||
6411 | StartDeferred(0x10000); | |
6412 | ||
11fdf7f2 TL |
6413 | const unsigned max_object = 4*1024*1024; |
6414 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0); | |
7c673cae FG |
6415 | } |
6416 | ||
6417 | TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { | |
6418 | if (string(GetParam()) != "bluestore") | |
6419 | return; | |
6420 | StartDeferred(0x10000); | |
11fdf7f2 TL |
6421 | SetVal(g_conf(), "bluestore_csum_type", "none"); |
6422 | g_ceph_context->_conf.apply_changes(nullptr); | |
6423 | const unsigned max_object = 4*1024*1024; | |
7c673cae | 6424 | |
11fdf7f2 | 6425 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0 ); |
7c673cae FG |
6426 | } |
6427 | ||
6428 | TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { | |
6429 | if (string(GetParam()) != "bluestore") | |
6430 | return; | |
6431 | StartDeferred(0x10000); | |
11fdf7f2 TL |
6432 | const unsigned max_object = 4*1024*1024; |
6433 | doMany4KWritesTest(store, 1, 1000, max_object, 4*1024, 0); | |
7c673cae FG |
6434 | } |
6435 | ||
11fdf7f2 | 6436 | #if defined(WITH_BLUESTORE) |
7c673cae FG |
6437 | void get_mempool_stats(uint64_t* total_bytes, uint64_t* total_items) |
6438 | { | |
31f18b77 FG |
6439 | uint64_t onode_allocated = mempool::bluestore_cache_onode::allocated_bytes(); |
6440 | uint64_t other_allocated = mempool::bluestore_cache_other::allocated_bytes(); | |
7c673cae | 6441 | |
31f18b77 FG |
6442 | uint64_t onode_items = mempool::bluestore_cache_onode::allocated_items(); |
6443 | uint64_t other_items = mempool::bluestore_cache_other::allocated_items(); | |
7c673cae FG |
6444 | cout << "onode(" << onode_allocated << "/" << onode_items |
6445 | << ") other(" << other_allocated << "/" << other_items | |
6446 | << ")" << std::endl; | |
6447 | *total_bytes = onode_allocated + other_allocated; | |
6448 | *total_items = onode_items; | |
6449 | } | |
6450 | ||
6451 | TEST_P(StoreTestSpecificAUSize, OnodeSizeTracking) { | |
6452 | ||
6453 | if (string(GetParam()) != "bluestore") | |
6454 | return; | |
6455 | ||
6456 | size_t block_size = 4096; | |
6457 | StartDeferred(block_size); | |
11fdf7f2 TL |
6458 | SetVal(g_conf(), "bluestore_compression_mode", "none"); |
6459 | SetVal(g_conf(), "bluestore_csum_type", "none"); | |
6460 | SetVal(g_conf(), "bluestore_cache_size_hdd", "400000000"); | |
6461 | SetVal(g_conf(), "bluestore_cache_size_ssd", "400000000"); | |
6462 | g_conf().apply_changes(nullptr); | |
7c673cae | 6463 | |
7c673cae FG |
6464 | int r; |
6465 | coll_t cid; | |
6466 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6467 | size_t obj_size = 4 * 1024 * 1024; | |
6468 | uint64_t total_bytes, total_bytes2; | |
6469 | uint64_t total_onodes; | |
6470 | get_mempool_stats(&total_bytes, &total_onodes); | |
6471 | ASSERT_EQ(total_onodes, 0u); | |
6472 | ||
11fdf7f2 | 6473 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6474 | { |
6475 | ObjectStore::Transaction t; | |
6476 | t.create_collection(cid, 0); | |
11fdf7f2 | 6477 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6478 | ASSERT_EQ(r, 0); |
6479 | } | |
6480 | { | |
6481 | ObjectStore::Transaction t; | |
6482 | bufferlist bl, orig, orig2; | |
6483 | ||
6484 | bl.append(std::string(obj_size, 'a')); | |
6485 | t.write(cid, hoid, 0, bl.length(), bl); | |
11fdf7f2 | 6486 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6487 | ASSERT_EQ(r, 0); |
6488 | } | |
6489 | get_mempool_stats(&total_bytes, &total_onodes); | |
6490 | ASSERT_NE(total_bytes, 0u); | |
6491 | ASSERT_EQ(total_onodes, 1u); | |
6492 | ||
6493 | { | |
6494 | ObjectStore::Transaction t; | |
6495 | t.truncate(cid, hoid, 0); | |
11fdf7f2 | 6496 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6497 | ASSERT_EQ(r, 0); |
6498 | } | |
6499 | ||
6500 | for(size_t i = 0; i < 1; ++i) { | |
6501 | bufferlist bl; | |
6502 | bl.append(std::string(block_size * (i+1), 'a')); | |
6503 | for( size_t j = 0; j < obj_size; j+= bl.length()) { | |
6504 | ObjectStore::Transaction t; | |
6505 | t.write(cid, hoid, j, bl.length(), bl); | |
11fdf7f2 | 6506 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6507 | ASSERT_EQ(r, 0); |
6508 | } | |
6509 | get_mempool_stats(&total_bytes2, &total_onodes); | |
6510 | ASSERT_NE(total_bytes2, 0u); | |
6511 | ASSERT_EQ(total_onodes, 1u); | |
6512 | } | |
6513 | { | |
6514 | cout <<" mempool dump:\n"; | |
6515 | JSONFormatter f(true); | |
6516 | f.open_object_section("transaction"); | |
6517 | mempool::dump(&f); | |
6518 | f.close_section(); | |
6519 | f.flush(cout); | |
6520 | cout << std::endl; | |
6521 | } | |
6522 | { | |
6523 | bufferlist bl; | |
6524 | for (size_t i = 0; i < obj_size; i += 0x1000) { | |
11fdf7f2 | 6525 | store->read(ch, hoid, i, 0x1000, bl); |
7c673cae FG |
6526 | } |
6527 | } | |
6528 | get_mempool_stats(&total_bytes, &total_onodes); | |
6529 | ASSERT_NE(total_bytes, 0u); | |
6530 | ASSERT_EQ(total_onodes, 1u); | |
6531 | ||
6532 | { | |
6533 | cout <<" mempool dump:\n"; | |
6534 | JSONFormatter f(true); | |
6535 | f.open_object_section("transaction"); | |
6536 | mempool::dump(&f); | |
6537 | f.close_section(); | |
6538 | f.flush(cout); | |
6539 | cout << std::endl; | |
6540 | } | |
6541 | { | |
6542 | ObjectStore::Transaction t; | |
6543 | t.remove(cid, hoid); | |
6544 | t.remove_collection(cid); | |
6545 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6546 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6547 | ASSERT_EQ(r, 0); |
6548 | } | |
7c673cae FG |
6549 | } |
6550 | ||
6551 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwrite) { | |
6552 | ||
6553 | if (string(GetParam()) != "bluestore") | |
6554 | return; | |
6555 | ||
6556 | size_t block_size = 4096; | |
6557 | StartDeferred(block_size); | |
11fdf7f2 TL |
6558 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
6559 | g_conf().apply_changes(nullptr); | |
7c673cae | 6560 | |
7c673cae FG |
6561 | int r; |
6562 | coll_t cid; | |
6563 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6564 | ||
6565 | const PerfCounters* logger = store->get_perf_counters(); | |
6566 | ||
11fdf7f2 | 6567 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6568 | { |
6569 | ObjectStore::Transaction t; | |
6570 | t.create_collection(cid, 0); | |
11fdf7f2 | 6571 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6572 | ASSERT_EQ(r, 0); |
6573 | } | |
6574 | { | |
6575 | ObjectStore::Transaction t; | |
6576 | bufferlist bl; | |
6577 | ||
6578 | bl.append(std::string(block_size * 2, 'a')); | |
6579 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6580 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6581 | ASSERT_EQ(r, 0); |
6582 | } | |
6583 | { | |
6584 | // overwrite at the beginning | |
6585 | ObjectStore::Transaction t; | |
6586 | bufferlist bl; | |
6587 | ||
6588 | bl.append(std::string(block_size, 'b')); | |
6589 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6590 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6591 | ASSERT_EQ(r, 0); |
6592 | } | |
6593 | { | |
6594 | // append | |
6595 | ObjectStore::Transaction t; | |
6596 | bufferlist bl; | |
6597 | ||
6598 | bl.append(std::string(block_size * 2, 'c')); | |
6599 | t.write(cid, hoid, block_size * 2, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6600 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6601 | ASSERT_EQ(r, 0); |
6602 | } | |
6603 | { | |
6604 | // append with a gap | |
6605 | ObjectStore::Transaction t; | |
6606 | bufferlist bl; | |
6607 | ||
6608 | bl.append(std::string(block_size * 2, 'd')); | |
6609 | t.write(cid, hoid, block_size * 5, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6610 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6611 | ASSERT_EQ(r, 0); |
6612 | } | |
6613 | { | |
6614 | // We need to issue a read to trigger cache stat update that refresh | |
6615 | // perf counters. additionally we need to wait some time for mempool | |
6616 | // thread to update stats. | |
6617 | sleep(1); | |
6618 | bufferlist bl, expected; | |
11fdf7f2 | 6619 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6620 | ASSERT_EQ(r, (int)block_size); |
6621 | expected.append(string(block_size, 'b')); | |
6622 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6623 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6624 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6625 | } | |
6626 | { | |
6627 | // overwrite at end | |
6628 | ObjectStore::Transaction t; | |
6629 | bufferlist bl; | |
6630 | ||
6631 | bl.append(std::string(block_size * 2, 'e')); | |
6632 | ||
6633 | // Currently we are unable to reuse blob when overwriting in a single step | |
6634 | t.write(cid, hoid, block_size * 6, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6635 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6636 | ASSERT_EQ(r, 0); |
6637 | } | |
6638 | { | |
6639 | // We need to issue a read to trigger cache stat update that refresh | |
6640 | // perf counters. additionally we need to wait some time for mempool | |
6641 | // thread to update stats. | |
6642 | sleep(1); | |
6643 | bufferlist bl, expected; | |
11fdf7f2 | 6644 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6645 | ASSERT_EQ(r, (int)block_size); |
6646 | expected.append(string(block_size, 'b')); | |
6647 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6648 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6649 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6650 | } | |
6651 | { | |
6652 | // fill the gap | |
6653 | ObjectStore::Transaction t; | |
6654 | bufferlist bl; | |
6655 | ||
6656 | bl.append(std::string(block_size, 'f')); | |
6657 | ||
6658 | t.write(cid, hoid, block_size * 4, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6659 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6660 | ASSERT_EQ(r, 0); |
6661 | } | |
6662 | { | |
6663 | // we need to wait some time for mempool | |
6664 | // thread to update stats to be able to check blob/extent numbers from | |
6665 | // perf counters. | |
6666 | sleep(1); | |
6667 | ||
6668 | bufferlist bl, expected; | |
11fdf7f2 | 6669 | r = store->read(ch, hoid, 0, block_size, bl); |
7c673cae FG |
6670 | ASSERT_EQ(r, (int)block_size); |
6671 | expected.append(string(block_size, 'b')); | |
6672 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6673 | ||
6674 | bl.clear(); | |
6675 | expected.clear(); | |
11fdf7f2 | 6676 | r = store->read(ch, hoid, block_size, block_size, bl); |
7c673cae FG |
6677 | ASSERT_EQ(r, (int)block_size); |
6678 | expected.append(string(block_size, 'a')); | |
6679 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6680 | ||
6681 | bl.clear(); | |
6682 | expected.clear(); | |
11fdf7f2 | 6683 | r = store->read(ch, hoid, block_size * 2, block_size * 2, bl); |
7c673cae FG |
6684 | ASSERT_EQ(r, (int)block_size * 2); |
6685 | expected.append(string(block_size * 2, 'c')); | |
6686 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6687 | ||
6688 | bl.clear(); | |
6689 | expected.clear(); | |
11fdf7f2 | 6690 | r = store->read(ch, hoid, block_size * 4, block_size, bl); |
7c673cae FG |
6691 | ASSERT_EQ(r, (int)block_size); |
6692 | expected.append(string(block_size, 'f')); | |
6693 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6694 | ||
6695 | bl.clear(); | |
6696 | expected.clear(); | |
11fdf7f2 | 6697 | r = store->read(ch, hoid, block_size * 5, block_size, bl); |
7c673cae FG |
6698 | ASSERT_EQ(r, (int)block_size); |
6699 | expected.append(string(block_size, 'd')); | |
6700 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6701 | ||
6702 | bl.clear(); | |
6703 | expected.clear(); | |
11fdf7f2 | 6704 | r = store->read(ch, hoid, block_size * 5, block_size * 3, bl); |
7c673cae FG |
6705 | ASSERT_EQ(r, (int)block_size * 3); |
6706 | expected.append(string(block_size, 'd')); | |
6707 | expected.append(string(block_size * 2, 'e')); | |
6708 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6709 | } | |
6710 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6711 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6712 | ||
6713 | ||
6714 | { | |
6715 | ObjectStore::Transaction t; | |
6716 | t.remove(cid, hoid); | |
6717 | t.remove_collection(cid); | |
6718 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6719 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6720 | ASSERT_EQ(r, 0); |
6721 | } | |
7c673cae FG |
6722 | } |
6723 | ||
6724 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) { | |
6725 | ||
6726 | if (string(GetParam()) != "bluestore") | |
6727 | return; | |
6728 | ||
6729 | size_t block_size = 4096; | |
6730 | StartDeferred(block_size); | |
11fdf7f2 TL |
6731 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
6732 | g_conf().apply_changes(nullptr); | |
7c673cae | 6733 | |
7c673cae FG |
6734 | int r; |
6735 | coll_t cid; | |
6736 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6737 | ||
11fdf7f2 TL |
6738 | auto ch = store->create_new_collection(cid); |
6739 | ||
7c673cae FG |
6740 | const PerfCounters* logger = store->get_perf_counters(); |
6741 | { | |
6742 | ObjectStore::Transaction t; | |
6743 | t.create_collection(cid, 0); | |
11fdf7f2 | 6744 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6745 | ASSERT_EQ(r, 0); |
6746 | } | |
6747 | { | |
6748 | ObjectStore::Transaction t; | |
6749 | bufferlist bl; | |
6750 | ||
6751 | bl.append(std::string(block_size * 2, 'a')); | |
6752 | t.write(cid, hoid, block_size * 10, bl.length(), bl, | |
6753 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6754 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6755 | ASSERT_EQ(r, 0); |
6756 | } | |
6757 | { | |
6758 | // prepend existing | |
6759 | ObjectStore::Transaction t; | |
6760 | bufferlist bl; | |
6761 | ||
6762 | bl.append(std::string(block_size, 'b')); | |
6763 | t.write(cid, hoid, block_size * 9, bl.length(), bl, | |
6764 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6765 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6766 | ASSERT_EQ(r, 0); |
6767 | } | |
6768 | { | |
6769 | // We need to issue a read to trigger cache stat update that refresh | |
6770 | // perf counters. additionally we need to wait some time for mempool | |
6771 | // thread to update stats. | |
6772 | sleep(1); | |
6773 | bufferlist bl, expected; | |
11fdf7f2 | 6774 | r = store->read(ch, hoid, block_size * 9, block_size * 2, bl); |
7c673cae FG |
6775 | ASSERT_EQ(r, (int)block_size * 2); |
6776 | expected.append(string(block_size, 'b')); | |
6777 | expected.append(string(block_size, 'a')); | |
6778 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6779 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6780 | ASSERT_EQ(logger->get(l_bluestore_extents), 1u); | |
6781 | } | |
6782 | ||
6783 | ||
6784 | { | |
6785 | // prepend existing with a gap | |
6786 | ObjectStore::Transaction t; | |
6787 | bufferlist bl; | |
6788 | ||
6789 | bl.append(std::string(block_size, 'c')); | |
6790 | t.write(cid, hoid, block_size * 7, bl.length(), bl, | |
6791 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6792 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6793 | ASSERT_EQ(r, 0); |
6794 | } | |
6795 | { | |
6796 | // We need to issue a read to trigger cache stat update that refresh | |
6797 | // perf counters. additionally we need to wait some time for mempool | |
6798 | // thread to update stats. | |
6799 | sleep(1); | |
6800 | bufferlist bl, expected; | |
11fdf7f2 | 6801 | r = store->read(ch, hoid, block_size * 7, block_size * 3, bl); |
7c673cae FG |
6802 | ASSERT_EQ(r, (int)block_size * 3); |
6803 | expected.append(string(block_size, 'c')); | |
6804 | expected.append(string(block_size, 0)); | |
6805 | expected.append(string(block_size, 'b')); | |
6806 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6807 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6808 | ASSERT_EQ(logger->get(l_bluestore_extents), 2u); | |
6809 | } | |
6810 | ||
6811 | { | |
6812 | // append after existing with a gap | |
6813 | ObjectStore::Transaction t; | |
6814 | bufferlist bl; | |
6815 | ||
6816 | bl.append(std::string(block_size, 'd')); | |
6817 | t.write(cid, hoid, block_size * 13, bl.length(), bl, | |
6818 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6819 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6820 | ASSERT_EQ(r, 0); |
6821 | } | |
6822 | { | |
6823 | // We need to issue a read to trigger cache stat update that refresh | |
6824 | // perf counters. additionally we need to wait some time for mempool | |
6825 | // thread to update stats. | |
6826 | sleep(1); | |
6827 | bufferlist bl, expected; | |
11fdf7f2 | 6828 | r = store->read(ch, hoid, block_size * 11, block_size * 3, bl); |
7c673cae FG |
6829 | ASSERT_EQ(r, (int)block_size * 3); |
6830 | expected.append(string(block_size, 'a')); | |
6831 | expected.append(string(block_size, 0)); | |
6832 | expected.append(string(block_size, 'd')); | |
6833 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6834 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6835 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
6836 | } | |
6837 | ||
6838 | { | |
6839 | // append twice to the next max_blob slot | |
6840 | ObjectStore::Transaction t; | |
6841 | bufferlist bl; | |
6842 | ||
6843 | bl.append(std::string(block_size, 'e')); | |
6844 | t.write(cid, hoid, block_size * 17, bl.length(), bl, | |
6845 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6846 | t.write(cid, hoid, block_size * 19, bl.length(), bl, | |
6847 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6848 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6849 | ASSERT_EQ(r, 0); |
6850 | } | |
6851 | { | |
6852 | // We need to issue a read to trigger cache stat update that refresh | |
6853 | // perf counters. additionally we need to wait some time for mempool | |
6854 | // thread to update stats. | |
6855 | sleep(1); | |
6856 | bufferlist bl, expected; | |
11fdf7f2 | 6857 | r = store->read(ch, hoid, block_size * 17, block_size * 3, bl); |
7c673cae FG |
6858 | ASSERT_EQ(r, (int)block_size * 3); |
6859 | expected.append(string(block_size, 'e')); | |
6860 | expected.append(string(block_size, 0)); | |
6861 | expected.append(string(block_size, 'e')); | |
6862 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6863 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
6864 | ASSERT_EQ(logger->get(l_bluestore_extents), 5u); | |
6865 | } | |
6866 | { | |
6867 | // fill gaps at the second slot | |
6868 | ObjectStore::Transaction t; | |
6869 | bufferlist bl; | |
6870 | ||
6871 | bl.append(std::string(block_size, 'f')); | |
6872 | t.write(cid, hoid, block_size * 16, bl.length(), bl, | |
6873 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6874 | t.write(cid, hoid, block_size * 18, bl.length(), bl, | |
6875 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6876 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6877 | ASSERT_EQ(r, 0); |
6878 | } | |
6879 | { | |
6880 | // We need to issue a read to trigger cache stat update that refresh | |
6881 | // perf counters. additionally we need to wait some time for mempool | |
6882 | // thread to update stats. | |
6883 | sleep(1); | |
6884 | bufferlist bl, expected; | |
11fdf7f2 | 6885 | r = store->read(ch, hoid, block_size * 16, block_size * 4, bl); |
7c673cae FG |
6886 | ASSERT_EQ(r, (int)block_size * 4); |
6887 | expected.append(string(block_size, 'f')); | |
6888 | expected.append(string(block_size, 'e')); | |
6889 | expected.append(string(block_size, 'f')); | |
6890 | expected.append(string(block_size, 'e')); | |
6891 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6892 | ASSERT_EQ(logger->get(l_bluestore_blobs), 2u); | |
6893 | ASSERT_EQ(logger->get(l_bluestore_extents), 4u); | |
6894 | } | |
6895 | { | |
6896 | ObjectStore::Transaction t; | |
6897 | t.remove(cid, hoid); | |
6898 | t.remove_collection(cid); | |
6899 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6900 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6901 | ASSERT_EQ(r, 0); |
6902 | } | |
7c673cae FG |
6903 | } |
6904 | ||
6905 | TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) { | |
6906 | ||
6907 | if (string(GetParam()) != "bluestore") | |
6908 | return; | |
6909 | ||
6910 | size_t block_size = 4096; | |
6911 | StartDeferred(block_size); | |
11fdf7f2 TL |
6912 | SetVal(g_conf(), "bluestore_max_blob_size", "65536"); |
6913 | g_conf().apply_changes(nullptr); | |
7c673cae | 6914 | |
7c673cae FG |
6915 | int r; |
6916 | coll_t cid; | |
6917 | ghobject_t hoid(hobject_t("test_hint", "", CEPH_NOSNAP, 0, -1, "")); | |
6918 | ||
6919 | const PerfCounters* logger = store->get_perf_counters(); | |
11fdf7f2 | 6920 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
6921 | |
6922 | { | |
6923 | ObjectStore::Transaction t; | |
6924 | t.create_collection(cid, 0); | |
11fdf7f2 | 6925 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6926 | ASSERT_EQ(r, 0); |
6927 | } | |
6928 | { | |
6929 | ObjectStore::Transaction t; | |
6930 | bufferlist bl; | |
6931 | ||
6932 | bl.append(std::string(block_size, 'a')); | |
6933 | t.write(cid, hoid, 0, bl.length(), bl, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
6934 | t.write(cid, hoid, block_size * 2, bl.length(), bl, | |
6935 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6936 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6937 | ASSERT_EQ(r, 0); |
6938 | } | |
6939 | { | |
6940 | // write small into the gap | |
6941 | ObjectStore::Transaction t; | |
6942 | bufferlist bl; | |
6943 | ||
6944 | bl.append(std::string(3, 'b')); | |
6945 | t.write(cid, hoid, block_size + 1, bl.length(), bl, | |
6946 | CEPH_OSD_OP_FLAG_FADVISE_WILLNEED); | |
11fdf7f2 | 6947 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6948 | ASSERT_EQ(r, 0); |
6949 | } | |
6950 | { | |
6951 | // We need to issue a read to trigger cache stat update that refresh | |
6952 | // perf counters. additionally we need to wait some time for mempool | |
6953 | // thread to update stats. | |
6954 | sleep(1); | |
6955 | bufferlist bl, expected; | |
11fdf7f2 | 6956 | r = store->read(ch, hoid, 0, block_size * 3, bl); |
7c673cae FG |
6957 | ASSERT_EQ(r, (int)block_size * 3); |
6958 | expected.append(string(block_size, 'a')); | |
6959 | expected.append(string(1, 0)); | |
6960 | expected.append(string(3, 'b')); | |
6961 | expected.append(string(block_size - 4, 0)); | |
6962 | expected.append(string(block_size, 'a')); | |
6963 | ASSERT_TRUE(bl_eq(expected, bl)); | |
6964 | ||
6965 | ASSERT_EQ(logger->get(l_bluestore_blobs), 1u); | |
6966 | ASSERT_EQ(logger->get(l_bluestore_extents), 3u); | |
6967 | } | |
6968 | { | |
6969 | ObjectStore::Transaction t; | |
6970 | t.remove(cid, hoid); | |
6971 | t.remove_collection(cid); | |
6972 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 6973 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
6974 | ASSERT_EQ(r, 0); |
6975 | } | |
7c673cae FG |
6976 | } |
6977 | ||
6978 | // The test case to reproduce an issue when write happens | |
6979 | // to a zero space between the extents sharing the same spanning blob | |
6980 | // with unloaded shard map. | |
6981 | // Second extent might be filled with zeros this way due to wrong result | |
6982 | // returned by has_any_extents() call in do_write_small. The latter is caused | |
6983 | // by incompletly loaded extent map. | |
6984 | TEST_P(StoreTestSpecificAUSize, SmallWriteOnShardedExtents) { | |
6985 | if (string(GetParam()) != "bluestore") | |
6986 | return; | |
6987 | ||
6988 | size_t block_size = 0x10000; | |
6989 | StartDeferred(block_size); | |
6990 | ||
11fdf7f2 TL |
6991 | SetVal(g_conf(), "bluestore_csum_type", "xxhash64"); |
6992 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); // for sure | |
7c673cae | 6993 | |
11fdf7f2 | 6994 | g_conf().apply_changes(nullptr); |
7c673cae | 6995 | |
7c673cae FG |
6996 | int r; |
6997 | coll_t cid; | |
6998 | ghobject_t hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
11fdf7f2 | 6999 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7000 | |
7001 | { | |
7002 | ObjectStore::Transaction t; | |
7003 | t.create_collection(cid, 0); | |
11fdf7f2 | 7004 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7005 | ASSERT_EQ(r, 0); |
7006 | } | |
7007 | { | |
7008 | //doing some tricks to have sharded extents/spanning objects | |
7009 | ObjectStore::Transaction t; | |
7010 | bufferlist bl, bl2; | |
7011 | ||
7012 | bl.append(std::string(0x80000, 'a')); | |
7013 | t.write(cid, hoid1, 0, bl.length(), bl, 0); | |
7014 | t.zero(cid, hoid1, 0x719e0, 0x75b0 ); | |
11fdf7f2 | 7015 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7016 | ASSERT_EQ(r, 0); |
7017 | ||
7018 | bl2.append(std::string(0x70000, 'b')); | |
7019 | t.write(cid, hoid1, 0, bl2.length(), bl2, 0); | |
7020 | t.zero(cid, hoid1, 0, 0x50000); | |
11fdf7f2 | 7021 | r = queue_transaction(store, ch, std::move(t)); |
31f18b77 | 7022 | ASSERT_EQ(r, 0); |
7c673cae FG |
7023 | |
7024 | } | |
11fdf7f2 | 7025 | ch.reset(); |
7c673cae FG |
7026 | store->umount(); |
7027 | store->mount(); | |
11fdf7f2 | 7028 | ch = store->open_collection(cid); |
7c673cae FG |
7029 | |
7030 | { | |
7031 | // do a write to zero space in between some extents sharing the same blob | |
7032 | ObjectStore::Transaction t; | |
7033 | bufferlist bl, bl2; | |
7034 | ||
7035 | bl.append(std::string(0x6520, 'c')); | |
7036 | t.write(cid, hoid1, 0x71c00, bl.length(), bl, 0); | |
7037 | ||
11fdf7f2 | 7038 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7039 | ASSERT_EQ(r, 0); |
7040 | } | |
7041 | ||
7042 | { | |
7043 | ObjectStore::Transaction t; | |
7044 | bufferlist bl, expected; | |
7045 | ||
11fdf7f2 | 7046 | r = store->read(ch, hoid1, 0x70000, 0x9c00, bl); |
7c673cae FG |
7047 | ASSERT_EQ(r, (int)0x9c00); |
7048 | expected.append(string(0x19e0, 'a')); | |
7049 | expected.append(string(0x220, 0)); | |
7050 | expected.append(string(0x6520, 'c')); | |
7051 | expected.append(string(0xe70, 0)); | |
7052 | expected.append(string(0xc70, 'a')); | |
7053 | ASSERT_TRUE(bl_eq(expected, bl)); | |
7054 | bl.clear(); | |
7055 | ||
7056 | } | |
7057 | ||
7058 | { | |
7059 | ObjectStore::Transaction t; | |
7060 | t.remove(cid, hoid1); | |
7061 | t.remove_collection(cid); | |
7062 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7063 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7064 | ASSERT_EQ(r, 0); |
7065 | } | |
7c673cae FG |
7066 | } |
7067 | ||
11fdf7f2 | 7068 | #endif //#if defined(WITH_BLUESTORE) |
7c673cae FG |
7069 | |
7070 | TEST_P(StoreTest, KVDBHistogramTest) { | |
7071 | if (string(GetParam()) != "bluestore") | |
7072 | return; | |
7073 | ||
7c673cae FG |
7074 | int NUM_OBJS = 200; |
7075 | int r = 0; | |
7076 | coll_t cid; | |
7077 | string base("testobj."); | |
7078 | bufferlist a; | |
7079 | bufferptr ap(0x1000); | |
7080 | memset(ap.c_str(), 'a', 0x1000); | |
7081 | a.append(ap); | |
11fdf7f2 | 7082 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7083 | { |
7084 | ObjectStore::Transaction t; | |
7085 | t.create_collection(cid, 0); | |
11fdf7f2 | 7086 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7087 | ASSERT_EQ(r, 0); |
7088 | } | |
7089 | for (int i = 0; i < NUM_OBJS; ++i) { | |
7090 | ObjectStore::Transaction t; | |
7091 | char buf[100]; | |
7092 | snprintf(buf, sizeof(buf), "%d", i); | |
7093 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
7094 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 7095 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7096 | ASSERT_EQ(r, 0); |
7097 | } | |
7098 | ||
7099 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7100 | store->generate_db_histogram(f); | |
7101 | f->flush(cout); | |
7102 | cout << std::endl; | |
7103 | } | |
7104 | ||
7105 | TEST_P(StoreTest, KVDBStatsTest) { | |
7106 | if (string(GetParam()) != "bluestore") | |
7107 | return; | |
7108 | ||
11fdf7f2 TL |
7109 | SetVal(g_conf(), "rocksdb_perf", "true"); |
7110 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
7111 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
7112 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
7113 | g_ceph_context->_conf.apply_changes(nullptr); | |
7c673cae FG |
7114 | int r = store->umount(); |
7115 | ASSERT_EQ(r, 0); | |
7116 | r = store->mount(); //to force rocksdb stats | |
7117 | ASSERT_EQ(r, 0); | |
7118 | ||
7c673cae FG |
7119 | int NUM_OBJS = 200; |
7120 | coll_t cid; | |
7121 | string base("testobj."); | |
7122 | bufferlist a; | |
7123 | bufferptr ap(0x1000); | |
7124 | memset(ap.c_str(), 'a', 0x1000); | |
7125 | a.append(ap); | |
11fdf7f2 | 7126 | auto ch = store->create_new_collection(cid); |
7c673cae FG |
7127 | { |
7128 | ObjectStore::Transaction t; | |
7129 | t.create_collection(cid, 0); | |
11fdf7f2 | 7130 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7131 | ASSERT_EQ(r, 0); |
7132 | } | |
7133 | for (int i = 0; i < NUM_OBJS; ++i) { | |
7134 | ObjectStore::Transaction t; | |
7135 | char buf[100]; | |
7136 | snprintf(buf, sizeof(buf), "%d", i); | |
7137 | ghobject_t hoid(hobject_t(sobject_t(base + string(buf), CEPH_NOSNAP))); | |
7138 | t.write(cid, hoid, 0, 0x1000, a); | |
11fdf7f2 | 7139 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7140 | ASSERT_EQ(r, 0); |
7141 | } | |
7142 | ||
7143 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7144 | store->get_db_statistics(f); | |
7145 | f->flush(cout); | |
7146 | cout << std::endl; | |
7c673cae FG |
7147 | } |
7148 | ||
11fdf7f2 | 7149 | #if defined(WITH_BLUESTORE) |
7c673cae | 7150 | TEST_P(StoreTestSpecificAUSize, garbageCollection) { |
7c673cae FG |
7151 | int r; |
7152 | coll_t cid; | |
7153 | int buf_len = 256 * 1024; | |
7154 | int overlap_offset = 64 * 1024; | |
7155 | int write_offset = buf_len; | |
7156 | if (string(GetParam()) != "bluestore") | |
7157 | return; | |
7158 | ||
7159 | #define WRITE_AT(offset, _length) {\ | |
7160 | ObjectStore::Transaction t;\ | |
7161 | if ((uint64_t)_length != bl.length()) { \ | |
7162 | buffer::ptr p(bl.c_str(), _length);\ | |
7163 | bufferlist bl_tmp;\ | |
7164 | bl_tmp.push_back(p);\ | |
7165 | t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\ | |
7166 | } else {\ | |
7167 | t.write(cid, hoid, offset, bl.length(), bl);\ | |
7168 | }\ | |
11fdf7f2 | 7169 | r = queue_transaction(store, ch, std::move(t));\ |
7c673cae FG |
7170 | ASSERT_EQ(r, 0);\ |
7171 | } | |
7172 | ||
7173 | StartDeferred(65536); | |
7174 | ||
11fdf7f2 TL |
7175 | SetVal(g_conf(), "bluestore_compression_max_blob_size", "524288"); |
7176 | SetVal(g_conf(), "bluestore_compression_min_blob_size", "262144"); | |
7177 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
7178 | SetVal(g_conf(), "bluestore_compression_mode", "force"); | |
7179 | g_conf().apply_changes(nullptr); | |
7180 | ||
7181 | auto ch = store->create_new_collection(cid); | |
7c673cae FG |
7182 | |
7183 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
7184 | { | |
7185 | bufferlist in; | |
11fdf7f2 | 7186 | r = store->read(ch, hoid, 0, 5, in); |
7c673cae FG |
7187 | ASSERT_EQ(-ENOENT, r); |
7188 | } | |
7189 | { | |
7190 | ObjectStore::Transaction t; | |
7191 | t.create_collection(cid, 0); | |
7192 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 7193 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7194 | ASSERT_EQ(r, 0); |
7195 | } | |
7196 | ||
7197 | std::string data; | |
7198 | data.resize(buf_len); | |
7199 | ||
7200 | { | |
7201 | { | |
11fdf7f2 | 7202 | bool exists = store->exists(ch, hoid); |
7c673cae FG |
7203 | ASSERT_TRUE(!exists); |
7204 | ||
7205 | ObjectStore::Transaction t; | |
7206 | t.touch(cid, hoid); | |
7207 | cerr << "Creating object " << hoid << std::endl; | |
11fdf7f2 | 7208 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7209 | ASSERT_EQ(r, 0); |
7210 | ||
11fdf7f2 | 7211 | exists = store->exists(ch, hoid); |
7c673cae FG |
7212 | ASSERT_EQ(true, exists); |
7213 | } | |
7214 | bufferlist bl; | |
7215 | ||
7216 | for(size_t i = 0; i < data.size(); i++) | |
7217 | data[i] = i % 256; | |
7218 | ||
7219 | bl.append(data); | |
7220 | ||
7221 | { | |
7222 | struct store_statfs_t statfs; | |
7223 | WRITE_AT(0, buf_len); | |
7224 | int r = store->statfs(&statfs); | |
7225 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7226 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7227 | } |
7228 | { | |
7229 | struct store_statfs_t statfs; | |
7230 | WRITE_AT(write_offset - 2 * overlap_offset, buf_len); | |
7231 | int r = store->statfs(&statfs); | |
7232 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7233 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7234 | const PerfCounters* counters = store->get_perf_counters(); |
7235 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0u); | |
7236 | } | |
7237 | ||
7238 | { | |
7239 | struct store_statfs_t statfs; | |
7240 | WRITE_AT(write_offset - overlap_offset, buf_len); | |
7241 | int r = store->statfs(&statfs); | |
7242 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7243 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7244 | const PerfCounters* counters = store->get_perf_counters(); |
7245 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x10000u); | |
7246 | } | |
7247 | { | |
7248 | struct store_statfs_t statfs; | |
7249 | WRITE_AT(write_offset - 3 * overlap_offset, buf_len); | |
7250 | int r = store->statfs(&statfs); | |
7251 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7252 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7253 | const PerfCounters* counters = store->get_perf_counters(); |
7254 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
7255 | } | |
7256 | { | |
7257 | struct store_statfs_t statfs; | |
7258 | WRITE_AT(write_offset + 1, overlap_offset-1); | |
7259 | int r = store->statfs(&statfs); | |
7260 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7261 | ASSERT_EQ(statfs.data_compressed_allocated, 0x20000); |
7c673cae FG |
7262 | const PerfCounters* counters = store->get_perf_counters(); |
7263 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x20000u); | |
7264 | } | |
7265 | { | |
7266 | struct store_statfs_t statfs; | |
7267 | WRITE_AT(write_offset + 1, overlap_offset); | |
7268 | int r = store->statfs(&statfs); | |
7269 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7270 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7271 | const PerfCounters* counters = store->get_perf_counters(); |
7272 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x3ffffu); | |
7273 | } | |
7274 | { | |
7275 | struct store_statfs_t statfs; | |
7276 | WRITE_AT(0, buf_len-1); | |
7277 | int r = store->statfs(&statfs); | |
7278 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7279 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7280 | const PerfCounters* counters = store->get_perf_counters(); |
7281 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
7282 | } | |
11fdf7f2 | 7283 | SetVal(g_conf(), "bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0 |
7c673cae FG |
7284 | { |
7285 | struct store_statfs_t statfs; | |
7286 | WRITE_AT(1, overlap_offset-2); | |
7287 | WRITE_AT(overlap_offset * 2 + 1, overlap_offset-2); | |
7288 | int r = store->statfs(&statfs); | |
7289 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7290 | ASSERT_EQ(statfs.data_compressed_allocated, 0x10000); |
7c673cae FG |
7291 | const PerfCounters* counters = store->get_perf_counters(); |
7292 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40001u); | |
7293 | } | |
7294 | { | |
7295 | struct store_statfs_t statfs; | |
7296 | WRITE_AT(overlap_offset + 1, overlap_offset-2); | |
7297 | int r = store->statfs(&statfs); | |
7298 | ASSERT_EQ(r, 0); | |
11fdf7f2 | 7299 | ASSERT_EQ(statfs.data_compressed_allocated, 0x0); |
7c673cae FG |
7300 | const PerfCounters* counters = store->get_perf_counters(); |
7301 | ASSERT_EQ(counters->get(l_bluestore_gc_merged), 0x40007u); | |
7302 | } | |
7303 | { | |
7304 | ObjectStore::Transaction t; | |
7305 | t.remove(cid, hoid); | |
7306 | cerr << "Cleaning" << std::endl; | |
11fdf7f2 | 7307 | r = queue_transaction(store, ch, std::move(t)); |
7c673cae FG |
7308 | ASSERT_EQ(r, 0); |
7309 | } | |
7310 | } | |
7c673cae | 7311 | } |
7c673cae | 7312 | |
b32b8144 FG |
7313 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice) { |
7314 | if (string(GetParam()) != "bluestore") | |
7315 | return; | |
7316 | ||
11fdf7f2 TL |
7317 | SetVal(g_conf(), "bluestore_block_size", |
7318 | stringify(0x280005000).c_str()); //10 Gb + 4K | |
7319 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7320 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
7321 | StartDeferred(0x4000); |
7322 | store->umount(); | |
7323 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7324 | store->mount(); | |
7325 | ||
b32b8144 FG |
7326 | } |
7327 | ||
7328 | TEST_P(StoreTestSpecificAUSize, fsckOnUnalignedDevice2) { | |
7329 | if (string(GetParam()) != "bluestore") | |
7330 | return; | |
7331 | ||
11fdf7f2 TL |
7332 | SetVal(g_conf(), "bluestore_block_size", |
7333 | stringify(0x280005000).c_str()); //10 Gb + 20K | |
7334 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7335 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
b32b8144 FG |
7336 | StartDeferred(0x1000); |
7337 | store->umount(); | |
7338 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7339 | store->mount(); | |
11fdf7f2 TL |
7340 | } |
7341 | ||
7342 | namespace { | |
7343 | ghobject_t make_object(const char* name, int64_t pool) { | |
7344 | sobject_t soid{name, CEPH_NOSNAP}; | |
7345 | uint32_t hash = std::hash<sobject_t>{}(soid); | |
7346 | return ghobject_t{hobject_t{soid, "", hash, pool, ""}}; | |
7347 | } | |
7348 | } | |
7349 | ||
7350 | TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) { | |
7351 | if (string(GetParam()) != "bluestore") | |
7352 | return; | |
7353 | const size_t offs_base = 65536 / 2; | |
7354 | ||
7355 | SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); | |
7356 | SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); | |
7357 | SetVal(g_conf(), "bluestore_max_blob_size", | |
7358 | stringify(2 * offs_base).c_str()); | |
7359 | SetVal(g_conf(), "bluestore_extent_map_shard_max_size", "12000"); | |
eafe8130 | 7360 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "false"); |
11fdf7f2 TL |
7361 | |
7362 | StartDeferred(0x10000); | |
7363 | ||
7364 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7365 | ||
7366 | // fill the store with some data | |
7367 | const uint64_t pool = 555; | |
7368 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7369 | auto ch = store->create_new_collection(cid); | |
7370 | ||
7371 | ghobject_t hoid = make_object("Object 1", pool); | |
7372 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7373 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7374 | ghobject_t hoid_cloned = hoid2; | |
7375 | hoid_cloned.hobj.snap = 1; | |
7376 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7377 | ghobject_t hoid3_cloned = hoid3; | |
7378 | hoid3_cloned.hobj.snap = 1; | |
7379 | bufferlist bl; | |
7380 | bl.append("1234512345"); | |
7381 | int r; | |
7382 | const size_t repeats = 16; | |
7383 | { | |
7384 | auto ch = store->create_new_collection(cid); | |
7385 | cerr << "create collection + write" << std::endl; | |
7386 | ObjectStore::Transaction t; | |
7387 | t.create_collection(cid, 0); | |
7388 | for( auto i = 0ul; i < repeats; ++i ) { | |
7389 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7390 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7391 | } | |
7392 | for( auto i = 0ul; i < repeats; ++i ) { | |
7393 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7394 | } | |
7395 | t.clone(cid, hoid2, hoid_cloned); | |
7396 | ||
7397 | r = queue_transaction(store, ch, std::move(t)); | |
7398 | ASSERT_EQ(r, 0); | |
7399 | } | |
7400 | ||
7401 | bstore->umount(); | |
7402 | //////////// leaked pextent fix //////////// | |
7403 | cerr << "fix leaked pextents" << std::endl; | |
7404 | ASSERT_EQ(bstore->fsck(false), 0); | |
7405 | ASSERT_EQ(bstore->repair(false), 0); | |
7406 | bstore->mount(); | |
7407 | bstore->inject_leaked(0x30000); | |
7408 | bstore->umount(); | |
7409 | ASSERT_EQ(bstore->fsck(false), 1); | |
7410 | ASSERT_EQ(bstore->repair(false), 0); | |
7411 | ASSERT_EQ(bstore->fsck(false), 0); | |
7412 | ||
7413 | //////////// false free fix //////////// | |
7414 | cerr << "fix false free pextents" << std::endl; | |
7415 | bstore->mount(); | |
7416 | bstore->inject_false_free(cid, hoid); | |
7417 | bstore->umount(); | |
7418 | ASSERT_EQ(bstore->fsck(false), 2); | |
7419 | ASSERT_EQ(bstore->repair(false), 0); | |
7420 | ASSERT_EQ(bstore->fsck(false), 0); | |
7421 | ||
7422 | //////////// verify invalid statfs /////////// | |
7423 | cerr << "fix invalid statfs" << std::endl; | |
7424 | store_statfs_t statfs0, statfs; | |
7425 | bstore->mount(); | |
7426 | ASSERT_EQ(bstore->statfs(&statfs0), 0); | |
7427 | statfs = statfs0; | |
7428 | statfs.allocated += 0x10000; | |
7429 | statfs.data_stored += 0x10000; | |
7430 | ASSERT_FALSE(statfs0 == statfs); | |
7431 | bstore->inject_statfs("bluestore_statfs", statfs); | |
7432 | bstore->umount(); | |
7433 | ||
eafe8130 | 7434 | ASSERT_EQ(bstore->fsck(false), 2); |
11fdf7f2 TL |
7435 | ASSERT_EQ(bstore->repair(false), 0); |
7436 | ASSERT_EQ(bstore->fsck(false), 0); | |
7437 | ASSERT_EQ(bstore->mount(), 0); | |
7438 | ASSERT_EQ(bstore->statfs(&statfs), 0); | |
7439 | // adjust free space to success in comparison | |
7440 | statfs0.available = statfs.available; | |
7441 | ASSERT_EQ(statfs0, statfs); | |
7442 | ||
7443 | ///////// undecodable shared blob key / stray shared blob records /////// | |
7444 | cerr << "undecodable shared blob key" << std::endl; | |
7445 | bstore->inject_broken_shared_blob_key("undec1", | |
7446 | bufferlist()); | |
7447 | bstore->inject_broken_shared_blob_key("undecodable key 2", | |
7448 | bufferlist()); | |
7449 | bstore->inject_broken_shared_blob_key("undecodable key 3", | |
7450 | bufferlist()); | |
7451 | bstore->umount(); | |
7452 | ASSERT_EQ(bstore->fsck(false), 3); | |
7453 | ASSERT_EQ(bstore->repair(false), 0); | |
7454 | ASSERT_EQ(bstore->fsck(false), 0); | |
7455 | ||
7456 | cerr << "misreferencing" << std::endl; | |
7457 | bstore->mount(); | |
7458 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, 0); | |
7459 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, (offs_base * repeats) / 2); | |
7460 | bstore->inject_misreference(cid, hoid, cid, hoid_dup, offs_base * (repeats -1) ); | |
7461 | ||
7462 | bstore->umount(); | |
7463 | ASSERT_EQ(bstore->fsck(false), 6); | |
7464 | ASSERT_EQ(bstore->repair(false), 0); | |
7465 | ||
7466 | ASSERT_EQ(bstore->fsck(true), 0); | |
7467 | ||
7468 | // reproducing issues #21040 & 20983 | |
7469 | SetVal(g_conf(), "bluestore_debug_inject_bug21040", "true"); | |
7470 | g_ceph_context->_conf.apply_changes(nullptr); | |
7471 | bstore->mount(); | |
7472 | ||
7473 | cerr << "repro bug #21040" << std::endl; | |
7474 | { | |
7475 | auto ch = store->open_collection(cid); | |
7476 | { | |
7477 | ObjectStore::Transaction t; | |
7478 | bl.append("0123456789012345"); | |
7479 | t.write(cid, hoid3, offs_base, bl.length(), bl); | |
7480 | bl.clear(); | |
7481 | bl.append('!'); | |
7482 | t.write(cid, hoid3, 0, bl.length(), bl); | |
7483 | ||
7484 | r = queue_transaction(store, ch, std::move(t)); | |
7485 | ASSERT_EQ(r, 0); | |
7486 | } | |
7487 | { | |
7488 | ObjectStore::Transaction t; | |
7489 | t.clone(cid, hoid3, hoid3_cloned); | |
7490 | r = queue_transaction(store, ch, std::move(t)); | |
7491 | ASSERT_EQ(r, 0); | |
7492 | } | |
7493 | ||
7494 | bstore->umount(); | |
7495 | ASSERT_EQ(bstore->fsck(false), 3); | |
7496 | ASSERT_LE(bstore->repair(false), 0); | |
7497 | ASSERT_EQ(bstore->fsck(false), 0); | |
7498 | } | |
7499 | ||
eafe8130 TL |
7500 | cerr << "Completing" << std::endl; |
7501 | bstore->mount(); | |
7502 | ||
7503 | } | |
7504 | ||
7505 | TEST_P(StoreTest, BluestoreRepairGlobalStats) | |
7506 | { | |
7507 | if (string(GetParam()) != "bluestore") | |
7508 | return; | |
7509 | const size_t offs_base = 65536 / 2; | |
7510 | ||
7511 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7512 | ||
7513 | // start with global stats | |
7514 | bstore->inject_global_statfs({}); | |
7515 | bstore->umount(); | |
7516 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7517 | bstore->mount(); | |
7518 | ||
7519 | // fill the store with some data | |
7520 | const uint64_t pool = 555; | |
7521 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7522 | auto ch = store->create_new_collection(cid); | |
7523 | ||
7524 | ghobject_t hoid = make_object("Object 1", pool); | |
7525 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7526 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7527 | ghobject_t hoid_cloned = hoid2; | |
7528 | hoid_cloned.hobj.snap = 1; | |
7529 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7530 | ghobject_t hoid3_cloned = hoid3; | |
7531 | hoid3_cloned.hobj.snap = 1; | |
7532 | bufferlist bl; | |
7533 | bl.append("1234512345"); | |
7534 | int r; | |
7535 | const size_t repeats = 16; | |
7536 | { | |
7537 | auto ch = store->create_new_collection(cid); | |
7538 | cerr << "create collection + write" << std::endl; | |
7539 | ObjectStore::Transaction t; | |
7540 | t.create_collection(cid, 0); | |
7541 | for( auto i = 0ul; i < repeats; ++i ) { | |
7542 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7543 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7544 | } | |
7545 | for( auto i = 0ul; i < repeats; ++i ) { | |
7546 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7547 | } | |
7548 | t.clone(cid, hoid2, hoid_cloned); | |
7549 | ||
7550 | r = queue_transaction(store, ch, std::move(t)); | |
7551 | ASSERT_EQ(r, 0); | |
7552 | } | |
7553 | ||
7554 | bstore->umount(); | |
7555 | ||
11fdf7f2 TL |
7556 | // enable per-pool stats collection hence causing fsck to fail |
7557 | cerr << "per-pool statfs" << std::endl; | |
eafe8130 | 7558 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); |
11fdf7f2 TL |
7559 | g_ceph_context->_conf.apply_changes(nullptr); |
7560 | ||
eafe8130 | 7561 | ASSERT_EQ(bstore->fsck(false), 1); |
11fdf7f2 TL |
7562 | ASSERT_EQ(bstore->repair(false), 0); |
7563 | ASSERT_EQ(bstore->fsck(false), 0); | |
7564 | ||
11fdf7f2 | 7565 | bstore->mount(); |
eafe8130 TL |
7566 | } |
7567 | ||
7568 | TEST_P(StoreTest, BluestoreRepairGlobalStatsFixOnMount) | |
7569 | { | |
7570 | if (string(GetParam()) != "bluestore") | |
7571 | return; | |
7572 | const size_t offs_base = 65536 / 2; | |
7573 | ||
7574 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7575 | ||
7576 | // start with global stats | |
7577 | bstore->inject_global_statfs({}); | |
7578 | bstore->umount(); | |
7579 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7580 | bstore->mount(); | |
7581 | ||
7582 | // fill the store with some data | |
7583 | const uint64_t pool = 555; | |
7584 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7585 | auto ch = store->create_new_collection(cid); | |
7586 | ||
7587 | ghobject_t hoid = make_object("Object 1", pool); | |
7588 | ghobject_t hoid_dup = make_object("Object 1(dup)", pool); | |
7589 | ghobject_t hoid2 = make_object("Object 2", pool); | |
7590 | ghobject_t hoid_cloned = hoid2; | |
7591 | hoid_cloned.hobj.snap = 1; | |
7592 | ghobject_t hoid3 = make_object("Object 3", pool); | |
7593 | ghobject_t hoid3_cloned = hoid3; | |
7594 | hoid3_cloned.hobj.snap = 1; | |
7595 | bufferlist bl; | |
7596 | bl.append("1234512345"); | |
7597 | int r; | |
7598 | const size_t repeats = 16; | |
7599 | { | |
7600 | auto ch = store->create_new_collection(cid); | |
7601 | cerr << "create collection + write" << std::endl; | |
7602 | ObjectStore::Transaction t; | |
7603 | t.create_collection(cid, 0); | |
7604 | for( auto i = 0ul; i < repeats; ++i ) { | |
7605 | t.write(cid, hoid, i * offs_base, bl.length(), bl); | |
7606 | t.write(cid, hoid_dup, i * offs_base, bl.length(), bl); | |
7607 | } | |
7608 | for( auto i = 0ul; i < repeats; ++i ) { | |
7609 | t.write(cid, hoid2, i * offs_base, bl.length(), bl); | |
7610 | } | |
7611 | t.clone(cid, hoid2, hoid_cloned); | |
7612 | ||
7613 | r = queue_transaction(store, ch, std::move(t)); | |
7614 | ASSERT_EQ(r, 0); | |
7615 | } | |
7616 | ||
7617 | bstore->umount(); | |
7618 | ||
7619 | // enable per-pool stats collection hence causing fsck to fail | |
7620 | cerr << "per-pool statfs" << std::endl; | |
7621 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_stats", "true"); | |
7622 | g_ceph_context->_conf.apply_changes(nullptr); | |
7623 | ||
7624 | ASSERT_EQ(bstore->fsck(false), 1); | |
7625 | ||
7626 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7627 | bstore->mount(); | |
7628 | bstore->umount(); | |
7629 | ASSERT_EQ(bstore->fsck(false), 0); | |
b32b8144 | 7630 | |
eafe8130 | 7631 | bstore->mount(); |
11fdf7f2 TL |
7632 | } |
7633 | ||
7634 | TEST_P(StoreTest, BluestoreStatistics) { | |
7635 | if (string(GetParam()) != "bluestore") | |
7636 | return; | |
7637 | ||
7638 | SetVal(g_conf(), "rocksdb_perf", "true"); | |
7639 | SetVal(g_conf(), "rocksdb_collect_compaction_stats", "true"); | |
7640 | SetVal(g_conf(), "rocksdb_collect_extended_stats","true"); | |
7641 | SetVal(g_conf(), "rocksdb_collect_memory_stats","true"); | |
7642 | ||
7643 | // disable cache | |
7644 | SetVal(g_conf(), "bluestore_cache_size_ssd", "0"); | |
7645 | SetVal(g_conf(), "bluestore_cache_size_hdd", "0"); | |
7646 | SetVal(g_conf(), "bluestore_cache_size", "0"); | |
7647 | g_ceph_context->_conf.apply_changes(nullptr); | |
7648 | ||
7649 | int r = store->umount(); | |
7650 | ASSERT_EQ(r, 0); | |
7651 | r = store->mount(); | |
7652 | ASSERT_EQ(r, 0); | |
7653 | ||
7654 | BlueStore* bstore = NULL; | |
7655 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
7656 | ||
7657 | coll_t cid; | |
7658 | ghobject_t hoid(hobject_t("test_db_statistics", "", CEPH_NOSNAP, 0, 0, "")); | |
7659 | auto ch = bstore->create_new_collection(cid); | |
7660 | bufferlist bl; | |
7661 | bl.append("0123456789abcdefghi"); | |
7662 | { | |
7663 | ObjectStore::Transaction t; | |
7664 | t.create_collection(cid, 0); | |
7665 | t.touch(cid, hoid); | |
7666 | t.write(cid, hoid, 0, bl.length(), bl); | |
7667 | cerr << "Write object" << std::endl; | |
7668 | r = queue_transaction(bstore, ch, std::move(t)); | |
7669 | ASSERT_EQ(r, 0); | |
7670 | } | |
7671 | { | |
7672 | bufferlist readback; | |
7673 | r = store->read(ch, hoid, 0, bl.length(), readback); | |
7674 | ASSERT_EQ(static_cast<int>(bl.length()), r); | |
7675 | ASSERT_TRUE(bl_eq(bl, readback)); | |
7676 | } | |
7677 | Formatter *f = Formatter::create("store_test", "json-pretty", "json-pretty"); | |
7678 | EXPECT_NO_THROW(store->get_db_statistics(f)); | |
7679 | f->flush(cout); | |
7680 | cout << std::endl; | |
7681 | } | |
7682 | ||
9f95a23c TL |
7683 | TEST_P(StoreTest, BluestorePerPoolOmapFixOnMount) |
7684 | { | |
7685 | if (string(GetParam()) != "bluestore") | |
7686 | return; | |
7687 | ||
7688 | BlueStore* bstore = dynamic_cast<BlueStore*> (store.get()); | |
7689 | const uint64_t pool = 555; | |
7690 | coll_t cid(spg_t(pg_t(0, pool), shard_id_t::NO_SHARD)); | |
7691 | ghobject_t oid = make_object("Object 1", pool); | |
7692 | ghobject_t oid2 = make_object("Object 2", pool); | |
7693 | // fill the store with some data | |
7694 | auto ch = store->create_new_collection(cid); | |
7695 | map<string, bufferlist> omap; | |
7696 | bufferlist h; | |
7697 | h.append("header"); | |
7698 | { | |
7699 | omap["omap_key"].append("omap value"); | |
7700 | ObjectStore::Transaction t; | |
7701 | t.create_collection(cid, 0); | |
7702 | t.touch(cid, oid); | |
7703 | t.omap_setheader(cid, oid, h); | |
7704 | t.touch(cid, oid2); | |
7705 | t.omap_setheader(cid, oid2, h); | |
7706 | int r = queue_transaction(store, ch, std::move(t)); | |
7707 | ASSERT_EQ(r, 0); | |
7708 | } | |
7709 | ||
7710 | // inject legacy omaps | |
7711 | bstore->inject_legacy_omap(); | |
7712 | bstore->inject_legacy_omap(cid, oid); | |
7713 | bstore->inject_legacy_omap(cid, oid2); | |
7714 | ||
7715 | bstore->umount(); | |
7716 | ||
7717 | // check we injected an issue | |
7718 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "false"); | |
7719 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
7720 | g_ceph_context->_conf.apply_changes(nullptr); | |
7721 | ASSERT_EQ(bstore->fsck(false), 3); | |
7722 | ||
7723 | // set autofix and mount | |
7724 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7725 | g_ceph_context->_conf.apply_changes(nullptr); | |
7726 | bstore->mount(); | |
7727 | bstore->umount(); | |
7728 | ||
7729 | // check we fixed it.. | |
7730 | ASSERT_EQ(bstore->fsck(false), 0); | |
7731 | bstore->mount(); | |
7732 | ||
7733 | // | |
7734 | // Now repro https://tracker.ceph.com/issues/43824 | |
7735 | // | |
7736 | // inject legacy omaps again | |
7737 | bstore->inject_legacy_omap(); | |
7738 | bstore->inject_legacy_omap(cid, oid); | |
7739 | bstore->inject_legacy_omap(cid, oid2); | |
7740 | bstore->umount(); | |
7741 | ||
7742 | // check we injected an issue | |
7743 | SetVal(g_conf(), "bluestore_fsck_quick_fix_on_mount", "true"); | |
7744 | SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); | |
7745 | g_ceph_context->_conf.apply_changes(nullptr); | |
7746 | bstore->mount(); | |
7747 | ch = store->open_collection(cid); | |
7748 | ||
7749 | { | |
7750 | // write to onode which will partiall revert per-pool | |
7751 | // omap repair done on mount due to #43824. | |
7752 | // And object removal will leave stray per-pool omap recs | |
7753 | // | |
7754 | ObjectStore::Transaction t; | |
7755 | bufferlist bl; | |
7756 | bl.append("data"); | |
7757 | //this triggers onode rec update and hence legacy omap | |
7758 | t.write(cid, oid, 0, bl.length(), bl); | |
7759 | t.remove(cid, oid2); // this will trigger stray per-pool omap | |
7760 | int r = queue_transaction(store, ch, std::move(t)); | |
7761 | ASSERT_EQ(r, 0); | |
7762 | } | |
7763 | bstore->umount(); | |
7764 | // check omap's been fixed. | |
7765 | ASSERT_EQ(bstore->fsck(false), 0); // this will fail without fix for #43824 | |
7766 | ||
7767 | bstore->mount(); | |
7768 | } | |
7769 | ||
11fdf7f2 TL |
7770 | TEST_P(StoreTestSpecificAUSize, BluestoreTinyDevFailure) { |
7771 | if (string(GetParam()) != "bluestore") | |
7772 | return; | |
7773 | // This caused superblock overwrite by bluefs, see | |
7774 | // https://tracker.ceph.com/issues/24480 | |
7775 | SetVal(g_conf(), "bluestore_block_size", | |
7776 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
7777 | SetVal(g_conf(), "bluestore_block_db_size", "0"); | |
7778 | SetVal(g_conf(), "bluestore_block_db_create", "false"); | |
7779 | SetVal(g_conf(), "bluestore_bluefs_min", | |
7780 | stringify(1024 * 1024 * 1024).c_str()); | |
7781 | StartDeferred(0x1000); | |
7782 | store->umount(); | |
7783 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7784 | store->mount(); | |
7785 | } | |
7786 | ||
7787 | TEST_P(StoreTestSpecificAUSize, BluestoreTinyDevFailure2) { | |
7788 | if (string(GetParam()) != "bluestore") | |
7789 | return; | |
7790 | ||
7791 | // This caused assert in allocator as initial bluefs extent as slow device | |
7792 | // overlaped with superblock | |
7793 | // https://tracker.ceph.com/issues/24480 | |
7794 | SetVal(g_conf(), "bluestore_block_size", | |
7795 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
7796 | SetVal(g_conf(), "bluestore_block_db_size", | |
7797 | stringify(1024 * 1024 * 1024).c_str()); //1 Gb | |
7798 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
7799 | SetVal(g_conf(), "bluestore_bluefs_min", | |
7800 | stringify(1024 * 1024 * 1024).c_str()); | |
7801 | StartDeferred(0x1000); | |
7802 | store->umount(); | |
7803 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7804 | store->mount(); | |
b32b8144 FG |
7805 | } |
7806 | ||
f64942e4 AA |
7807 | TEST_P(StoreTest, SpuriousReadErrorTest) { |
7808 | if (string(GetParam()) != "bluestore") | |
7809 | return; | |
7810 | ||
f64942e4 AA |
7811 | int r; |
7812 | auto logger = store->get_perf_counters(); | |
7813 | coll_t cid; | |
11fdf7f2 | 7814 | auto ch = store->create_new_collection(cid); |
f64942e4 AA |
7815 | ghobject_t hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP))); |
7816 | { | |
7817 | ObjectStore::Transaction t; | |
7818 | t.create_collection(cid, 0); | |
7819 | cerr << "Creating collection " << cid << std::endl; | |
11fdf7f2 | 7820 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
7821 | ASSERT_EQ(r, 0); |
7822 | } | |
7823 | bufferlist test_data; | |
7824 | bufferptr ap(0x2000); | |
7825 | memset(ap.c_str(), 'a', 0x2000); | |
7826 | test_data.append(ap); | |
7827 | { | |
7828 | ObjectStore::Transaction t; | |
7829 | t.write(cid, hoid, 0, 0x2000, test_data); | |
11fdf7f2 | 7830 | r = queue_transaction(store, ch, std::move(t)); |
f64942e4 AA |
7831 | ASSERT_EQ(r, 0); |
7832 | // force cache clear | |
7833 | EXPECT_EQ(store->umount(), 0); | |
7834 | EXPECT_EQ(store->mount(), 0); | |
7835 | } | |
f6b5b4d7 | 7836 | ch = store->open_collection(cid); |
f64942e4 AA |
7837 | |
7838 | cerr << "Injecting CRC error with no retry, expecting EIO" << std::endl; | |
11fdf7f2 TL |
7839 | SetVal(g_conf(), "bluestore_retry_disk_reads", "0"); |
7840 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "1"); | |
7841 | g_ceph_context->_conf.apply_changes(nullptr); | |
f64942e4 AA |
7842 | { |
7843 | bufferlist in; | |
11fdf7f2 | 7844 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
7845 | ASSERT_EQ(-EIO, r); |
7846 | ASSERT_EQ(logger->get(l_bluestore_read_eio), 1u); | |
7847 | ASSERT_EQ(logger->get(l_bluestore_reads_with_retries), 0u); | |
7848 | } | |
7849 | ||
7850 | cerr << "Injecting CRC error with retries, expecting success after several retries" << std::endl; | |
11fdf7f2 TL |
7851 | SetVal(g_conf(), "bluestore_retry_disk_reads", "255"); |
7852 | SetVal(g_conf(), "bluestore_debug_inject_csum_err_probability", "0.8"); | |
f64942e4 AA |
7853 | /** |
7854 | * Probabilistic test: 25 reads, each has a 80% chance of failing with 255 retries | |
7855 | * Probability of at least one retried read: 1 - (0.2 ** 25) = 100% - 3e-18 | |
7856 | * Probability of a random test failure: 1 - ((1 - (0.8 ** 255)) ** 25) ~= 5e-24 | |
7857 | */ | |
11fdf7f2 | 7858 | g_ceph_context->_conf.apply_changes(nullptr); |
f64942e4 AA |
7859 | { |
7860 | for (int i = 0; i < 25; ++i) { | |
7861 | bufferlist in; | |
11fdf7f2 | 7862 | r = store->read(ch, hoid, 0, 0x2000, in, CEPH_OSD_OP_FLAG_FADVISE_NOCACHE); |
f64942e4 AA |
7863 | ASSERT_EQ(0x2000, r); |
7864 | ASSERT_TRUE(bl_eq(test_data, in)); | |
7865 | } | |
7866 | ASSERT_GE(logger->get(l_bluestore_reads_with_retries), 1u); | |
7867 | } | |
11fdf7f2 TL |
7868 | } |
7869 | ||
7870 | TEST_P(StoreTest, allocateBlueFSTest) { | |
7871 | if (string(GetParam()) != "bluestore") | |
7872 | return; | |
7873 | ||
7874 | BlueStore* bstore = NULL; | |
7875 | EXPECT_NO_THROW(bstore = dynamic_cast<BlueStore*> (store.get())); | |
7876 | ||
7877 | struct store_statfs_t statfs; | |
7878 | store->statfs(&statfs); | |
7879 | ||
7880 | uint64_t to_alloc = g_conf().get_val<Option::size_t>("bluefs_alloc_size"); | |
7881 | ||
7882 | int r = bstore->allocate_bluefs_freespace(to_alloc, to_alloc, nullptr); | |
7883 | ASSERT_EQ(r, 0); | |
7884 | r = bstore->allocate_bluefs_freespace(statfs.total, statfs.total, nullptr); | |
7885 | ASSERT_EQ(r, -ENOSPC); | |
7886 | r = bstore->allocate_bluefs_freespace(to_alloc * 16, to_alloc * 16, nullptr); | |
7887 | ASSERT_EQ(r, 0); | |
7888 | store->umount(); | |
7889 | ASSERT_EQ(store->fsck(false), 0); // do fsck explicitly | |
7890 | r = store->mount(); | |
7891 | ASSERT_EQ(r, 0); | |
7892 | } | |
7893 | ||
7894 | TEST_P(StoreTest, mergeRegionTest) { | |
7895 | if (string(GetParam()) != "bluestore") | |
7896 | return; | |
7897 | ||
7898 | SetVal(g_conf(), "bluestore_fsck_on_mount", "true"); | |
7899 | SetVal(g_conf(), "bluestore_fsck_on_umount", "true"); | |
7900 | SetVal(g_conf(), "bdev_debug_inflight_ios", "true"); | |
7901 | g_ceph_context->_conf.apply_changes(nullptr); | |
7902 | ||
7903 | uint32_t chunk_size = g_ceph_context->_conf->bdev_block_size; | |
7904 | int r = -1; | |
7905 | coll_t cid; | |
7906 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
7907 | auto ch = store->create_new_collection(cid); | |
7908 | { | |
7909 | ObjectStore::Transaction t; | |
7910 | t.create_collection(cid, 0); | |
7911 | r = queue_transaction(store, ch, std::move(t)); | |
7912 | ASSERT_EQ(r, 0); | |
7913 | } | |
7914 | { | |
7915 | ObjectStore::Transaction t; | |
7916 | t.touch(cid, hoid); | |
7917 | cerr << "Creating object " << hoid << std::endl; | |
7918 | r = queue_transaction(store, ch, std::move(t)); | |
7919 | ASSERT_EQ(r, 0); | |
7920 | } | |
7921 | bufferlist bl5; | |
7922 | bl5.append("abcde"); | |
7923 | uint64_t offset = 0; | |
7924 | { // 1. same region | |
7925 | ObjectStore::Transaction t; | |
7926 | t.write(cid, hoid, offset, 5, bl5); | |
7927 | t.write(cid, hoid, 0xa + offset, 5, bl5); | |
7928 | t.write(cid, hoid, 0x14 + offset, 5, bl5); | |
7929 | r = queue_transaction(store, ch, std::move(t)); | |
7930 | ASSERT_EQ(r, 0); | |
7931 | } | |
7932 | { // 2. adjacent regions | |
7933 | ObjectStore::Transaction t; | |
7934 | offset = chunk_size; | |
7935 | t.write(cid, hoid, offset, 5, bl5); | |
7936 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
7937 | r = queue_transaction(store, ch, std::move(t)); | |
7938 | ASSERT_EQ(r, 0); | |
7939 | } | |
7940 | { // 3. front merge | |
7941 | ObjectStore::Transaction t; | |
7942 | offset = chunk_size * 2; | |
7943 | t.write(cid, hoid, offset, 5, bl5); | |
7944 | t.write(cid, hoid, offset + chunk_size - 2, 5, bl5); | |
7945 | r = queue_transaction(store, ch, std::move(t)); | |
7946 | ASSERT_EQ(r, 0); | |
7947 | } | |
7948 | { // 4. back merge | |
7949 | ObjectStore::Transaction t; | |
7950 | bufferlist blc2; | |
7951 | blc2.append_zero(chunk_size + 2); | |
f64942e4 | 7952 | |
11fdf7f2 TL |
7953 | offset = chunk_size * 3; |
7954 | t.write(cid, hoid, offset, chunk_size + 2, blc2); | |
7955 | t.write(cid, hoid, offset + chunk_size + 3, 5, bl5); | |
7956 | r = queue_transaction(store, ch, std::move(t)); | |
7957 | ASSERT_EQ(r, 0); | |
7958 | } | |
7959 | { // 5. overlapping | |
7960 | ObjectStore::Transaction t; | |
7961 | uint64_t final_len = 0; | |
7962 | offset = chunk_size * 10; | |
7963 | bufferlist bl2c2; | |
7964 | bl2c2.append_zero(chunk_size * 2); | |
7965 | t.write(cid, hoid, offset + chunk_size * 3 - 3, chunk_size * 2, bl2c2); | |
7966 | bl2c2.append_zero(2); | |
7967 | t.write(cid, hoid, offset + chunk_size - 2, chunk_size * 2 + 2, bl2c2); | |
7968 | r = queue_transaction(store, ch, std::move(t)); | |
7969 | ASSERT_EQ(r, 0); | |
7970 | ||
7971 | final_len = (offset + chunk_size * 3 - 3) + (chunk_size * 2); | |
7972 | bufferlist bl; | |
7973 | r = store->read(ch, hoid, 0, final_len, bl); | |
7974 | ASSERT_EQ(final_len, static_cast<uint64_t>(r)); | |
7975 | } | |
f64942e4 | 7976 | } |
9f95a23c TL |
7977 | |
7978 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { | |
7979 | if (string(GetParam()) != "bluestore") | |
7980 | return; | |
7981 | ||
7982 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); | |
7983 | StartDeferred(0x1000); | |
7984 | ||
7985 | int r; | |
7986 | coll_t cid; | |
7987 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
7988 | auto ch = store->create_new_collection(cid); | |
7989 | { | |
7990 | ObjectStore::Transaction t; | |
7991 | t.create_collection(cid, 0); | |
7992 | cerr << "Creating collection " << cid << std::endl; | |
7993 | r = queue_transaction(store, ch, std::move(t)); | |
7994 | ASSERT_EQ(r, 0); | |
7995 | } | |
7996 | { | |
7997 | ObjectStore::Transaction t; | |
7998 | bufferlist bl, orig; | |
7999 | string s(g_ceph_context->_conf->bluestore_max_blob_size_hdd, '0'); | |
8000 | bl.append(s); | |
8001 | t.write(cid, hoid, 0, bl.length(), bl); | |
8002 | cerr << "write" << std::endl; | |
8003 | r = queue_transaction(store, ch, std::move(t)); | |
8004 | ASSERT_EQ(r, 0); | |
8005 | ||
8006 | const PerfCounters* logger = store->get_perf_counters(); | |
8007 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 1u); | |
8008 | } | |
8009 | } | |
8010 | ||
8011 | TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsSsd) { | |
8012 | if (string(GetParam()) != "bluestore") | |
8013 | return; | |
8014 | ||
8015 | SetVal(g_conf(), "bluestore_debug_enforce_settings", "ssd"); | |
8016 | StartDeferred(0x1000); | |
8017 | ||
8018 | int r; | |
8019 | coll_t cid; | |
8020 | ghobject_t hoid(hobject_t(sobject_t("Object", CEPH_NOSNAP))); | |
8021 | auto ch = store->create_new_collection(cid); | |
8022 | { | |
8023 | ObjectStore::Transaction t; | |
8024 | t.create_collection(cid, 0); | |
8025 | cerr << "Creating collection " << cid << std::endl; | |
8026 | r = queue_transaction(store, ch, std::move(t)); | |
8027 | ASSERT_EQ(r, 0); | |
8028 | } | |
8029 | { | |
8030 | ObjectStore::Transaction t; | |
8031 | bufferlist bl, orig; | |
8032 | string s(g_ceph_context->_conf->bluestore_max_blob_size_ssd * 8, '0'); | |
8033 | bl.append(s); | |
8034 | t.write(cid, hoid, 0, bl.length(), bl); | |
8035 | cerr << "write" << std::endl; | |
8036 | r = queue_transaction(store, ch, std::move(t)); | |
8037 | ASSERT_EQ(r, 0); | |
8038 | ||
8039 | const PerfCounters* logger = store->get_perf_counters(); | |
8040 | ASSERT_EQ(logger->get(l_bluestore_write_big_blobs), 8u); | |
8041 | } | |
8042 | } | |
eafe8130 TL |
8043 | |
8044 | TEST_P(StoreTestSpecificAUSize, ReproNoBlobMultiTest) { | |
8045 | ||
8046 | if(string(GetParam()) != "bluestore") | |
8047 | return; | |
8048 | ||
8049 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8050 | SetVal(g_conf(), "bluestore_block_db_size", "4294967296"); | |
8051 | SetVal(g_conf(), "bluestore_block_size", "12884901888"); | |
8052 | SetVal(g_conf(), "bluestore_max_blob_size", "524288"); | |
8053 | ||
8054 | g_conf().apply_changes(nullptr); | |
8055 | ||
8056 | StartDeferred(65536); | |
8057 | ||
8058 | int r; | |
8059 | coll_t cid; | |
8060 | ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); | |
8061 | ghobject_t hoid2 = hoid; | |
8062 | hoid2.hobj.snap = 1; | |
8063 | ||
8064 | auto ch = store->create_new_collection(cid); | |
8065 | { | |
8066 | ObjectStore::Transaction t; | |
8067 | t.create_collection(cid, 0); | |
8068 | cerr << "Creating collection " << cid << std::endl; | |
8069 | r = queue_transaction(store, ch, std::move(t)); | |
8070 | ASSERT_EQ(r, 0); | |
8071 | } | |
8072 | { | |
8073 | bool exists = store->exists(ch, hoid); | |
8074 | ASSERT_TRUE(!exists); | |
8075 | ||
8076 | ObjectStore::Transaction t; | |
8077 | t.touch(cid, hoid); | |
8078 | cerr << "Creating object " << hoid << std::endl; | |
8079 | r = queue_transaction(store, ch, std::move(t)); | |
8080 | ASSERT_EQ(r, 0); | |
8081 | ||
8082 | exists = store->exists(ch, hoid); | |
8083 | ASSERT_EQ(true, exists); | |
8084 | } | |
8085 | { | |
8086 | uint64_t offs = 0; | |
8087 | bufferlist bl; | |
8088 | const int size = 0x100; | |
8089 | bufferptr ap(size); | |
8090 | memset(ap.c_str(), 'a', size); | |
8091 | bl.append(ap); | |
8092 | int i = 0; | |
8093 | uint64_t blob_size = 524288; | |
8094 | uint64_t total = 0; | |
8095 | for (i = 0; i <= 512; i++) { | |
8096 | offs = 0 + i * size; | |
8097 | ObjectStore::Transaction t; | |
8098 | ghobject_t hoid2 = hoid; | |
8099 | hoid2.hobj.snap = i + 1; | |
8100 | while (offs < 128 * 1024 * 1024) { | |
8101 | ||
8102 | t.write(cid, hoid, offs, ap.length(), bl); | |
8103 | offs += blob_size; | |
8104 | total += ap.length(); | |
8105 | } | |
8106 | t.clone(cid, hoid, hoid2); | |
8107 | r = queue_transaction(store, ch, std::move(t)); | |
8108 | ASSERT_EQ(r, 0); | |
8109 | } | |
8110 | cerr << "Total written = " << total << std::endl; | |
8111 | } | |
8112 | { | |
8113 | cerr << "Finalizing" << std::endl; | |
8114 | const PerfCounters* logger = store->get_perf_counters(); | |
8115 | ASSERT_GE(logger->get(l_bluestore_gc_merged), 1024*1024*1024); | |
8116 | } | |
8117 | } | |
8118 | ||
9f95a23c TL |
8119 | void doManySetAttr(ObjectStore* store, |
8120 | std::function<void(ObjectStore*)> do_check_fn) | |
8121 | { | |
8122 | MixedGenerator gen(447); | |
8123 | gen_type rng(time(NULL)); | |
8124 | coll_t cid(spg_t(pg_t(0, 447), shard_id_t::NO_SHARD)); | |
8125 | ||
f6b5b4d7 | 8126 | SyntheticWorkloadState test_obj(store, &gen, &rng, cid, 0, 0, 0); |
9f95a23c | 8127 | test_obj.init(); |
f6b5b4d7 TL |
8128 | size_t object_count = 256; |
8129 | for (size_t i = 0; i < object_count; ++i) { | |
9f95a23c TL |
8130 | if (!(i % 10)) cerr << "seeding object " << i << std::endl; |
8131 | test_obj.touch(); | |
8132 | } | |
f6b5b4d7 | 8133 | for (size_t i = 0; i < object_count; ++i) { |
9f95a23c TL |
8134 | if (!(i % 100)) { |
8135 | cerr << "Op " << i << std::endl; | |
8136 | test_obj.print_internal_state(); | |
8137 | } | |
f6b5b4d7 | 8138 | test_obj.set_fixed_attrs(1024, 64, 4096); // 1024 attributes, 64 bytes name and 4K value |
9f95a23c TL |
8139 | } |
8140 | test_obj.wait_for_done(); | |
8141 | ||
f6b5b4d7 TL |
8142 | std::cout << "done" << std::endl; |
8143 | do_check_fn(store); | |
9f95a23c TL |
8144 | AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); |
8145 | ceph_assert(admin_socket); | |
8146 | ||
8147 | ceph::bufferlist in, out; | |
8148 | ostringstream err; | |
8149 | ||
f6b5b4d7 TL |
8150 | auto r = admin_socket->execute_command( |
8151 | { "{\"prefix\": \"bluefs stats\"}" }, | |
9f95a23c | 8152 | in, err, &out); |
f6b5b4d7 TL |
8153 | if (r != 0) { |
8154 | cerr << "failure querying: " << cpp_strerror(r) << std::endl; | |
8155 | } else { | |
8156 | std::cout << std::string(out.c_str(), out.length()) << std::endl; | |
9f95a23c | 8157 | } |
9f95a23c TL |
8158 | do_check_fn(store); |
8159 | test_obj.shutdown(); | |
8160 | } | |
8161 | ||
8162 | TEST_P(StoreTestSpecificAUSize, SpilloverTest) { | |
8163 | if (string(GetParam()) != "bluestore") | |
8164 | return; | |
8165 | ||
8166 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8167 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8168 | SetVal(g_conf(), "bluestore_volume_selection_policy", "rocksdb_original"); | |
8169 | ||
8170 | g_conf().apply_changes(nullptr); | |
8171 | ||
8172 | StartDeferred(65536); | |
8173 | doManySetAttr(store.get(), | |
8174 | [&](ObjectStore* _store) { | |
8175 | ||
8176 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8177 | ceph_assert(bstore); | |
f6b5b4d7 | 8178 | bstore->compact(); |
9f95a23c TL |
8179 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8180 | //experimentally it was discovered that this case results in 400+MB spillover | |
8181 | //using lower 300MB threshold just to be safe enough | |
f6b5b4d7 TL |
8182 | std::cout << "db_used:" << logger->get(l_bluefs_db_used_bytes) << std::endl; |
8183 | std::cout << "slow_used:" << logger->get(l_bluefs_slow_used_bytes) << std::endl; | |
9f95a23c | 8184 | |
f6b5b4d7 TL |
8185 | // Disabling any validation/assertion for now as it looks like |
8186 | // we're unable to 100% force RocksDB to spillover. | |
8187 | // Leaving test case hoping to fix that one day though. | |
8188 | //ASSERT_GE(logger->get(l_bluefs_slow_used_bytes), 16 * 1024 * 1024); | |
9f95a23c TL |
8189 | } |
8190 | ); | |
8191 | } | |
8192 | ||
8193 | TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) { | |
8194 | if (string(GetParam()) != "bluestore") | |
8195 | return; | |
8196 | ||
8197 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8198 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8199 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
8200 | SetVal(g_conf(), "bluestore_volume_selection_reserved", "1"); // just use non-zero to enable | |
8201 | ||
8202 | g_conf().apply_changes(nullptr); | |
8203 | ||
8204 | StartDeferred(65536); | |
8205 | doManySetAttr(store.get(), | |
8206 | [&](ObjectStore* _store) { | |
8207 | ||
8208 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8209 | ceph_assert(bstore); | |
f6b5b4d7 | 8210 | bstore->compact(); |
9f95a23c TL |
8211 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8212 | ASSERT_EQ(0, logger->get(l_bluefs_slow_used_bytes)); | |
8213 | } | |
8214 | ); | |
8215 | } | |
8216 | ||
8217 | TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) { | |
8218 | if (string(GetParam()) != "bluestore") | |
8219 | return; | |
8220 | ||
8221 | SetVal(g_conf(), "bluestore_block_db_create", "true"); | |
8222 | SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); | |
8223 | SetVal(g_conf(), "bluestore_volume_selection_policy", "use_some_extra"); | |
8224 | //default 2.0 factor results in too high threshold, using less value | |
8225 | // that results in less but still present spillover. | |
8226 | SetVal(g_conf(), "bluestore_volume_selection_reserved_factor", "0.5"); | |
8227 | ||
8228 | g_conf().apply_changes(nullptr); | |
8229 | ||
8230 | StartDeferred(65536); | |
8231 | doManySetAttr(store.get(), | |
8232 | [&](ObjectStore* _store) { | |
8233 | ||
8234 | BlueStore* bstore = dynamic_cast<BlueStore*> (_store); | |
8235 | ceph_assert(bstore); | |
f6b5b4d7 | 8236 | bstore->compact(); |
9f95a23c TL |
8237 | const PerfCounters* logger = bstore->get_bluefs_perf_counters(); |
8238 | ASSERT_LE(logger->get(l_bluefs_slow_used_bytes), 300 * 1024 * 1024); // see SpilloverTest for 300MB choice rationale | |
8239 | } | |
8240 | ); | |
8241 | } | |
8242 | ||
11fdf7f2 | 8243 | #endif // WITH_BLUESTORE |
f64942e4 | 8244 | |
7c673cae FG |
8245 | int main(int argc, char **argv) { |
8246 | vector<const char*> args; | |
8247 | argv_to_vec(argc, (const char **)argv, args); | |
7c673cae FG |
8248 | |
8249 | auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, | |
11fdf7f2 TL |
8250 | CODE_ENVIRONMENT_UTILITY, |
8251 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
7c673cae FG |
8252 | common_init_finish(g_ceph_context); |
8253 | ||
11fdf7f2 TL |
8254 | // make sure we can adjust any config settings |
8255 | g_ceph_context->_conf._clear_safe_to_start_threads(); | |
8256 | ||
8257 | g_ceph_context->_conf.set_val_or_die("osd_journal_size", "400"); | |
8258 | g_ceph_context->_conf.set_val_or_die("filestore_index_retry_probability", "0.5"); | |
8259 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_timeout", "1000"); | |
8260 | g_ceph_context->_conf.set_val_or_die("filestore_op_thread_suicide_timeout", "10000"); | |
8261 | //g_ceph_context->_conf.set_val_or_die("filestore_fiemap", "true"); | |
8262 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mkfs", "false"); | |
8263 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_mount", "false"); | |
8264 | g_ceph_context->_conf.set_val_or_die("bluestore_fsck_on_umount", "false"); | |
8265 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_misc", "true"); | |
8266 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_small_allocations", "4"); | |
8267 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_freelist", "true"); | |
8268 | g_ceph_context->_conf.set_val_or_die("bluestore_clone_cow", "true"); | |
8269 | g_ceph_context->_conf.set_val_or_die("bluestore_max_alloc_size", "196608"); | |
7c673cae FG |
8270 | |
8271 | // set small cache sizes so we see trimming during Synthetic tests | |
11fdf7f2 TL |
8272 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_hdd", "4000000"); |
8273 | g_ceph_context->_conf.set_val_or_die("bluestore_cache_size_ssd", "4000000"); | |
7c673cae FG |
8274 | |
8275 | // very short *_max prealloc so that we fall back to async submits | |
11fdf7f2 TL |
8276 | g_ceph_context->_conf.set_val_or_die("bluestore_blobid_prealloc", "10"); |
8277 | g_ceph_context->_conf.set_val_or_die("bluestore_nid_prealloc", "10"); | |
8278 | g_ceph_context->_conf.set_val_or_die("bluestore_debug_randomize_serial_transaction", | |
7c673cae FG |
8279 | "10"); |
8280 | ||
11fdf7f2 | 8281 | g_ceph_context->_conf.set_val_or_die("bdev_debug_aio", "true"); |
7c673cae FG |
8282 | |
8283 | // specify device size | |
11fdf7f2 | 8284 | g_ceph_context->_conf.set_val_or_die("bluestore_block_size", |
94b18763 | 8285 | stringify(DEF_STORE_TEST_BLOCKDEV_SIZE)); |
7c673cae | 8286 | |
11fdf7f2 | 8287 | g_ceph_context->_conf.set_val_or_die( |
7c673cae | 8288 | "enable_experimental_unrecoverable_data_corrupting_features", "*"); |
11fdf7f2 | 8289 | g_ceph_context->_conf.apply_changes(nullptr); |
7c673cae FG |
8290 | |
8291 | ::testing::InitGoogleTest(&argc, argv); | |
8292 | return RUN_ALL_TESTS(); | |
8293 | } | |
8294 | ||
8295 | /* | |
8296 | * Local Variables: | |
8297 | * compile-command: "cd ../.. ; make ceph_test_objectstore && | |
8298 | * ./ceph_test_objectstore \ | |
8299 | * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20 | |
8300 | * " | |
8301 | * End: | |
8302 | */ |