]> git.proxmox.com Git - ceph.git/blob - ceph/src/test/objectstore/test_bluestore_types.cc
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import ceph 16.2.7
[ceph.git] / ceph / src / test / objectstore / test_bluestore_types.cc
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3
4 #include "include/types.h"
5 #include "os/bluestore/bluestore_types.h"
6 #include "gtest/gtest.h"
7 #include "include/stringify.h"
8 #include "common/ceph_time.h"
9 #include "os/bluestore/BlueStore.h"
10 #include "os/bluestore/AvlAllocator.h"
11 #include "common/ceph_argparse.h"
12 #include "global/global_init.h"
13 #include "global/global_context.h"
14
15 #include <sstream>
16
17 #define _STR(x) #x
18 #define STRINGIFY(x) _STR(x)
19
20 TEST(bluestore, sizeof) {
21 #define P(t) cout << STRINGIFY(t) << "\t" << sizeof(t) << std::endl
22 P(BlueStore::Onode);
23 P(BlueStore::Extent);
24 P(BlueStore::Blob);
25 P(BlueStore::SharedBlob);
26 P(BlueStore::ExtentMap);
27 P(BlueStore::extent_map_t);
28 P(BlueStore::blob_map_t);
29 P(BlueStore::BufferSpace);
30 P(BlueStore::Buffer);
31 P(bluestore_onode_t);
32 P(bluestore_blob_t);
33 P(PExtentVector);
34 P(bluestore_shared_blob_t);
35 P(bluestore_extent_ref_map_t);
36 P(bluestore_extent_ref_map_t::record_t);
37 P(bluestore_blob_use_tracker_t);
38 P(std::atomic_int);
39 P(BlueStore::SharedBlobRef);
40 P(boost::intrusive::set_base_hook<>);
41 P(boost::intrusive::unordered_set_base_hook<>);
42 P(bufferlist);
43 P(bufferptr);
44 P(range_seg_t);
45 cout << "map<uint64_t,uint64_t>\t" << sizeof(map<uint64_t,uint64_t>) << std::endl;
46 cout << "map<char,char>\t" << sizeof(map<char,char>) << std::endl;
47 }
48
49 TEST(bluestore_extent_ref_map_t, add)
50 {
51 bluestore_extent_ref_map_t m;
52 m.get(10, 10);
53 ASSERT_EQ(1u, m.ref_map.size());
54 cout << m << std::endl;
55 m.get(20, 10);
56 cout << m << std::endl;
57 ASSERT_EQ(1u, m.ref_map.size());
58 ASSERT_EQ(20u, m.ref_map[10].length);
59 ASSERT_EQ(1u, m.ref_map[10].refs);
60 m.get(40, 10);
61 cout << m << std::endl;
62 ASSERT_EQ(2u, m.ref_map.size());
63 m.get(30, 10);
64 cout << m << std::endl;
65 ASSERT_EQ(1u, m.ref_map.size());
66 m.get(50, 10);
67 cout << m << std::endl;
68 ASSERT_EQ(1u, m.ref_map.size());
69 m.get(5, 5);
70 cout << m << std::endl;
71 ASSERT_EQ(1u, m.ref_map.size());
72 }
73
74 TEST(bluestore_extent_ref_map_t, get)
75 {
76 bluestore_extent_ref_map_t m;
77 m.get(00, 30);
78 cout << m << std::endl;
79 m.get(10, 10);
80 cout << m << std::endl;
81 ASSERT_EQ(3u, m.ref_map.size());
82 ASSERT_EQ(10u, m.ref_map[0].length);
83 ASSERT_EQ(1u, m.ref_map[0].refs);
84 ASSERT_EQ(10u, m.ref_map[10].length);
85 ASSERT_EQ(2u, m.ref_map[10].refs);
86 ASSERT_EQ(10u, m.ref_map[20].length);
87 ASSERT_EQ(1u, m.ref_map[20].refs);
88 m.get(20, 5);
89 cout << m << std::endl;
90 ASSERT_EQ(3u, m.ref_map.size());
91 ASSERT_EQ(15u, m.ref_map[10].length);
92 ASSERT_EQ(2u, m.ref_map[10].refs);
93 ASSERT_EQ(5u, m.ref_map[25].length);
94 ASSERT_EQ(1u, m.ref_map[25].refs);
95 m.get(5, 20);
96 cout << m << std::endl;
97 ASSERT_EQ(4u, m.ref_map.size());
98 ASSERT_EQ(5u, m.ref_map[0].length);
99 ASSERT_EQ(1u, m.ref_map[0].refs);
100 ASSERT_EQ(5u, m.ref_map[5].length);
101 ASSERT_EQ(2u, m.ref_map[5].refs);
102 ASSERT_EQ(15u, m.ref_map[10].length);
103 ASSERT_EQ(3u, m.ref_map[10].refs);
104 ASSERT_EQ(5u, m.ref_map[25].length);
105 ASSERT_EQ(1u, m.ref_map[25].refs);
106 m.get(25, 3);
107 cout << m << std::endl;
108 ASSERT_EQ(5u, m.ref_map.size());
109 ASSERT_EQ(5u, m.ref_map[0].length);
110 ASSERT_EQ(1u, m.ref_map[0].refs);
111 ASSERT_EQ(5u, m.ref_map[5].length);
112 ASSERT_EQ(2u, m.ref_map[5].refs);
113 ASSERT_EQ(15u, m.ref_map[10].length);
114 ASSERT_EQ(3u, m.ref_map[10].refs);
115 ASSERT_EQ(3u, m.ref_map[25].length);
116 ASSERT_EQ(2u, m.ref_map[25].refs);
117 ASSERT_EQ(2u, m.ref_map[28].length);
118 ASSERT_EQ(1u, m.ref_map[28].refs);
119 }
120
121 TEST(bluestore_extent_ref_map_t, put)
122 {
123 bluestore_extent_ref_map_t m;
124 PExtentVector r;
125 bool maybe_unshared = false;
126 m.get(10, 30);
127 maybe_unshared = true;
128 m.put(10, 30, &r, &maybe_unshared);
129 cout << m << " " << r << " " << (int)maybe_unshared << std::endl;
130 ASSERT_EQ(0u, m.ref_map.size());
131 ASSERT_EQ(1u, r.size());
132 ASSERT_EQ(10u, r[0].offset);
133 ASSERT_EQ(30u, r[0].length);
134 ASSERT_TRUE(maybe_unshared);
135 r.clear();
136 m.get(10, 30);
137 m.get(20, 10);
138 maybe_unshared = true;
139 m.put(10, 30, &r, &maybe_unshared);
140 cout << m << " " << r << " " << (int)maybe_unshared << std::endl;
141 ASSERT_EQ(1u, m.ref_map.size());
142 ASSERT_EQ(10u, m.ref_map[20].length);
143 ASSERT_EQ(1u, m.ref_map[20].refs);
144 ASSERT_EQ(2u, r.size());
145 ASSERT_EQ(10u, r[0].offset);
146 ASSERT_EQ(10u, r[0].length);
147 ASSERT_EQ(30u, r[1].offset);
148 ASSERT_EQ(10u, r[1].length);
149 ASSERT_TRUE(maybe_unshared);
150 r.clear();
151 m.get(30, 10);
152 m.get(30, 10);
153 maybe_unshared = true;
154 m.put(20, 15, &r, &maybe_unshared);
155 cout << m << " " << r << " " << (int)maybe_unshared << std::endl;
156 ASSERT_EQ(2u, m.ref_map.size());
157 ASSERT_EQ(5u, m.ref_map[30].length);
158 ASSERT_EQ(1u, m.ref_map[30].refs);
159 ASSERT_EQ(5u, m.ref_map[35].length);
160 ASSERT_EQ(2u, m.ref_map[35].refs);
161 ASSERT_EQ(1u, r.size());
162 ASSERT_EQ(20u, r[0].offset);
163 ASSERT_EQ(10u, r[0].length);
164 ASSERT_FALSE(maybe_unshared);
165 r.clear();
166 maybe_unshared = true;
167 m.put(33, 5, &r, &maybe_unshared);
168 cout << m << " " << r << " " << (int)maybe_unshared << std::endl;
169 ASSERT_EQ(3u, m.ref_map.size());
170 ASSERT_EQ(3u, m.ref_map[30].length);
171 ASSERT_EQ(1u, m.ref_map[30].refs);
172 ASSERT_EQ(3u, m.ref_map[35].length);
173 ASSERT_EQ(1u, m.ref_map[35].refs);
174 ASSERT_EQ(2u, m.ref_map[38].length);
175 ASSERT_EQ(2u, m.ref_map[38].refs);
176 ASSERT_EQ(1u, r.size());
177 ASSERT_EQ(33u, r[0].offset);
178 ASSERT_EQ(2u, r[0].length);
179 ASSERT_FALSE(maybe_unshared);
180 r.clear();
181 maybe_unshared = true;
182 m.put(38, 2, &r, &maybe_unshared);
183 cout << m << " " << r << " " << (int)maybe_unshared << std::endl;
184 ASSERT_TRUE(maybe_unshared);
185 }
186
187 TEST(bluestore_extent_ref_map_t, contains)
188 {
189 bluestore_extent_ref_map_t m;
190 m.get(10, 30);
191 ASSERT_TRUE(m.contains(10, 30));
192 ASSERT_TRUE(m.contains(10, 10));
193 ASSERT_TRUE(m.contains(30, 10));
194 ASSERT_FALSE(m.contains(0, 10));
195 ASSERT_FALSE(m.contains(0, 20));
196 ASSERT_FALSE(m.contains(0, 100));
197 ASSERT_FALSE(m.contains(40, 10));
198 ASSERT_FALSE(m.contains(30, 11));
199 m.get(40, 10);
200 m.get(40, 10);
201 ASSERT_TRUE(m.contains(30, 11));
202 ASSERT_TRUE(m.contains(30, 20));
203 ASSERT_TRUE(m.contains(10, 40));
204 ASSERT_FALSE(m.contains(0, 50));
205 ASSERT_FALSE(m.contains(40, 20));
206 m.get(60, 100);
207 ASSERT_TRUE(m.contains(60, 10));
208 ASSERT_TRUE(m.contains(40, 10));
209 ASSERT_FALSE(m.contains(40, 11));
210 ASSERT_FALSE(m.contains(40, 20));
211 ASSERT_FALSE(m.contains(40, 30));
212 ASSERT_FALSE(m.contains(40, 3000));
213 ASSERT_FALSE(m.contains(4000, 30));
214 }
215
216 TEST(bluestore_extent_ref_map_t, intersects)
217 {
218 bluestore_extent_ref_map_t m;
219 m.get(10, 30);
220 ASSERT_TRUE(m.intersects(10, 30));
221 ASSERT_TRUE(m.intersects(0, 11));
222 ASSERT_TRUE(m.intersects(10, 40));
223 ASSERT_TRUE(m.intersects(15, 40));
224 ASSERT_FALSE(m.intersects(0, 10));
225 ASSERT_FALSE(m.intersects(0, 5));
226 ASSERT_FALSE(m.intersects(40, 20));
227 ASSERT_FALSE(m.intersects(41, 20));
228 m.get(40, 10);
229 m.get(40, 10);
230 ASSERT_TRUE(m.intersects(0, 100));
231 ASSERT_TRUE(m.intersects(10, 35));
232 ASSERT_TRUE(m.intersects(45, 10));
233 ASSERT_FALSE(m.intersects(50, 5));
234 m.get(60, 100);
235 ASSERT_TRUE(m.intersects(45, 10));
236 ASSERT_TRUE(m.intersects(55, 10));
237 ASSERT_TRUE(m.intersects(50, 11));
238 ASSERT_FALSE(m.intersects(50, 10));
239 ASSERT_FALSE(m.intersects(51, 9));
240 ASSERT_FALSE(m.intersects(55, 1));
241 }
242
243 TEST(bluestore_blob_t, calc_csum)
244 {
245 bufferlist bl;
246 bl.append("asdfghjkqwertyuizxcvbnm,");
247 bufferlist bl2;
248 bl2.append("xxxxXXXXyyyyYYYYzzzzZZZZ");
249 bufferlist f;
250 f.substr_of(bl, 0, 8);
251 bufferlist m;
252 m.substr_of(bl, 8, 8);
253 bufferlist e;
254 e.substr_of(bl, 16, 8);
255 bufferlist n;
256 n.append("12345678");
257
258 for (unsigned csum_type = Checksummer::CSUM_NONE + 1;
259 csum_type < Checksummer::CSUM_MAX;
260 ++csum_type) {
261 cout << "csum_type " << Checksummer::get_csum_type_string(csum_type)
262 << std::endl;
263
264 bluestore_blob_t b;
265 int bad_off;
266 uint64_t bad_csum;
267 ASSERT_EQ(0, b.verify_csum(0, bl, &bad_off, &bad_csum));
268 ASSERT_EQ(-1, bad_off);
269
270 b.init_csum(csum_type, 3, 24);
271 cout << " value size " << b.get_csum_value_size() << std::endl;
272 b.calc_csum(0, bl);
273 ASSERT_EQ(0, b.verify_csum(0, bl, &bad_off, &bad_csum));
274 ASSERT_EQ(-1, bad_off);
275 ASSERT_EQ(-1, b.verify_csum(0, bl2, &bad_off, &bad_csum));
276 ASSERT_EQ(0, bad_off);
277
278 ASSERT_EQ(0, b.verify_csum(0, f, &bad_off, &bad_csum));
279 ASSERT_EQ(-1, bad_off);
280 ASSERT_EQ(-1, b.verify_csum(8, f, &bad_off, &bad_csum));
281 ASSERT_EQ(8, bad_off);
282 ASSERT_EQ(-1, b.verify_csum(16, f, &bad_off, &bad_csum));
283 ASSERT_EQ(16, bad_off);
284
285 ASSERT_EQ(-1, b.verify_csum(0, m, &bad_off, &bad_csum));
286 ASSERT_EQ(0, bad_off);
287 ASSERT_EQ(0, b.verify_csum(8, m, &bad_off, &bad_csum));
288 ASSERT_EQ(-1, bad_off);
289 ASSERT_EQ(-1, b.verify_csum(16, m, &bad_off, &bad_csum));
290 ASSERT_EQ(16, bad_off);
291
292 ASSERT_EQ(-1, b.verify_csum(0, e, &bad_off, &bad_csum));
293 ASSERT_EQ(0, bad_off);
294 ASSERT_EQ(-1, b.verify_csum(8, e, &bad_off, &bad_csum));
295 ASSERT_EQ(8, bad_off);
296 ASSERT_EQ(0, b.verify_csum(16, e, &bad_off, &bad_csum));
297 ASSERT_EQ(-1, bad_off);
298
299 b.calc_csum(8, n);
300 ASSERT_EQ(0, b.verify_csum(0, f, &bad_off, &bad_csum));
301 ASSERT_EQ(-1, bad_off);
302 ASSERT_EQ(0, b.verify_csum(8, n, &bad_off, &bad_csum));
303 ASSERT_EQ(-1, bad_off);
304 ASSERT_EQ(0, b.verify_csum(16, e, &bad_off, &bad_csum));
305 ASSERT_EQ(-1, bad_off);
306 ASSERT_EQ(-1, b.verify_csum(0, bl, &bad_off, &bad_csum));
307 ASSERT_EQ(8, bad_off);
308 }
309 }
310
311 TEST(bluestore_blob_t, csum_bench)
312 {
313 bufferlist bl;
314 bufferptr bp(10485760);
315 for (char *a = bp.c_str(); a < bp.c_str() + bp.length(); ++a)
316 *a = (unsigned long)a & 0xff;
317 bl.append(bp);
318 int count = 256;
319 for (unsigned csum_type = 1;
320 csum_type < Checksummer::CSUM_MAX;
321 ++csum_type) {
322 bluestore_blob_t b;
323 b.init_csum(csum_type, 12, bl.length());
324 ceph::mono_clock::time_point start = ceph::mono_clock::now();
325 for (int i = 0; i<count; ++i) {
326 b.calc_csum(0, bl);
327 }
328 ceph::mono_clock::time_point end = ceph::mono_clock::now();
329 auto dur = std::chrono::duration_cast<ceph::timespan>(end - start);
330 double mbsec = (double)count * (double)bl.length() / 1000000.0 / (double)dur.count() * 1000000000.0;
331 cout << "csum_type " << Checksummer::get_csum_type_string(csum_type)
332 << ", " << dur << " seconds, "
333 << mbsec << " MB/sec" << std::endl;
334 }
335 }
336
337 TEST(Blob, put_ref)
338 {
339 {
340 BlueStore store(g_ceph_context, "", 4096);
341 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
342 g_ceph_context, "lru", NULL);
343 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
344 g_ceph_context, "lru", NULL);
345
346 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
347 BlueStore::Blob b;
348 b.shared_blob = new BlueStore::SharedBlob(coll.get());
349 b.dirty_blob().allocated_test(bluestore_pextent_t(0x40715000, 0x2000));
350 b.dirty_blob().allocated_test(
351 bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x8000));
352 b.dirty_blob().allocated_test(bluestore_pextent_t(0x4071f000, 0x5000));
353 b.get_ref(coll.get(), 0, 0x1200);
354 b.get_ref(coll.get(), 0xae00, 0x4200);
355 ASSERT_EQ(0x5400u, b.get_referenced_bytes());
356 cout << b << std::endl;
357 PExtentVector r;
358
359 ASSERT_FALSE(b.put_ref(coll.get(), 0, 0x1200, &r));
360 ASSERT_EQ(0x4200u, b.get_referenced_bytes());
361 cout << " r " << r << std::endl;
362 cout << b << std::endl;
363
364 r.clear();
365 ASSERT_TRUE(b.put_ref(coll.get(), 0xae00, 0x4200, &r));
366 ASSERT_EQ(0u, b.get_referenced_bytes());
367 cout << " r " << r << std::endl;
368 cout << b << std::endl;
369 }
370
371 unsigned mas = 4096;
372 BlueStore store(g_ceph_context, "", 8192);
373 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
374 g_ceph_context, "lru", NULL);
375 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
376 g_ceph_context, "lru", NULL);
377 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
378
379 {
380 BlueStore::Blob B;
381 B.shared_blob = new BlueStore::SharedBlob(coll.get());
382 bluestore_blob_t& b = B.dirty_blob();
383 PExtentVector r;
384 b.allocated_test(bluestore_pextent_t(0, mas * 2));
385 B.get_ref(coll.get(), 0, mas*2);
386 ASSERT_EQ(mas * 2, B.get_referenced_bytes());
387 ASSERT_TRUE(b.is_allocated(0, mas*2));
388 ASSERT_TRUE(B.put_ref(coll.get(), 0, mas*2, &r));
389 ASSERT_EQ(0u, B.get_referenced_bytes());
390 cout << "r " << r << " " << b << std::endl;
391 ASSERT_EQ(1u, r.size());
392 ASSERT_EQ(0u, r[0].offset);
393 ASSERT_EQ(mas*2, r[0].length);
394 ASSERT_FALSE(b.is_allocated(0, mas*2));
395 ASSERT_FALSE(b.is_allocated(0, mas));
396 ASSERT_FALSE(b.is_allocated(mas, 0));
397 ASSERT_FALSE(b.get_extents()[0].is_valid());
398 ASSERT_EQ(mas*2, b.get_extents()[0].length);
399 }
400 {
401 BlueStore::Blob B;
402 B.shared_blob = new BlueStore::SharedBlob(coll.get());
403 bluestore_blob_t& b = B.dirty_blob();
404 PExtentVector r;
405 b.allocated_test(bluestore_pextent_t(123, mas * 2));
406 B.get_ref(coll.get(), 0, mas*2);
407 ASSERT_EQ(mas * 2, B.get_referenced_bytes());
408 ASSERT_FALSE(B.put_ref(coll.get(), 0, mas, &r));
409 ASSERT_EQ(mas, B.get_referenced_bytes());
410 cout << "r " << r << " " << b << std::endl;
411 ASSERT_EQ(0u, r.size());
412 ASSERT_TRUE(b.is_allocated(0, mas*2));
413 ASSERT_TRUE(B.put_ref(coll.get(), mas, mas, &r));
414 ASSERT_EQ(0u, B.get_referenced_bytes());
415 ASSERT_EQ(0u, B.get_referenced_bytes());
416 cout << "r " << r << " " << b << std::endl;
417 ASSERT_EQ(1u, r.size());
418 ASSERT_EQ(123u, r[0].offset);
419 ASSERT_EQ(mas*2, r[0].length);
420 ASSERT_FALSE(b.is_allocated(0, mas*2));
421 ASSERT_FALSE(b.get_extents()[0].is_valid());
422 ASSERT_EQ(mas*2, b.get_extents()[0].length);
423 }
424 {
425 BlueStore::Blob B;
426 B.shared_blob = new BlueStore::SharedBlob(coll.get());
427 bluestore_blob_t& b = B.dirty_blob();
428 PExtentVector r;
429 b.allocated_test(bluestore_pextent_t(1, mas));
430 b.allocated_test(bluestore_pextent_t(2, mas));
431 b.allocated_test(bluestore_pextent_t(3, mas));
432 b.allocated_test(bluestore_pextent_t(4, mas));
433 B.get_ref(coll.get(), 0, mas*4);
434 ASSERT_EQ(mas * 4, B.get_referenced_bytes());
435 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
436 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
437 cout << "r " << r << " " << b << std::endl;
438 ASSERT_EQ(0u, r.size());
439 ASSERT_TRUE(b.is_allocated(0, mas*4));
440 ASSERT_TRUE(b.is_allocated(mas, mas));
441 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas, &r));
442 ASSERT_EQ(mas * 2, B.get_referenced_bytes());
443 cout << "r " << r << " " << b << std::endl;
444 ASSERT_EQ(0u, r.size());
445 ASSERT_TRUE(b.is_allocated(mas*2, mas));
446 ASSERT_TRUE(b.is_allocated(0, mas*4));
447 ASSERT_FALSE(B.put_ref(coll.get(), mas*3, mas, &r));
448 ASSERT_EQ(mas, B.get_referenced_bytes());
449 cout << "r " << r << " " << b << std::endl;
450 ASSERT_EQ(2u, r.size());
451 ASSERT_EQ(3u, r[0].offset);
452 ASSERT_EQ(mas, r[0].length);
453 ASSERT_EQ(4u, r[1].offset);
454 ASSERT_EQ(mas, r[1].length);
455 ASSERT_TRUE(b.is_allocated(0, mas*2));
456 ASSERT_FALSE(b.is_allocated(mas*2, mas*2));
457 ASSERT_TRUE(b.get_extents()[0].is_valid());
458 ASSERT_TRUE(b.get_extents()[1].is_valid());
459 ASSERT_FALSE(b.get_extents()[2].is_valid());
460 ASSERT_EQ(3u, b.get_extents().size());
461 }
462 {
463 BlueStore::Blob B;
464 B.shared_blob = new BlueStore::SharedBlob(coll.get());
465 bluestore_blob_t& b = B.dirty_blob();
466 PExtentVector r;
467 b.allocated_test(bluestore_pextent_t(1, mas));
468 b.allocated_test(bluestore_pextent_t(2, mas));
469 b.allocated_test(bluestore_pextent_t(3, mas));
470 b.allocated_test(bluestore_pextent_t(4, mas));
471 b.allocated_test(bluestore_pextent_t(5, mas));
472 b.allocated_test(bluestore_pextent_t(6, mas));
473 B.get_ref(coll.get(), 0, mas*6);
474 ASSERT_EQ(mas * 6, B.get_referenced_bytes());
475 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
476 ASSERT_EQ(mas * 5, B.get_referenced_bytes());
477 cout << "r " << r << " " << b << std::endl;
478 ASSERT_EQ(0u, r.size());
479 ASSERT_TRUE(b.is_allocated(0, mas*6));
480 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas, &r));
481 ASSERT_EQ(mas * 4, B.get_referenced_bytes());
482 cout << "r " << r << " " << b << std::endl;
483 ASSERT_EQ(0u, r.size());
484 ASSERT_TRUE(b.is_allocated(0, mas*6));
485 ASSERT_FALSE(B.put_ref(coll.get(), mas*3, mas, &r));
486 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
487 cout << "r " << r << " " << b << std::endl;
488 ASSERT_EQ(2u, r.size());
489 ASSERT_EQ(3u, r[0].offset);
490 ASSERT_EQ(mas, r[0].length);
491 ASSERT_EQ(4u, r[1].offset);
492 ASSERT_EQ(mas, r[1].length);
493 ASSERT_TRUE(b.is_allocated(0, mas*2));
494 ASSERT_FALSE(b.is_allocated(mas*2, mas*2));
495 ASSERT_TRUE(b.is_allocated(mas*4, mas*2));
496 ASSERT_EQ(5u, b.get_extents().size());
497 ASSERT_TRUE(b.get_extents()[0].is_valid());
498 ASSERT_TRUE(b.get_extents()[1].is_valid());
499 ASSERT_FALSE(b.get_extents()[2].is_valid());
500 ASSERT_TRUE(b.get_extents()[3].is_valid());
501 ASSERT_TRUE(b.get_extents()[4].is_valid());
502 }
503 {
504 BlueStore::Blob B;
505 B.shared_blob = new BlueStore::SharedBlob(coll.get());
506 bluestore_blob_t& b = B.dirty_blob();
507 PExtentVector r;
508 b.allocated_test(bluestore_pextent_t(1, mas * 6));
509 B.get_ref(coll.get(), 0, mas*6);
510 ASSERT_EQ(mas * 6, B.get_referenced_bytes());
511 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
512 ASSERT_EQ(mas * 5, B.get_referenced_bytes());
513 cout << "r " << r << " " << b << std::endl;
514 ASSERT_EQ(0u, r.size());
515 ASSERT_TRUE(b.is_allocated(0, mas*6));
516 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas, &r));
517 ASSERT_EQ(mas * 4, B.get_referenced_bytes());
518 cout << "r " << r << " " << b << std::endl;
519 ASSERT_EQ(0u, r.size());
520 ASSERT_TRUE(b.is_allocated(0, mas*6));
521 ASSERT_FALSE(B.put_ref(coll.get(), mas*3, mas, &r));
522 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
523 cout << "r " << r << " " << b << std::endl;
524 ASSERT_EQ(1u, r.size());
525 ASSERT_EQ(0x2001u, r[0].offset);
526 ASSERT_EQ(mas*2, r[0].length);
527 ASSERT_TRUE(b.is_allocated(0, mas*2));
528 ASSERT_FALSE(b.is_allocated(mas*2, mas*2));
529 ASSERT_TRUE(b.is_allocated(mas*4, mas*2));
530 ASSERT_EQ(3u, b.get_extents().size());
531 ASSERT_TRUE(b.get_extents()[0].is_valid());
532 ASSERT_FALSE(b.get_extents()[1].is_valid());
533 ASSERT_TRUE(b.get_extents()[2].is_valid());
534 }
535 {
536 BlueStore::Blob B;
537 B.shared_blob = new BlueStore::SharedBlob(coll.get());
538 bluestore_blob_t& b = B.dirty_blob();
539 PExtentVector r;
540 b.allocated_test(bluestore_pextent_t(1, mas * 4));
541 b.allocated_test(bluestore_pextent_t(2, mas * 4));
542 b.allocated_test(bluestore_pextent_t(3, mas * 4));
543 B.get_ref(coll.get(), 0, mas*12);
544 ASSERT_EQ(mas * 12, B.get_referenced_bytes());
545 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
546 ASSERT_EQ(mas * 11, B.get_referenced_bytes());
547 cout << "r " << r << " " << b << std::endl;
548 ASSERT_EQ(0u, r.size());
549 ASSERT_TRUE(b.is_allocated(0, mas*12));
550 ASSERT_FALSE(B.put_ref(coll.get(), mas*9, mas, &r));
551 ASSERT_EQ(mas * 10, B.get_referenced_bytes());
552 cout << "r " << r << " " << b << std::endl;
553 ASSERT_EQ(0u, r.size());
554 ASSERT_TRUE(b.is_allocated(0, mas*12));
555 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas*7, &r));
556 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
557 cout << "r " << r << " " << b << std::endl;
558 ASSERT_EQ(3u, r.size());
559 ASSERT_EQ(0x2001u, r[0].offset);
560 ASSERT_EQ(mas*2, r[0].length);
561 ASSERT_EQ(0x2u, r[1].offset);
562 ASSERT_EQ(mas*4, r[1].length);
563 ASSERT_EQ(0x3u, r[2].offset);
564 ASSERT_EQ(mas*2, r[2].length);
565 ASSERT_TRUE(b.is_allocated(0, mas*2));
566 ASSERT_FALSE(b.is_allocated(mas*2, mas*8));
567 ASSERT_TRUE(b.is_allocated(mas*10, mas*2));
568 ASSERT_EQ(3u, b.get_extents().size());
569 ASSERT_TRUE(b.get_extents()[0].is_valid());
570 ASSERT_FALSE(b.get_extents()[1].is_valid());
571 ASSERT_TRUE(b.get_extents()[2].is_valid());
572 }
573 {
574 BlueStore::Blob B;
575 B.shared_blob = new BlueStore::SharedBlob(coll.get());
576 bluestore_blob_t& b = B.dirty_blob();
577 PExtentVector r;
578 b.allocated_test(bluestore_pextent_t(1, mas * 4));
579 b.allocated_test(bluestore_pextent_t(2, mas * 4));
580 b.allocated_test(bluestore_pextent_t(3, mas * 4));
581 B.get_ref(coll.get(), 0, mas*12);
582 ASSERT_EQ(mas * 12, B.get_referenced_bytes());
583 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
584 ASSERT_EQ(mas * 11, B.get_referenced_bytes());
585 cout << "r " << r << " " << b << std::endl;
586 ASSERT_EQ(0u, r.size());
587 ASSERT_TRUE(b.is_allocated(0, mas*12));
588 ASSERT_FALSE(B.put_ref(coll.get(), mas*9, mas, &r));
589 ASSERT_EQ(mas * 10, B.get_referenced_bytes());
590 cout << "r " << r << " " << b << std::endl;
591 ASSERT_EQ(0u, r.size());
592 ASSERT_TRUE(b.is_allocated(0, mas*12));
593 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas*7, &r));
594 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
595 cout << "r " << r << " " << b << std::endl;
596 ASSERT_EQ(3u, r.size());
597 ASSERT_EQ(0x2001u, r[0].offset);
598 ASSERT_EQ(mas*2, r[0].length);
599 ASSERT_EQ(0x2u, r[1].offset);
600 ASSERT_EQ(mas*4, r[1].length);
601 ASSERT_EQ(0x3u, r[2].offset);
602 ASSERT_EQ(mas*2, r[2].length);
603 ASSERT_TRUE(b.is_allocated(0, mas*2));
604 ASSERT_FALSE(b.is_allocated(mas*2, mas*8));
605 ASSERT_TRUE(b.is_allocated(mas*10, mas*2));
606 ASSERT_EQ(3u, b.get_extents().size());
607 ASSERT_TRUE(b.get_extents()[0].is_valid());
608 ASSERT_FALSE(b.get_extents()[1].is_valid());
609 ASSERT_TRUE(b.get_extents()[2].is_valid());
610 ASSERT_FALSE(B.put_ref(coll.get(), 0, mas, &r));
611 ASSERT_EQ(mas * 2, B.get_referenced_bytes());
612 cout << "r " << r << " " << b << std::endl;
613 ASSERT_EQ(1u, r.size());
614 ASSERT_EQ(0x1u, r[0].offset);
615 ASSERT_EQ(mas*2, r[0].length);
616 ASSERT_EQ(2u, b.get_extents().size());
617 ASSERT_FALSE(b.get_extents()[0].is_valid());
618 ASSERT_TRUE(b.get_extents()[1].is_valid());
619 ASSERT_TRUE(B.put_ref(coll.get(), mas*10, mas*2, &r));
620 ASSERT_EQ(mas * 0, B.get_referenced_bytes());
621 cout << "r " << r << " " << b << std::endl;
622 ASSERT_EQ(1u, r.size());
623 ASSERT_EQ(0x2003u, r[0].offset);
624 ASSERT_EQ(mas*2, r[0].length);
625 ASSERT_EQ(1u, b.get_extents().size());
626 ASSERT_FALSE(b.get_extents()[0].is_valid());
627 }
628 {
629 BlueStore::Blob B;
630 B.shared_blob = new BlueStore::SharedBlob(coll.get());
631 bluestore_blob_t& b = B.dirty_blob();
632 PExtentVector r;
633 b.allocated_test(bluestore_pextent_t(1, mas * 4));
634 b.allocated_test(bluestore_pextent_t(2, mas * 4));
635 b.allocated_test(bluestore_pextent_t(3, mas * 4));
636 B.get_ref(coll.get(), 0, mas*12);
637 ASSERT_EQ(mas * 12, B.get_referenced_bytes());
638 ASSERT_FALSE(B.put_ref(coll.get(), mas, mas, &r));
639 ASSERT_EQ(mas * 11, B.get_referenced_bytes());
640 cout << "r " << r << " " << b << std::endl;
641 ASSERT_EQ(0u, r.size());
642 ASSERT_TRUE(b.is_allocated(0, mas*12));
643 ASSERT_FALSE(B.put_ref(coll.get(), mas*9, mas, &r));
644 ASSERT_EQ(mas * 10, B.get_referenced_bytes());
645 cout << "r " << r << " " << b << std::endl;
646 ASSERT_EQ(0u, r.size());
647 ASSERT_TRUE(b.is_allocated(0, mas*12));
648 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas*7, &r));
649 ASSERT_EQ(mas * 3, B.get_referenced_bytes());
650 cout << "r " << r << " " << b << std::endl;
651 ASSERT_EQ(3u, r.size());
652 ASSERT_EQ(0x2001u, r[0].offset);
653 ASSERT_EQ(mas*2, r[0].length);
654 ASSERT_EQ(0x2u, r[1].offset);
655 ASSERT_EQ(mas*4, r[1].length);
656 ASSERT_EQ(0x3u, r[2].offset);
657 ASSERT_EQ(mas*2, r[2].length);
658 ASSERT_TRUE(b.is_allocated(0, mas*2));
659 ASSERT_FALSE(b.is_allocated(mas*2, mas*8));
660 ASSERT_TRUE(b.is_allocated(mas*10, mas*2));
661 ASSERT_EQ(3u, b.get_extents().size());
662 ASSERT_TRUE(b.get_extents()[0].is_valid());
663 ASSERT_FALSE(b.get_extents()[1].is_valid());
664 ASSERT_TRUE(b.get_extents()[2].is_valid());
665 ASSERT_FALSE(B.put_ref(coll.get(), mas*10, mas*2, &r));
666 ASSERT_EQ(mas * 1, B.get_referenced_bytes());
667 cout << "r " << r << " " << b << std::endl;
668 ASSERT_EQ(1u, r.size());
669 ASSERT_EQ(0x2003u, r[0].offset);
670 ASSERT_EQ(mas*2, r[0].length);
671 ASSERT_EQ(2u, b.get_extents().size());
672 ASSERT_TRUE(b.get_extents()[0].is_valid());
673 ASSERT_FALSE(b.get_extents()[1].is_valid());
674 ASSERT_TRUE(B.put_ref(coll.get(), 0, mas, &r));
675 ASSERT_EQ(mas * 0, B.get_referenced_bytes());
676 cout << "r " << r << " " << b << std::endl;
677 ASSERT_EQ(1u, r.size());
678 ASSERT_EQ(0x1u, r[0].offset);
679 ASSERT_EQ(mas*2, r[0].length);
680 ASSERT_EQ(1u, b.get_extents().size());
681 ASSERT_FALSE(b.get_extents()[0].is_valid());
682 }
683 {
684 BlueStore::Blob B;
685 B.shared_blob = new BlueStore::SharedBlob(coll.get());
686 bluestore_blob_t& b = B.dirty_blob();
687 PExtentVector r;
688 b.allocated_test(bluestore_pextent_t(1, mas * 8));
689 B.get_ref(coll.get(), 0, mas*8);
690 ASSERT_EQ(mas * 8, B.get_referenced_bytes());
691 ASSERT_FALSE(B.put_ref(coll.get(), 0, mas, &r));
692 ASSERT_EQ(mas * 7, B.get_referenced_bytes());
693 cout << "r " << r << " " << b << std::endl;
694 ASSERT_EQ(0u, r.size());
695 ASSERT_TRUE(b.is_allocated(0, mas*8));
696 ASSERT_FALSE(B.put_ref(coll.get(), mas*7, mas, &r));
697 ASSERT_EQ(mas * 6, B.get_referenced_bytes());
698 cout << "r " << r << " " << b << std::endl;
699 ASSERT_EQ(0u, r.size());
700 ASSERT_TRUE(b.is_allocated(0, mas*8));
701 ASSERT_FALSE(B.put_ref(coll.get(), mas*2, mas, &r));
702 ASSERT_EQ(mas * 5, B.get_referenced_bytes());
703 cout << "r " << r << " " << b << std::endl;
704 ASSERT_EQ(0u, r.size());
705 ASSERT_TRUE(b.is_allocated(0, 8));
706 ASSERT_FALSE(B.put_ref(coll.get(), mas*3, mas*4, &r));
707 ASSERT_EQ(mas * 1, B.get_referenced_bytes());
708 ASSERT_EQ(1u, r.size());
709 ASSERT_EQ(0x2001u, r[0].offset);
710 ASSERT_EQ(mas*6, r[0].length);
711 ASSERT_TRUE(b.is_allocated(0, mas*2));
712 ASSERT_FALSE(b.is_allocated(mas*2, mas*6));
713 ASSERT_EQ(2u, b.get_extents().size());
714 ASSERT_TRUE(b.get_extents()[0].is_valid());
715 ASSERT_FALSE(b.get_extents()[1].is_valid());
716 ASSERT_TRUE(B.put_ref(coll.get(), mas, mas, &r));
717 ASSERT_EQ(mas * 0, B.get_referenced_bytes());
718 cout << "r " << r << " " << b << std::endl;
719 ASSERT_EQ(1u, r.size());
720 ASSERT_EQ(0x1u, r[0].offset);
721 ASSERT_EQ(mas*2, r[0].length);
722 ASSERT_EQ(1u, b.get_extents().size());
723 ASSERT_FALSE(b.get_extents()[0].is_valid());
724 }
725 // verify csum chunk size if factored in properly
726 {
727 BlueStore::Blob B;
728 B.shared_blob = new BlueStore::SharedBlob(coll.get());
729 bluestore_blob_t& b = B.dirty_blob();
730 PExtentVector r;
731 b.allocated_test(bluestore_pextent_t(0, mas*4));
732 b.init_csum(Checksummer::CSUM_CRC32C, 14, mas * 4);
733 B.get_ref(coll.get(), 0, mas*4);
734 ASSERT_EQ(mas * 4, B.get_referenced_bytes());
735 ASSERT_TRUE(b.is_allocated(0, mas*4));
736 ASSERT_FALSE(B.put_ref(coll.get(), 0, mas*3, &r));
737 ASSERT_EQ(mas * 1, B.get_referenced_bytes());
738 cout << "r " << r << " " << b << std::endl;
739 ASSERT_EQ(0u, r.size());
740 ASSERT_TRUE(b.is_allocated(0, mas*4));
741 ASSERT_TRUE(b.get_extents()[0].is_valid());
742 ASSERT_EQ(mas*4, b.get_extents()[0].length);
743 }
744 {
745 BlueStore::Blob B;
746 B.shared_blob = new BlueStore::SharedBlob(coll.get());
747 bluestore_blob_t& b = B.dirty_blob();
748 b.allocated_test(bluestore_pextent_t(0x40101000, 0x4000));
749 b.allocated_test(bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET,
750 0x13000));
751
752 b.allocated_test(bluestore_pextent_t(0x40118000, 0x7000));
753 B.get_ref(coll.get(), 0x0, 0x3800);
754 B.get_ref(coll.get(), 0x17c00, 0x6400);
755 ASSERT_EQ(0x3800u + 0x6400u, B.get_referenced_bytes());
756 b.set_flag(bluestore_blob_t::FLAG_SHARED);
757 b.init_csum(Checksummer::CSUM_CRC32C, 12, 0x1e000);
758
759 cout << "before: " << B << std::endl;
760 PExtentVector r;
761 ASSERT_FALSE(B.put_ref(coll.get(), 0x1800, 0x2000, &r));
762 ASSERT_EQ(0x3800u + 0x6400u - 0x2000u, B.get_referenced_bytes());
763 cout << "after: " << B << std::endl;
764 cout << "r " << r << std::endl;
765 }
766 {
767 BlueStore::Blob B;
768 B.shared_blob = new BlueStore::SharedBlob(coll.get());
769 bluestore_blob_t& b = B.dirty_blob();
770 b.allocated_test(bluestore_pextent_t(1, 0x5000));
771 b.allocated_test(bluestore_pextent_t(2, 0x5000));
772 B.get_ref(coll.get(), 0x0, 0xa000);
773 ASSERT_EQ(0xa000u, B.get_referenced_bytes());
774 cout << "before: " << B << std::endl;
775 PExtentVector r;
776 ASSERT_FALSE(B.put_ref(coll.get(), 0x8000, 0x2000, &r));
777 cout << "after: " << B << std::endl;
778 cout << "r " << r << std::endl;
779 ASSERT_EQ(0x8000u, B.get_referenced_bytes());
780 ASSERT_EQ(1u, r.size());
781 ASSERT_EQ(0x3002u, r[0].offset);
782 ASSERT_EQ(0x2000u, r[0].length);
783 }
784 {
785 BlueStore::Blob B;
786 B.shared_blob = new BlueStore::SharedBlob(coll.get());
787 bluestore_blob_t& b = B.dirty_blob();
788 b.allocated_test(bluestore_pextent_t(1, 0x7000));
789 b.allocated_test(bluestore_pextent_t(2, 0x7000));
790 B.get_ref(coll.get(), 0x0, 0xe000);
791 ASSERT_EQ(0xe000u, B.get_referenced_bytes());
792 cout << "before: " << B << std::endl;
793 PExtentVector r;
794 ASSERT_FALSE(B.put_ref(coll.get(), 0, 0xb000, &r));
795 ASSERT_EQ(0x3000u, B.get_referenced_bytes());
796 cout << "after: " << B << std::endl;
797 cout << "r " << r << std::endl;
798 ASSERT_EQ(0x3000u, B.get_referenced_bytes());
799 ASSERT_EQ(2u, r.size());
800 ASSERT_EQ(1u, r[0].offset);
801 ASSERT_EQ(0x7000u, r[0].length);
802 ASSERT_EQ(2u, r[1].offset);
803 ASSERT_EQ(0x3000u, r[1].length); // we have 0x1000 bytes less due to
804 // alignment caused by min_alloc_size = 0x2000
805 }
806 {
807 BlueStore store(g_ceph_context, "", 0x4000);
808 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
809 g_ceph_context, "lru", NULL);
810 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
811 g_ceph_context, "lru", NULL);
812
813 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
814 BlueStore::Blob B;
815 B.shared_blob = new BlueStore::SharedBlob(coll.get());
816 bluestore_blob_t& b = B.dirty_blob();
817 b.allocated_test(bluestore_pextent_t(1, 0x5000));
818 b.allocated_test(bluestore_pextent_t(2, 0x7000));
819 B.get_ref(coll.get(), 0x0, 0xc000);
820 ASSERT_EQ(0xc000u, B.get_referenced_bytes());
821 cout << "before: " << B << std::endl;
822 PExtentVector r;
823 ASSERT_FALSE(B.put_ref(coll.get(), 0x2000, 0xa000, &r));
824 cout << "after: " << B << std::endl;
825 cout << "r " << r << std::endl;
826 ASSERT_EQ(0x2000u, B.get_referenced_bytes());
827 ASSERT_EQ(2u, r.size());
828 ASSERT_EQ(0x4001u, r[0].offset);
829 ASSERT_EQ(0x1000u, r[0].length);
830 ASSERT_EQ(2u, r[1].offset);
831 ASSERT_EQ(0x7000u, r[1].length);
832 ASSERT_EQ(1u, b.get_extents()[0].offset);
833 ASSERT_EQ(0x4000u, b.get_extents()[0].length);
834 }
835 }
836
837 TEST(bluestore_blob_t, can_split)
838 {
839 bluestore_blob_t a;
840 ASSERT_TRUE(a.can_split());
841 a.flags = bluestore_blob_t::FLAG_SHARED;
842 ASSERT_FALSE(a.can_split());
843 a.flags = bluestore_blob_t::FLAG_COMPRESSED;
844 ASSERT_FALSE(a.can_split());
845 a.flags = bluestore_blob_t::FLAG_HAS_UNUSED;
846 ASSERT_FALSE(a.can_split());
847 }
848
849 TEST(bluestore_blob_t, can_split_at)
850 {
851 bluestore_blob_t a;
852 a.allocated_test(bluestore_pextent_t(0x10000, 0x2000));
853 a.allocated_test(bluestore_pextent_t(0x20000, 0x2000));
854 ASSERT_TRUE(a.can_split_at(0x1000));
855 ASSERT_TRUE(a.can_split_at(0x1800));
856 a.init_csum(Checksummer::CSUM_CRC32C, 12, 0x4000);
857 ASSERT_TRUE(a.can_split_at(0x1000));
858 ASSERT_TRUE(a.can_split_at(0x2000));
859 ASSERT_TRUE(a.can_split_at(0x3000));
860 ASSERT_FALSE(a.can_split_at(0x2800));
861 }
862
863 TEST(bluestore_blob_t, prune_tail)
864 {
865 bluestore_blob_t a;
866 a.allocated_test(bluestore_pextent_t(0x10000, 0x2000));
867 a.allocated_test(bluestore_pextent_t(0x20000, 0x2000));
868 ASSERT_FALSE(a.can_prune_tail());
869 a.allocated_test(
870 bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x2000));
871 ASSERT_TRUE(a.can_prune_tail());
872 a.prune_tail();
873 ASSERT_FALSE(a.can_prune_tail());
874 ASSERT_EQ(2u, a.get_extents().size());
875 ASSERT_EQ(0x4000u, a.get_logical_length());
876
877 a.allocated_test(
878 bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x2000));
879 a.init_csum(Checksummer::CSUM_CRC32C_8, 12, 0x6000);
880 ASSERT_EQ(6u, a.csum_data.length());
881 ASSERT_TRUE(a.can_prune_tail());
882 a.prune_tail();
883 ASSERT_FALSE(a.can_prune_tail());
884 ASSERT_EQ(2u, a.get_extents().size());
885 ASSERT_EQ(0x4000u, a.get_logical_length());
886 ASSERT_EQ(4u, a.csum_data.length());
887
888 bluestore_blob_t b;
889 b.allocated_test(
890 bluestore_pextent_t(bluestore_pextent_t::INVALID_OFFSET, 0x2000));
891 ASSERT_FALSE(a.can_prune_tail());
892 }
893
894 TEST(Blob, split)
895 {
896 BlueStore store(g_ceph_context, "", 4096);
897 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
898 g_ceph_context, "lru", NULL);
899 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
900 g_ceph_context, "lru", NULL);
901 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
902 {
903 BlueStore::Blob L, R;
904 L.shared_blob = new BlueStore::SharedBlob(coll.get());
905 R.shared_blob = new BlueStore::SharedBlob(coll.get());
906 L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x2000));
907 L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
908 L.get_ref(coll.get(), 0, 0x2000);
909 L.split(coll.get(), 0x1000, &R);
910 ASSERT_EQ(0x1000u, L.get_blob().get_logical_length());
911 ASSERT_EQ(4u, L.get_blob().csum_data.length());
912 ASSERT_EQ(1u, L.get_blob().get_extents().size());
913 ASSERT_EQ(0x2000u, L.get_blob().get_extents().front().offset);
914 ASSERT_EQ(0x1000u, L.get_blob().get_extents().front().length);
915 ASSERT_EQ(0x1000u, L.get_referenced_bytes());
916 ASSERT_EQ(0x1000u, R.get_blob().get_logical_length());
917 ASSERT_EQ(4u, R.get_blob().csum_data.length());
918 ASSERT_EQ(1u, R.get_blob().get_extents().size());
919 ASSERT_EQ(0x3000u, R.get_blob().get_extents().front().offset);
920 ASSERT_EQ(0x1000u, R.get_blob().get_extents().front().length);
921 ASSERT_EQ(0x1000u, R.get_referenced_bytes());
922 }
923 {
924 BlueStore::Blob L, R;
925 L.shared_blob = new BlueStore::SharedBlob(coll.get());
926 R.shared_blob = new BlueStore::SharedBlob(coll.get());
927 L.dirty_blob().allocated_test(bluestore_pextent_t(0x2000, 0x1000));
928 L.dirty_blob().allocated_test(bluestore_pextent_t(0x12000, 0x1000));
929 L.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
930 L.get_ref(coll.get(), 0, 0x1000);
931 L.get_ref(coll.get(), 0x1000, 0x1000);
932 L.split(coll.get(), 0x1000, &R);
933 ASSERT_EQ(0x1000u, L.get_blob().get_logical_length());
934 ASSERT_EQ(4u, L.get_blob().csum_data.length());
935 ASSERT_EQ(1u, L.get_blob().get_extents().size());
936 ASSERT_EQ(0x2000u, L.get_blob().get_extents().front().offset);
937 ASSERT_EQ(0x1000u, L.get_blob().get_extents().front().length);
938 ASSERT_EQ(0x1000u, L.get_referenced_bytes());
939 ASSERT_EQ(0x1000u, R.get_blob().get_logical_length());
940 ASSERT_EQ(4u, R.get_blob().csum_data.length());
941 ASSERT_EQ(1u, R.get_blob().get_extents().size());
942 ASSERT_EQ(0x12000u, R.get_blob().get_extents().front().offset);
943 ASSERT_EQ(0x1000u, R.get_blob().get_extents().front().length);
944 ASSERT_EQ(0x1000u, R.get_referenced_bytes());
945 }
946 }
947
948 TEST(Blob, legacy_decode)
949 {
950 BlueStore store(g_ceph_context, "", 4096);
951 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
952 g_ceph_context, "lru", NULL);
953 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
954 g_ceph_context, "lru", NULL);
955 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
956 bufferlist bl, bl2;
957 {
958 BlueStore::Blob B;
959
960 B.shared_blob = new BlueStore::SharedBlob(coll.get());
961 B.dirty_blob().allocated_test(bluestore_pextent_t(0x1, 0x2000));
962 B.dirty_blob().init_csum(Checksummer::CSUM_CRC32C, 12, 0x2000);
963 B.get_ref(coll.get(), 0, 0xff0);
964 B.get_ref(coll.get(), 0x1fff, 1);
965
966 bluestore_extent_ref_map_t fake_ref_map;
967 fake_ref_map.get(0, 0xff0);
968 fake_ref_map.get(0x1fff, 1);
969
970 size_t bound = 0, bound2 = 0;
971
972 B.bound_encode(
973 bound,
974 1, /*struct_v*/
975 0, /*sbid*/
976 false);
977 fake_ref_map.bound_encode(bound);
978
979 B.bound_encode(
980 bound2,
981 2, /*struct_v*/
982 0, /*sbid*/
983 true);
984
985 {
986 auto app = bl.get_contiguous_appender(bound);
987 auto app2 = bl2.get_contiguous_appender(bound2);
988 B.encode(
989 app,
990 1, /*struct_v*/
991 0, /*sbid*/
992 false);
993 fake_ref_map.encode(app);
994
995 B.encode(
996 app2,
997 2, /*struct_v*/
998 0, /*sbid*/
999 true);
1000 }
1001
1002 auto p = bl.front().begin_deep();
1003 auto p2 = bl2.front().begin_deep();
1004 BlueStore::Blob Bres, Bres2;
1005 Bres.shared_blob = new BlueStore::SharedBlob(coll.get());
1006 Bres2.shared_blob = new BlueStore::SharedBlob(coll.get());
1007
1008 uint64_t sbid, sbid2;
1009 Bres.decode(
1010 coll.get(),
1011 p,
1012 1, /*struct_v*/
1013 &sbid,
1014 true);
1015 Bres2.decode(
1016 coll.get(),
1017 p2,
1018 2, /*struct_v*/
1019 &sbid2,
1020 true);
1021
1022 ASSERT_EQ(0xff0u + 1u, Bres.get_blob_use_tracker().get_referenced_bytes());
1023 ASSERT_EQ(0xff0u + 1u, Bres2.get_blob_use_tracker().get_referenced_bytes());
1024 ASSERT_TRUE(Bres.get_blob_use_tracker().equal(Bres2.get_blob_use_tracker()));
1025 }
1026 }
1027
1028 TEST(ExtentMap, seek_lextent)
1029 {
1030 BlueStore store(g_ceph_context, "", 4096);
1031 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
1032 g_ceph_context, "lru", NULL);
1033 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
1034 g_ceph_context, "lru", NULL);
1035
1036 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1037 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1038 BlueStore::ExtentMap em(&onode);
1039 BlueStore::BlobRef br(new BlueStore::Blob);
1040 br->shared_blob = new BlueStore::SharedBlob(coll.get());
1041
1042 ASSERT_EQ(em.extent_map.end(), em.seek_lextent(0));
1043 ASSERT_EQ(em.extent_map.end(), em.seek_lextent(100));
1044
1045 em.extent_map.insert(*new BlueStore::Extent(100, 0, 100, br));
1046 auto a = em.find(100);
1047 ASSERT_EQ(a, em.seek_lextent(0));
1048 ASSERT_EQ(a, em.seek_lextent(99));
1049 ASSERT_EQ(a, em.seek_lextent(100));
1050 ASSERT_EQ(a, em.seek_lextent(101));
1051 ASSERT_EQ(a, em.seek_lextent(199));
1052 ASSERT_EQ(em.extent_map.end(), em.seek_lextent(200));
1053
1054 em.extent_map.insert(*new BlueStore::Extent(200, 0, 100, br));
1055 auto b = em.find(200);
1056 ASSERT_EQ(a, em.seek_lextent(0));
1057 ASSERT_EQ(a, em.seek_lextent(99));
1058 ASSERT_EQ(a, em.seek_lextent(100));
1059 ASSERT_EQ(a, em.seek_lextent(101));
1060 ASSERT_EQ(a, em.seek_lextent(199));
1061 ASSERT_EQ(b, em.seek_lextent(200));
1062 ASSERT_EQ(b, em.seek_lextent(299));
1063 ASSERT_EQ(em.extent_map.end(), em.seek_lextent(300));
1064
1065 em.extent_map.insert(*new BlueStore::Extent(400, 0, 100, br));
1066 auto d = em.find(400);
1067 ASSERT_EQ(a, em.seek_lextent(0));
1068 ASSERT_EQ(a, em.seek_lextent(99));
1069 ASSERT_EQ(a, em.seek_lextent(100));
1070 ASSERT_EQ(a, em.seek_lextent(101));
1071 ASSERT_EQ(a, em.seek_lextent(199));
1072 ASSERT_EQ(b, em.seek_lextent(200));
1073 ASSERT_EQ(b, em.seek_lextent(299));
1074 ASSERT_EQ(d, em.seek_lextent(300));
1075 ASSERT_EQ(d, em.seek_lextent(399));
1076 ASSERT_EQ(d, em.seek_lextent(400));
1077 ASSERT_EQ(d, em.seek_lextent(499));
1078 ASSERT_EQ(em.extent_map.end(), em.seek_lextent(500));
1079 }
1080
1081 TEST(ExtentMap, has_any_lextents)
1082 {
1083 BlueStore store(g_ceph_context, "", 4096);
1084 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
1085 g_ceph_context, "lru", NULL);
1086 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
1087 g_ceph_context, "lru", NULL);
1088 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1089 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1090 BlueStore::ExtentMap em(&onode);
1091 BlueStore::BlobRef b(new BlueStore::Blob);
1092 b->shared_blob = new BlueStore::SharedBlob(coll.get());
1093
1094 ASSERT_FALSE(em.has_any_lextents(0, 0));
1095 ASSERT_FALSE(em.has_any_lextents(0, 1000));
1096 ASSERT_FALSE(em.has_any_lextents(1000, 1000));
1097
1098 em.extent_map.insert(*new BlueStore::Extent(100, 0, 100, b));
1099 ASSERT_FALSE(em.has_any_lextents(0, 50));
1100 ASSERT_FALSE(em.has_any_lextents(0, 100));
1101 ASSERT_FALSE(em.has_any_lextents(50, 50));
1102 ASSERT_TRUE(em.has_any_lextents(50, 51));
1103 ASSERT_TRUE(em.has_any_lextents(50, 100051));
1104 ASSERT_TRUE(em.has_any_lextents(100, 100));
1105 ASSERT_TRUE(em.has_any_lextents(100, 1));
1106 ASSERT_TRUE(em.has_any_lextents(199, 1));
1107 ASSERT_TRUE(em.has_any_lextents(199, 2));
1108 ASSERT_FALSE(em.has_any_lextents(200, 2));
1109
1110 em.extent_map.insert(*new BlueStore::Extent(200, 0, 100, b));
1111 ASSERT_TRUE(em.has_any_lextents(199, 1));
1112 ASSERT_TRUE(em.has_any_lextents(199, 2));
1113 ASSERT_TRUE(em.has_any_lextents(200, 2));
1114 ASSERT_TRUE(em.has_any_lextents(200, 200));
1115 ASSERT_TRUE(em.has_any_lextents(299, 1));
1116 ASSERT_FALSE(em.has_any_lextents(300, 1));
1117
1118 em.extent_map.insert(*new BlueStore::Extent(400, 0, 100, b));
1119 ASSERT_TRUE(em.has_any_lextents(0, 10000));
1120 ASSERT_TRUE(em.has_any_lextents(199, 1));
1121 ASSERT_FALSE(em.has_any_lextents(300, 1));
1122 ASSERT_FALSE(em.has_any_lextents(300, 100));
1123 ASSERT_FALSE(em.has_any_lextents(399, 1));
1124 ASSERT_TRUE(em.has_any_lextents(400, 1));
1125 ASSERT_TRUE(em.has_any_lextents(400, 100));
1126 ASSERT_TRUE(em.has_any_lextents(400, 1000));
1127 ASSERT_TRUE(em.has_any_lextents(499, 1000));
1128 ASSERT_FALSE(em.has_any_lextents(500, 1000));
1129 }
1130
1131 void erase_and_delete(BlueStore::ExtentMap& em, size_t v)
1132 {
1133 auto d = em.find(v);
1134 ASSERT_NE(d, em.extent_map.end());
1135 em.extent_map.erase(d);
1136 delete &*d;
1137 }
1138
1139 TEST(ExtentMap, compress_extent_map)
1140 {
1141 BlueStore store(g_ceph_context, "", 4096);
1142 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
1143 g_ceph_context, "lru", NULL);
1144 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
1145 g_ceph_context, "lru", NULL);
1146
1147 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1148 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1149 BlueStore::ExtentMap em(&onode);
1150 BlueStore::BlobRef b1(new BlueStore::Blob);
1151 BlueStore::BlobRef b2(new BlueStore::Blob);
1152 BlueStore::BlobRef b3(new BlueStore::Blob);
1153 b1->shared_blob = new BlueStore::SharedBlob(coll.get());
1154 b2->shared_blob = new BlueStore::SharedBlob(coll.get());
1155 b3->shared_blob = new BlueStore::SharedBlob(coll.get());
1156
1157 em.extent_map.insert(*new BlueStore::Extent(0, 0, 100, b1));
1158 em.extent_map.insert(*new BlueStore::Extent(100, 0, 100, b2));
1159 ASSERT_EQ(0, em.compress_extent_map(0, 10000));
1160 ASSERT_EQ(2u, em.extent_map.size());
1161
1162 em.extent_map.insert(*new BlueStore::Extent(200, 100, 100, b2));
1163 em.extent_map.insert(*new BlueStore::Extent(300, 200, 100, b2));
1164 ASSERT_EQ(0, em.compress_extent_map(0, 0));
1165 ASSERT_EQ(0, em.compress_extent_map(100000, 1000));
1166 ASSERT_EQ(2, em.compress_extent_map(0, 100000));
1167 ASSERT_EQ(2u, em.extent_map.size());
1168 erase_and_delete(em, 100);
1169 em.extent_map.insert(*new BlueStore::Extent(100, 0, 100, b2));
1170 em.extent_map.insert(*new BlueStore::Extent(200, 100, 100, b3));
1171 em.extent_map.insert(*new BlueStore::Extent(300, 200, 100, b2));
1172 ASSERT_EQ(0, em.compress_extent_map(0, 1));
1173 ASSERT_EQ(0, em.compress_extent_map(0, 100000));
1174 ASSERT_EQ(4u, em.extent_map.size());
1175
1176 em.extent_map.insert(*new BlueStore::Extent(400, 300, 100, b2));
1177 em.extent_map.insert(*new BlueStore::Extent(500, 500, 100, b2));
1178 em.extent_map.insert(*new BlueStore::Extent(600, 600, 100, b2));
1179 em.extent_map.insert(*new BlueStore::Extent(700, 0, 100, b1));
1180 em.extent_map.insert(*new BlueStore::Extent(800, 0, 100, b3));
1181 ASSERT_EQ(0, em.compress_extent_map(0, 99));
1182 ASSERT_EQ(0, em.compress_extent_map(800, 1000));
1183 ASSERT_EQ(2, em.compress_extent_map(100, 500));
1184 ASSERT_EQ(7u, em.extent_map.size());
1185 erase_and_delete(em, 300);
1186 erase_and_delete(em, 500);
1187 erase_and_delete(em, 700);
1188 em.extent_map.insert(*new BlueStore::Extent(400, 300, 100, b2));
1189 em.extent_map.insert(*new BlueStore::Extent(500, 400, 100, b2));
1190 em.extent_map.insert(*new BlueStore::Extent(700, 500, 100, b2));
1191 ASSERT_EQ(1, em.compress_extent_map(0, 1000));
1192 ASSERT_EQ(6u, em.extent_map.size());
1193 }
1194
1195
1196 void clear_and_dispose(BlueStore::old_extent_map_t& old_em)
1197 {
1198 auto oep = old_em.begin();
1199 while (oep != old_em.end()) {
1200 auto &lo = *oep;
1201 oep = old_em.erase(oep);
1202 delete &lo;
1203 }
1204 }
1205
1206 TEST(GarbageCollector, BasicTest)
1207 {
1208 BlueStore::OnodeCacheShard *oc = BlueStore::OnodeCacheShard::create(
1209 g_ceph_context, "lru", NULL);
1210 BlueStore::BufferCacheShard *bc = BlueStore::BufferCacheShard::create(
1211 g_ceph_context, "lru", NULL);
1212
1213 BlueStore store(g_ceph_context, "", 4096);
1214 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1215 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1216 BlueStore::ExtentMap em(&onode);
1217
1218 BlueStore::old_extent_map_t old_extents;
1219
1220
1221 /*
1222 min_alloc_size = 4096
1223 original disposition
1224 extent1 <loffs = 100, boffs = 100, len = 10>
1225 -> blob1<compressed, len_on_disk=4096, logical_len=8192>
1226 extent2 <loffs = 200, boffs = 200, len = 10>
1227 -> blob2<raw, len_on_disk=4096, llen=4096>
1228 extent3 <loffs = 300, boffs = 300, len = 10>
1229 -> blob1<compressed, len_on_disk=4096, llen=8192>
1230 extent4 <loffs = 4096, boffs = 0, len = 10>
1231 -> blob3<raw, len_on_disk=4096, llen=4096>
1232 on write(300~100) resulted in
1233 extent1 <loffs = 100, boffs = 100, len = 10>
1234 -> blob1<compressed, len_on_disk=4096, logical_len=8192>
1235 extent2 <loffs = 200, boffs = 200, len = 10>
1236 -> blob2<raw, len_on_disk=4096, llen=4096>
1237 extent3 <loffs = 300, boffs = 300, len = 100>
1238 -> blob4<raw, len_on_disk=4096, llen=4096>
1239 extent4 <loffs = 4096, boffs = 0, len = 10>
1240 -> blob3<raw, len_on_disk=4096, llen=4096>
1241 */
1242 {
1243 BlueStore::GarbageCollector gc(g_ceph_context);
1244 int64_t saving;
1245 BlueStore::BlobRef b1(new BlueStore::Blob);
1246 BlueStore::BlobRef b2(new BlueStore::Blob);
1247 BlueStore::BlobRef b3(new BlueStore::Blob);
1248 BlueStore::BlobRef b4(new BlueStore::Blob);
1249 b1->shared_blob = new BlueStore::SharedBlob(coll.get());
1250 b2->shared_blob = new BlueStore::SharedBlob(coll.get());
1251 b3->shared_blob = new BlueStore::SharedBlob(coll.get());
1252 b4->shared_blob = new BlueStore::SharedBlob(coll.get());
1253 b1->dirty_blob().set_compressed(0x2000, 0x1000);
1254 b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x1000));
1255 b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x1000));
1256 b3->dirty_blob().allocated_test(bluestore_pextent_t(2, 0x1000));
1257 b4->dirty_blob().allocated_test(bluestore_pextent_t(3, 0x1000));
1258 em.extent_map.insert(*new BlueStore::Extent(100, 100, 10, b1));
1259 b1->get_ref(coll.get(), 100, 10);
1260 em.extent_map.insert(*new BlueStore::Extent(200, 200, 10, b2));
1261 b2->get_ref(coll.get(), 200, 10);
1262 em.extent_map.insert(*new BlueStore::Extent(300, 300, 100, b4));
1263 b4->get_ref(coll.get(), 300, 100);
1264 em.extent_map.insert(*new BlueStore::Extent(4096, 0, 10, b3));
1265 b3->get_ref(coll.get(), 0, 10);
1266
1267 old_extents.push_back(*new BlueStore::OldExtent(300, 300, 10, b1));
1268
1269 saving = gc.estimate(300, 100, em, old_extents, 4096);
1270 ASSERT_EQ(saving, 1);
1271 auto& to_collect = gc.get_extents_to_collect();
1272 ASSERT_EQ(to_collect.num_intervals(), 1u);
1273 {
1274 auto it = to_collect.begin();
1275 using p = decltype(*it);
1276 auto v = p{100ul, 10ul};
1277 ASSERT_EQ(*it, v);
1278 }
1279 em.clear();
1280 clear_and_dispose(old_extents);
1281 }
1282 /*
1283 original disposition
1284 min_alloc_size = 0x10000
1285 extent1 <loffs = 0, boffs = 0, len = 0x40000>
1286 -> blob1<compressed, len_on_disk=0x20000, logical_len=0x40000>
1287 Write 0x8000~37000 resulted in the following extent map prior to GC
1288 for the last write_small(0x30000~0xf000):
1289
1290 extent1 <loffs = 0, boffs = 0, len = 0x8000>
1291 -> blob1<compressed, len_on_disk=0x20000, logical_len=0x40000>
1292 extent2 <loffs = 0x8000, boffs = 0x8000, len = 0x8000>
1293 -> blob2<raw, len_on_disk=0x10000, llen=0x10000>
1294 extent3 <loffs = 0x10000, boffs = 0, len = 0x20000>
1295 -> blob3<raw, len_on_disk=0x20000, llen=0x20000>
1296 extent4 <loffs = 0x30000, boffs = 0, len = 0xf000>
1297 -> blob4<raw, len_on_disk=0x10000, llen=0x10000>
1298 extent5 <loffs = 0x3f000, boffs = 0x3f000, len = 0x1000>
1299 -> blob1<compressed, len_on_disk=0x20000, llen=0x40000>
1300 */
1301 {
1302 BlueStore store(g_ceph_context, "", 0x10000);
1303 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1304 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1305 BlueStore::ExtentMap em(&onode);
1306
1307 BlueStore::old_extent_map_t old_extents;
1308 BlueStore::GarbageCollector gc(g_ceph_context);
1309 int64_t saving;
1310 BlueStore::BlobRef b1(new BlueStore::Blob);
1311 BlueStore::BlobRef b2(new BlueStore::Blob);
1312 BlueStore::BlobRef b3(new BlueStore::Blob);
1313 BlueStore::BlobRef b4(new BlueStore::Blob);
1314 b1->shared_blob = new BlueStore::SharedBlob(coll.get());
1315 b2->shared_blob = new BlueStore::SharedBlob(coll.get());
1316 b3->shared_blob = new BlueStore::SharedBlob(coll.get());
1317 b4->shared_blob = new BlueStore::SharedBlob(coll.get());
1318 b1->dirty_blob().set_compressed(0x40000, 0x20000);
1319 b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x20000));
1320 b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x10000));
1321 b3->dirty_blob().allocated_test(bluestore_pextent_t(2, 0x20000));
1322 b4->dirty_blob().allocated_test(bluestore_pextent_t(3, 0x10000));
1323
1324 em.extent_map.insert(*new BlueStore::Extent(0, 0, 0x8000, b1));
1325 b1->get_ref(coll.get(), 0, 0x8000);
1326 em.extent_map.insert(
1327 *new BlueStore::Extent(0x8000, 0x8000, 0x8000, b2)); // new extent
1328 b2->get_ref(coll.get(), 0x8000, 0x8000);
1329 em.extent_map.insert(
1330 *new BlueStore::Extent(0x10000, 0, 0x20000, b3)); // new extent
1331 b3->get_ref(coll.get(), 0, 0x20000);
1332 em.extent_map.insert(
1333 *new BlueStore::Extent(0x30000, 0, 0xf000, b4)); // new extent
1334 b4->get_ref(coll.get(), 0, 0xf000);
1335 em.extent_map.insert(*new BlueStore::Extent(0x3f000, 0x3f000, 0x1000, b1));
1336 b1->get_ref(coll.get(), 0x3f000, 0x1000);
1337
1338 old_extents.push_back(*new BlueStore::OldExtent(0x8000, 0x8000, 0x8000, b1));
1339 old_extents.push_back(
1340 *new BlueStore::OldExtent(0x10000, 0x10000, 0x20000, b1));
1341 old_extents.push_back(*new BlueStore::OldExtent(0x30000, 0x30000, 0xf000, b1));
1342
1343 saving = gc.estimate(0x30000, 0xf000, em, old_extents, 0x10000);
1344 ASSERT_EQ(saving, 2);
1345 auto& to_collect = gc.get_extents_to_collect();
1346 ASSERT_EQ(to_collect.num_intervals(), 2u);
1347 {
1348 auto it1 = to_collect.begin();
1349 auto it2 = ++to_collect.begin();
1350 using p = decltype(*it1);
1351 {
1352 auto v1 = p{0x0ul ,0x8000ul};
1353 auto v2 = p{0x0ul, 0x8000ul};
1354 ASSERT_TRUE(*it1 == v1 || *it2 == v2);
1355 }
1356 {
1357 auto v1 = p{0x3f000ul, 0x1000ul};
1358 auto v2 = p{0x3f000ul, 0x1000ul};
1359 ASSERT_TRUE(*it1 == v1 || *it2 == v2);
1360 }
1361 }
1362
1363 em.clear();
1364 clear_and_dispose(old_extents);
1365 }
1366 /*
1367 original disposition
1368 min_alloc_size = 0x1000
1369 extent1 <loffs = 0, boffs = 0, len = 0x4000>
1370 -> blob1<compressed, len_on_disk=0x2000, logical_len=0x4000>
1371 write 0x3000~4000 resulted in the following extent map
1372 (future feature - suppose we can compress incoming write prior to
1373 GC invocation)
1374
1375 extent1 <loffs = 0, boffs = 0, len = 0x4000>
1376 -> blob1<compressed, len_on_disk=0x2000, logical_len=0x4000>
1377 extent2 <loffs = 0x3000, boffs = 0, len = 0x4000>
1378 -> blob2<compressed, len_on_disk=0x2000, llen=0x4000>
1379 */
1380 {
1381 BlueStore::GarbageCollector gc(g_ceph_context);
1382 int64_t saving;
1383 BlueStore::BlobRef b1(new BlueStore::Blob);
1384 BlueStore::BlobRef b2(new BlueStore::Blob);
1385 b1->shared_blob = new BlueStore::SharedBlob(coll.get());
1386 b2->shared_blob = new BlueStore::SharedBlob(coll.get());
1387 b1->dirty_blob().set_compressed(0x4000, 0x2000);
1388 b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x2000));
1389 b2->dirty_blob().set_compressed(0x4000, 0x2000);
1390 b2->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x2000));
1391
1392 em.extent_map.insert(*new BlueStore::Extent(0, 0, 0x3000, b1));
1393 b1->get_ref(coll.get(), 0, 0x3000);
1394 em.extent_map.insert(
1395 *new BlueStore::Extent(0x3000, 0, 0x4000, b2)); // new extent
1396 b2->get_ref(coll.get(), 0, 0x4000);
1397
1398 old_extents.push_back(*new BlueStore::OldExtent(0x3000, 0x3000, 0x1000, b1));
1399
1400 saving = gc.estimate(0x3000, 0x4000, em, old_extents, 0x1000);
1401 ASSERT_EQ(saving, 0);
1402 auto& to_collect = gc.get_extents_to_collect();
1403 ASSERT_EQ(to_collect.num_intervals(), 0u);
1404 em.clear();
1405 clear_and_dispose(old_extents);
1406 }
1407 /*
1408 original disposition
1409 min_alloc_size = 0x10000
1410 extent0 <loffs = 0, boffs = 0, len = 0x20000>
1411 -> blob0<compressed, len_on_disk=0x10000, logical_len=0x20000>
1412 extent1 <loffs = 0x20000, boffs = 0, len = 0x20000>
1413 -> blob1<compressed, len_on_disk=0x10000, logical_len=0x20000>
1414 write 0x8000~37000 resulted in the following extent map prior
1415 to GC for the last write_small(0x30000~0xf000)
1416
1417 extent0 <loffs = 0, boffs = 0, len = 0x8000>
1418 -> blob0<compressed, len_on_disk=0x10000, logical_len=0x20000>
1419 extent2 <loffs = 0x8000, boffs = 0x8000, len = 0x8000>
1420 -> blob2<raw, len_on_disk=0x10000, llen=0x10000>
1421 extent3 <loffs = 0x10000, boffs = 0, len = 0x20000>
1422 -> blob3<raw, len_on_disk=0x20000, llen=0x20000>
1423 extent4 <loffs = 0x30000, boffs = 0, len = 0xf000>
1424 -> blob4<raw, len_on_disk=0x1000, llen=0x1000>
1425 extent5 <loffs = 0x3f000, boffs = 0x1f000, len = 0x1000>
1426 -> blob1<compressed, len_on_disk=0x10000, llen=0x20000>
1427 */
1428 {
1429 BlueStore store(g_ceph_context, "", 0x10000);
1430 auto coll = ceph::make_ref<BlueStore::Collection>(&store, oc, bc, coll_t());
1431 BlueStore::Onode onode(coll.get(), ghobject_t(), "");
1432 BlueStore::ExtentMap em(&onode);
1433
1434 BlueStore::old_extent_map_t old_extents;
1435 BlueStore::GarbageCollector gc(g_ceph_context);
1436 int64_t saving;
1437 BlueStore::BlobRef b0(new BlueStore::Blob);
1438 BlueStore::BlobRef b1(new BlueStore::Blob);
1439 BlueStore::BlobRef b2(new BlueStore::Blob);
1440 BlueStore::BlobRef b3(new BlueStore::Blob);
1441 BlueStore::BlobRef b4(new BlueStore::Blob);
1442 b0->shared_blob = new BlueStore::SharedBlob(coll.get());
1443 b1->shared_blob = new BlueStore::SharedBlob(coll.get());
1444 b2->shared_blob = new BlueStore::SharedBlob(coll.get());
1445 b3->shared_blob = new BlueStore::SharedBlob(coll.get());
1446 b4->shared_blob = new BlueStore::SharedBlob(coll.get());
1447 b0->dirty_blob().set_compressed(0x2000, 0x1000);
1448 b0->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x10000));
1449 b1->dirty_blob().set_compressed(0x20000, 0x10000);
1450 b1->dirty_blob().allocated_test(bluestore_pextent_t(0, 0x10000));
1451 b2->dirty_blob().allocated_test(bluestore_pextent_t(1, 0x10000));
1452 b3->dirty_blob().allocated_test(bluestore_pextent_t(2, 0x20000));
1453 b4->dirty_blob().allocated_test(bluestore_pextent_t(3, 0x1000));
1454
1455 em.extent_map.insert(*new BlueStore::Extent(0, 0, 0x8000, b0));
1456 b0->get_ref(coll.get(), 0, 0x8000);
1457 em.extent_map.insert(
1458 *new BlueStore::Extent(0x8000, 0x8000, 0x8000, b2)); // new extent
1459 b2->get_ref(coll.get(), 0x8000, 0x8000);
1460 em.extent_map.insert(
1461 *new BlueStore::Extent(0x10000, 0, 0x20000, b3)); // new extent
1462 b3->get_ref(coll.get(), 0, 0x20000);
1463 em.extent_map.insert(
1464 *new BlueStore::Extent(0x30000, 0, 0xf000, b4)); // new extent
1465 b4->get_ref(coll.get(), 0, 0xf000);
1466 em.extent_map.insert(*new BlueStore::Extent(0x3f000, 0x1f000, 0x1000, b1));
1467 b1->get_ref(coll.get(), 0x1f000, 0x1000);
1468
1469 old_extents.push_back(*new BlueStore::OldExtent(0x8000, 0x8000, 0x8000, b0));
1470 old_extents.push_back(
1471 *new BlueStore::OldExtent(0x10000, 0x10000, 0x10000, b0));
1472 old_extents.push_back(
1473 *new BlueStore::OldExtent(0x20000, 0x00000, 0x1f000, b1));
1474
1475 saving = gc.estimate(0x30000, 0xf000, em, old_extents, 0x10000);
1476 ASSERT_EQ(saving, 2);
1477 auto& to_collect = gc.get_extents_to_collect();
1478 ASSERT_EQ(to_collect.num_intervals(), 2u);
1479 {
1480 auto it1 = to_collect.begin();
1481 auto it2 = ++to_collect.begin();
1482 using p = decltype(*it1);
1483 {
1484 auto v1 = p{0x0ul, 0x8000ul};
1485 auto v2 = p{0x0ul, 0x8000ul};
1486 ASSERT_TRUE(*it1 == v1 || *it2 == v2);
1487 }
1488 {
1489 auto v1 = p{0x3f000ul, 0x1000ul};
1490 auto v2 = p{0x3f000ul, 0x1000ul};
1491 ASSERT_TRUE(*it1 == v1 || *it2 == v2);
1492 }
1493 }
1494
1495 em.clear();
1496 clear_and_dispose(old_extents);
1497 }
1498 }
1499
1500 TEST(BlueStoreRepairer, StoreSpaceTracker)
1501 {
1502 BlueStoreRepairer::StoreSpaceTracker bmap0;
1503 bmap0.init((uint64_t)4096 * 1024 * 1024 * 1024, 0x1000);
1504 ASSERT_EQ(bmap0.granularity, 2 * 1024 * 1024U);
1505 ASSERT_EQ(bmap0.collections_bfs.size(), 2048u * 1024u);
1506 ASSERT_EQ(bmap0.objects_bfs.size(), 2048u * 1024u);
1507
1508 BlueStoreRepairer::StoreSpaceTracker bmap;
1509 bmap.init(0x2000 * 0x1000 - 1, 0x1000, 512 * 1024);
1510 ASSERT_EQ(bmap.granularity, 0x1000u);
1511 ASSERT_EQ(bmap.collections_bfs.size(), 0x2000u);
1512 ASSERT_EQ(bmap.objects_bfs.size(), 0x2000u);
1513
1514 coll_t cid;
1515 ghobject_t hoid;
1516
1517 ASSERT_FALSE(bmap.is_used(cid, 0));
1518 ASSERT_FALSE(bmap.is_used(hoid, 0));
1519 bmap.set_used(0, 1, cid, hoid);
1520 ASSERT_TRUE(bmap.is_used(cid, 0));
1521 ASSERT_TRUE(bmap.is_used(hoid, 0));
1522
1523 ASSERT_FALSE(bmap.is_used(cid, 0x1023));
1524 ASSERT_FALSE(bmap.is_used(hoid, 0x1023));
1525 ASSERT_FALSE(bmap.is_used(cid, 0x2023));
1526 ASSERT_FALSE(bmap.is_used(hoid, 0x2023));
1527 ASSERT_FALSE(bmap.is_used(cid, 0x3023));
1528 ASSERT_FALSE(bmap.is_used(hoid, 0x3023));
1529 bmap.set_used(0x1023, 0x3000, cid, hoid);
1530 ASSERT_TRUE(bmap.is_used(cid, 0x1023));
1531 ASSERT_TRUE(bmap.is_used(hoid, 0x1023));
1532 ASSERT_TRUE(bmap.is_used(cid, 0x2023));
1533 ASSERT_TRUE(bmap.is_used(hoid, 0x2023));
1534 ASSERT_TRUE(bmap.is_used(cid, 0x3023));
1535 ASSERT_TRUE(bmap.is_used(hoid, 0x3023));
1536
1537 ASSERT_FALSE(bmap.is_used(cid, 0x9001));
1538 ASSERT_FALSE(bmap.is_used(hoid, 0x9001));
1539 ASSERT_FALSE(bmap.is_used(cid, 0xa001));
1540 ASSERT_FALSE(bmap.is_used(hoid, 0xa001));
1541 ASSERT_FALSE(bmap.is_used(cid, 0xb000));
1542 ASSERT_FALSE(bmap.is_used(hoid, 0xb000));
1543 ASSERT_FALSE(bmap.is_used(cid, 0xc000));
1544 ASSERT_FALSE(bmap.is_used(hoid, 0xc000));
1545 bmap.set_used(0x9001, 0x2fff, cid, hoid);
1546 ASSERT_TRUE(bmap.is_used(cid, 0x9001));
1547 ASSERT_TRUE(bmap.is_used(hoid, 0x9001));
1548 ASSERT_TRUE(bmap.is_used(cid, 0xa001));
1549 ASSERT_TRUE(bmap.is_used(hoid, 0xa001));
1550 ASSERT_TRUE(bmap.is_used(cid, 0xb001));
1551 ASSERT_TRUE(bmap.is_used(hoid, 0xb001));
1552 ASSERT_FALSE(bmap.is_used(cid, 0xc000));
1553 ASSERT_FALSE(bmap.is_used(hoid, 0xc000));
1554
1555 bmap.set_used(0xa001, 0x2, cid, hoid);
1556 ASSERT_TRUE(bmap.is_used(cid, 0x9001));
1557 ASSERT_TRUE(bmap.is_used(hoid, 0x9001));
1558 ASSERT_TRUE(bmap.is_used(cid, 0xa001));
1559 ASSERT_TRUE(bmap.is_used(hoid, 0xa001));
1560 ASSERT_TRUE(bmap.is_used(cid, 0xb001));
1561 ASSERT_TRUE(bmap.is_used(hoid, 0xb001));
1562 ASSERT_FALSE(bmap.is_used(cid, 0xc000));
1563 ASSERT_FALSE(bmap.is_used(hoid, 0xc000));
1564
1565 ASSERT_FALSE(bmap.is_used(cid, 0xc0000));
1566 ASSERT_FALSE(bmap.is_used(hoid, 0xc0000));
1567 ASSERT_FALSE(bmap.is_used(cid, 0xc1000));
1568 ASSERT_FALSE(bmap.is_used(hoid, 0xc1000));
1569
1570 bmap.set_used(0xc0000, 0x2000, cid, hoid);
1571 ASSERT_TRUE(bmap.is_used(cid, 0xc0000));
1572 ASSERT_TRUE(bmap.is_used(hoid, 0xc0000));
1573 ASSERT_TRUE(bmap.is_used(cid, 0xc1000));
1574 ASSERT_TRUE(bmap.is_used(hoid, 0xc1000));
1575
1576 interval_set<uint64_t> extents;
1577 extents.insert(0,0x500);
1578 extents.insert(0x800,0x100);
1579 extents.insert(0x1000,0x1000);
1580 extents.insert(0xa001,1);
1581 extents.insert(0xa0000,0xff8);
1582
1583 ASSERT_EQ(3u, bmap.filter_out(extents));
1584 ASSERT_TRUE(bmap.is_used(cid));
1585 ASSERT_TRUE(bmap.is_used(hoid));
1586
1587 BlueStoreRepairer::StoreSpaceTracker bmap2;
1588 bmap2.init((uint64_t)0x3223b1d1000, 0x10000);
1589 ASSERT_EQ(0x1a0000u, bmap2.granularity);
1590 ASSERT_EQ(0x1edae4u, bmap2.collections_bfs.size());
1591 ASSERT_EQ(0x1edae4u, bmap2.objects_bfs.size());
1592 bmap2.set_used(0x3223b190000, 0x10000, cid, hoid);
1593 ASSERT_TRUE(bmap2.is_used(cid, 0x3223b190000));
1594 ASSERT_TRUE(bmap2.is_used(hoid, 0x3223b190000));
1595 ASSERT_TRUE(bmap2.is_used(cid, 0x3223b19f000));
1596 ASSERT_TRUE(bmap2.is_used(hoid, 0x3223b19ffff));
1597 }
1598
1599 TEST(bluestore_blob_t, unused)
1600 {
1601 {
1602 bluestore_blob_t b;
1603 uint64_t min_alloc_size = 64 << 10; // 64 kB
1604
1605 // _do_write_small 0x0~1000
1606 uint64_t offset = 0x0;
1607 uint64_t length = 0x1000; // 4kB
1608 uint64_t suggested_boff = 0;
1609 PExtentVector extents;
1610 extents.emplace_back(0x1a560000, min_alloc_size);
1611 b.allocated(p2align(suggested_boff, min_alloc_size), 0 /*no matter*/, extents);
1612 b.mark_used(offset, length);
1613 ASSERT_FALSE(b.is_unused(offset, length));
1614
1615 // _do_write_small 0x2000~1000
1616 offset = 0x2000;
1617 length = 0x1000;
1618 b.add_unused(0, 0x10000);
1619 ASSERT_TRUE(b.is_unused(offset, length));
1620 b.mark_used(offset, length);
1621 ASSERT_FALSE(b.is_unused(offset, length));
1622
1623 // _do_write_small 0xc000~2000
1624 offset = 0xc000;
1625 length = 0x2000;
1626 ASSERT_TRUE(b.is_unused(offset, length));
1627 b.mark_used(offset, length);
1628 ASSERT_FALSE(b.is_unused(offset, length));
1629 }
1630
1631 {
1632 bluestore_blob_t b;
1633 uint64_t min_alloc_size = 64 << 10; // 64 kB
1634
1635 // _do_write_small 0x11000~1000
1636 uint64_t offset = 0x11000;
1637 uint64_t length = 0x1000; // 4kB
1638 uint64_t suggested_boff = 0x11000;
1639 PExtentVector extents;
1640 extents.emplace_back(0x1a560000, min_alloc_size);
1641 b.allocated(p2align(suggested_boff, min_alloc_size), 0 /*no matter*/, extents);
1642 b.add_unused(0, offset);
1643 b.add_unused(offset + length, min_alloc_size * 2 - offset - length);
1644 b.mark_used(offset, length);
1645 ASSERT_FALSE(b.is_unused(offset, length));
1646
1647 // _do_write_small 0x15000~3000
1648 offset = 0x15000;
1649 length = 0x3000;
1650 ASSERT_TRUE(b.is_unused(offset, length));
1651 b.mark_used(offset, length);
1652 ASSERT_FALSE(b.is_unused(offset, length));
1653 }
1654
1655 {
1656 // reuse blob
1657 bluestore_blob_t b;
1658 uint64_t min_alloc_size = 64 << 10; // 64 kB
1659
1660 // _do_write_small 0x2a000~1000
1661 // and 0x1d000~1000
1662 uint64_t unused_granularity = 0x3000;
1663 // offsets and lenght below are selected to
1664 // be aligned with unused_granularity
1665 uint64_t offset0 = 0x2a000;
1666 uint64_t offset = 0x1d000;
1667 uint64_t length = 0x1000; // 4kB
1668 PExtentVector extents;
1669 extents.emplace_back(0x410000, min_alloc_size);
1670 b.allocated(p2align(offset0, min_alloc_size), min_alloc_size, extents);
1671 b.add_unused(0, min_alloc_size * 3);
1672 b.mark_used(offset0, length);
1673 ASSERT_FALSE(b.is_unused(offset0, length));
1674 ASSERT_TRUE(b.is_unused(offset, length));
1675
1676 extents.clear();
1677 extents.emplace_back(0x430000, min_alloc_size);
1678 b.allocated(p2align(offset, min_alloc_size), min_alloc_size, extents);
1679 b.mark_used(offset, length);
1680 ASSERT_FALSE(b.is_unused(offset0, length));
1681 ASSERT_FALSE(b.is_unused(offset, length));
1682 ASSERT_FALSE(b.is_unused(offset, unused_granularity));
1683
1684 ASSERT_TRUE(b.is_unused(0, offset / unused_granularity * unused_granularity));
1685 ASSERT_TRUE(b.is_unused(offset + length, offset0 - offset - length));
1686 auto end0_aligned = round_up_to(offset0 + length, unused_granularity);
1687 ASSERT_TRUE(b.is_unused(end0_aligned, min_alloc_size * 3 - end0_aligned));
1688 }
1689 }
1690 // This UT is primarily intended to show how repair procedure
1691 // causes erroneous write to INVALID_OFFSET which is reported in
1692 // https://tracker.ceph.com/issues/51682
1693 // Basic map_any functionality is tested as well though.
1694 //
1695 TEST(bluestore_blob_t, wrong_map_bl_in_51682)
1696 {
1697 {
1698 bluestore_blob_t b;
1699 uint64_t min_alloc_size = 4 << 10; // 64 kB
1700
1701 b.allocated_test(bluestore_pextent_t(0x17ba000, 4 * min_alloc_size));
1702 b.allocated_test(bluestore_pextent_t(0x17bf000, 4 * min_alloc_size));
1703 b.allocated_test(
1704 bluestore_pextent_t(
1705 bluestore_pextent_t::INVALID_OFFSET,
1706 1 * min_alloc_size));
1707 b.allocated_test(bluestore_pextent_t(0x153c44d000, 7 * min_alloc_size));
1708
1709 b.mark_used(0, 0x8000);
1710 b.mark_used(0x9000, 0x7000);
1711
1712 string s(0x7000, 'a');
1713 bufferlist bl;
1714 bl.append(s);
1715 const size_t num_expected_entries = 5;
1716 uint64_t expected[num_expected_entries][2] = {
1717 {0x17ba000, 0x4000},
1718 {0x17bf000, 0x3000},
1719 {0x17c0000, 0x3000},
1720 {0xffffffffffffffff, 0x1000},
1721 {0x153c44d000, 0x3000}};
1722 size_t expected_pos = 0;
1723 b.map_bl(0, bl,
1724 [&](uint64_t o, bufferlist& bl) {
1725 ASSERT_EQ(o, expected[expected_pos][0]);
1726 ASSERT_EQ(bl.length(), expected[expected_pos][1]);
1727 ++expected_pos;
1728 });
1729 // 0x5000 is an improper offset presumably provided when doing a repair
1730 b.map_bl(0x5000, bl,
1731 [&](uint64_t o, bufferlist& bl) {
1732 ASSERT_EQ(o, expected[expected_pos][0]);
1733 ASSERT_EQ(bl.length(), expected[expected_pos][1]);
1734 ++expected_pos;
1735 });
1736 ASSERT_EQ(expected_pos, num_expected_entries);
1737 }
1738 }
1739
1740 int main(int argc, char **argv) {
1741 vector<const char*> args;
1742 argv_to_vec(argc, (const char **)argv, args);
1743 auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
1744 CODE_ENVIRONMENT_UTILITY,
1745 CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
1746 common_init_finish(g_ceph_context);
1747 ::testing::InitGoogleTest(&argc, argv);
1748 return RUN_ALL_TESTS();
1749 }
Ceph