1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
8 * Author: Loic Dachary <loic@dachary.org>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU Library Public License as published by
12 * the Free Software Foundation; either version 2, or (at your option)
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU Library Public License for more details.
24 #include "gtest/gtest.h"
25 #include "osd/PGLog.h"
26 #include "osd/OSDMap.h"
27 #include "include/coredumpctl.h"
28 #include "../objectstore/store_test_fixture.h"
32 struct PGLogTestBase
{
33 static hobject_t
mk_obj(unsigned id
) {
42 static eversion_t
mk_evt(unsigned ep
, unsigned v
) {
43 return eversion_t(ep
, v
);
45 static pg_log_entry_t
mk_ple_mod(
46 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
, osd_reqid_t reqid
) {
48 e
.mark_unrollbackable();
49 e
.op
= pg_log_entry_t::MODIFY
;
56 static pg_log_entry_t
mk_ple_dt(
57 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
, osd_reqid_t reqid
) {
59 e
.mark_unrollbackable();
60 e
.op
= pg_log_entry_t::DELETE
;
67 static pg_log_entry_t
mk_ple_ldt(
68 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
) {
70 e
.mark_unrollbackable();
71 e
.op
= pg_log_entry_t::LOST_DELETE
;
77 static pg_log_entry_t
mk_ple_mod_rb(
78 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
, osd_reqid_t reqid
) {
80 e
.op
= pg_log_entry_t::MODIFY
;
87 static pg_log_entry_t
mk_ple_dt_rb(
88 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
, osd_reqid_t reqid
) {
90 e
.op
= pg_log_entry_t::DELETE
;
97 static pg_log_entry_t
mk_ple_err(
98 const hobject_t
&hoid
, eversion_t v
, osd_reqid_t reqid
) {
100 e
.op
= pg_log_entry_t::ERROR
;
103 e
.prior_version
= eversion_t(0, 0);
107 static pg_log_entry_t
mk_ple_mod(
108 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
) {
109 return mk_ple_mod(hoid
, v
, pv
, osd_reqid_t());
111 static pg_log_entry_t
mk_ple_dt(
112 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
) {
113 return mk_ple_dt(hoid
, v
, pv
, osd_reqid_t());
115 static pg_log_entry_t
mk_ple_mod_rb(
116 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
) {
117 return mk_ple_mod_rb(hoid
, v
, pv
, osd_reqid_t());
119 static pg_log_entry_t
mk_ple_dt_rb(
120 const hobject_t
&hoid
, eversion_t v
, eversion_t pv
) {
121 return mk_ple_dt_rb(hoid
, v
, pv
, osd_reqid_t());
123 static pg_log_entry_t
mk_ple_err(
124 const hobject_t
&hoid
, eversion_t v
) {
125 return mk_ple_err(hoid
, v
, osd_reqid_t());
130 class PGLogTest
: virtual public ::testing::Test
, protected PGLog
, public PGLogTestBase
{
132 PGLogTest() : PGLog(g_ceph_context
) {}
133 void SetUp() override
{
134 missing
.may_include_deletes
= true;
137 #include "common/ceph_context.h"
138 #include "common/config.h"
140 void TearDown() override
{
146 list
<pg_log_entry_t
> base
;
147 list
<pg_log_entry_t
> auth
;
148 list
<pg_log_entry_t
> div
;
153 set
<hobject_t
> toremove
;
154 list
<pg_log_entry_t
> torollback
;
155 bool deletes_during_peering
;
163 TestCase() : deletes_during_peering(false) {}
165 init
.may_include_deletes
= !deletes_during_peering
;
166 final
.may_include_deletes
= !deletes_during_peering
;
167 fullauth
.log
.insert(fullauth
.log
.end(), base
.begin(), base
.end());
168 fullauth
.log
.insert(fullauth
.log
.end(), auth
.begin(), auth
.end());
169 fulldiv
.log
.insert(fulldiv
.log
.end(), base
.begin(), base
.end());
170 fulldiv
.log
.insert(fulldiv
.log
.end(), div
.begin(), div
.end());
172 fullauth
.head
= authinfo
.last_update
= fullauth
.log
.rbegin()->version
;
173 authinfo
.last_complete
= fullauth
.log
.rbegin()->version
;
174 authinfo
.log_tail
= fullauth
.log
.begin()->version
;
175 authinfo
.log_tail
.version
--;
176 fullauth
.tail
= authinfo
.log_tail
;
177 authinfo
.last_backfill
= hobject_t::get_max();
179 fulldiv
.head
= divinfo
.last_update
= fulldiv
.log
.rbegin()->version
;
180 divinfo
.last_complete
= eversion_t();
181 divinfo
.log_tail
= fulldiv
.log
.begin()->version
;
182 divinfo
.log_tail
.version
--;
183 fulldiv
.tail
= divinfo
.log_tail
;
184 divinfo
.last_backfill
= hobject_t::get_max();
186 if (init
.get_items().empty()) {
187 divinfo
.last_complete
= divinfo
.last_update
;
189 eversion_t fmissing
= init
.get_items().at(init
.get_rmissing().begin()->second
).need
;
190 for (list
<pg_log_entry_t
>::const_iterator i
= fulldiv
.log
.begin();
191 i
!= fulldiv
.log
.end();
193 if (i
->version
< fmissing
)
194 divinfo
.last_complete
= i
->version
;
203 void set_div_bounds(eversion_t head
, eversion_t tail
) {
204 fulldiv
.tail
= divinfo
.log_tail
= tail
;
205 fulldiv
.head
= divinfo
.last_update
= head
;
207 void set_auth_bounds(eversion_t head
, eversion_t tail
) {
208 fullauth
.tail
= authinfo
.log_tail
= tail
;
209 fullauth
.head
= authinfo
.last_update
= head
;
211 const IndexedLog
&get_fullauth() const { return fullauth
; }
212 const IndexedLog
&get_fulldiv() const { return fulldiv
; }
213 const pg_info_t
&get_authinfo() const { return authinfo
; }
214 const pg_info_t
&get_divinfo() const { return divinfo
; }
215 }; // struct TestCase
217 struct LogHandler
: public PGLog::LogEntryHandler
{
218 set
<hobject_t
> removed
;
219 list
<pg_log_entry_t
> rolledback
;
222 const pg_log_entry_t
&entry
) override
{
223 rolledback
.push_back(entry
);
226 const pg_log_entry_t
&entry
) override
{}
228 const hobject_t
&hoid
) override
{
229 removed
.insert(hoid
);
231 void try_stash(const hobject_t
&, version_t
) override
{
232 // lost/unfound cases are not tested yet
235 const pg_log_entry_t
&entry
) override
{}
238 template <typename missing_t
>
240 const TestCase
&tcase
,
241 const missing_t
&missing
) {
242 ASSERT_EQ(tcase
.final
.get_items().size(), missing
.get_items().size());
243 for (auto i
= missing
.get_items().begin();
244 i
!= missing
.get_items().end();
246 EXPECT_TRUE(tcase
.final
.get_items().count(i
->first
));
247 EXPECT_EQ(tcase
.final
.get_items().find(i
->first
)->second
.need
, i
->second
.need
);
248 EXPECT_EQ(tcase
.final
.get_items().find(i
->first
)->second
.have
, i
->second
.have
);
250 bool correct
= missing
.debug_verify_from_init(tcase
.init
, &(std::cout
));
251 ASSERT_TRUE(correct
);
254 void verify_sideeffects(
255 const TestCase
&tcase
,
256 const LogHandler
&handler
) {
257 ASSERT_EQ(tcase
.toremove
.size(), handler
.removed
.size());
258 ASSERT_EQ(tcase
.torollback
.size(), handler
.rolledback
.size());
261 list
<pg_log_entry_t
>::const_iterator titer
= tcase
.torollback
.begin();
262 list
<pg_log_entry_t
>::const_iterator hiter
= handler
.rolledback
.begin();
263 for (; titer
!= tcase
.torollback
.end(); ++titer
, ++hiter
) {
264 EXPECT_EQ(titer
->version
, hiter
->version
);
269 set
<hobject_t
>::const_iterator titer
= tcase
.toremove
.begin();
270 set
<hobject_t
>::const_iterator hiter
= handler
.removed
.begin();
271 for (; titer
!= tcase
.toremove
.end(); ++titer
, ++hiter
) {
272 EXPECT_EQ(*titer
, *hiter
);
277 void test_merge_log(const TestCase
&tcase
) {
279 log
= tcase
.get_fulldiv();
280 pg_info_t info
= tcase
.get_divinfo();
282 missing
= tcase
.init
;
286 olog
= tcase
.get_fullauth();
287 pg_info_t oinfo
= tcase
.get_authinfo();
290 bool dirty_info
= false;
291 bool dirty_big_info
= false;
293 oinfo
, std::move(olog
), pg_shard_t(1, shard_id_t(0)), info
,
294 &h
, dirty_info
, dirty_big_info
);
296 ASSERT_EQ(info
.last_update
, oinfo
.last_update
);
297 verify_missing(tcase
, missing
);
298 verify_sideeffects(tcase
, h
);
301 void test_proc_replica_log(const TestCase
&tcase
) {
303 log
= tcase
.get_fullauth();
304 pg_info_t info
= tcase
.get_authinfo();
306 pg_missing_t omissing
= tcase
.init
;
309 olog
= tcase
.get_fulldiv();
310 pg_info_t oinfo
= tcase
.get_divinfo();
313 oinfo
, olog
, omissing
, pg_shard_t(1, shard_id_t(0)));
315 ceph_assert(oinfo
.last_update
>= log
.tail
);
317 if (!tcase
.base
.empty()) {
318 ASSERT_EQ(tcase
.base
.rbegin()->version
, oinfo
.last_update
);
321 for (list
<pg_log_entry_t
>::const_iterator i
= tcase
.auth
.begin();
322 i
!= tcase
.auth
.end();
324 if (i
->version
> oinfo
.last_update
) {
325 if (i
->is_delete() && tcase
.deletes_during_peering
) {
326 omissing
.rm(i
->soid
, i
->version
);
328 omissing
.add_next_event(*i
);
332 verify_missing(tcase
, omissing
);
333 } // test_proc_replica_log
335 void run_test_case(const TestCase
&tcase
) {
336 test_merge_log(tcase
);
337 test_proc_replica_log(tcase
);
339 }; // class PGLogTest
341 struct TestHandler
: public PGLog::LogEntryHandler
{
342 list
<hobject_t
> &removed
;
343 explicit TestHandler(list
<hobject_t
> &removed
) : removed(removed
) {}
346 const pg_log_entry_t
&entry
) override
{}
348 const pg_log_entry_t
&entry
) override
{}
350 const hobject_t
&hoid
) override
{
351 removed
.push_back(hoid
);
353 void cant_rollback(const pg_log_entry_t
&entry
) {}
354 void try_stash(const hobject_t
&, version_t
) override
{
355 // lost/unfound cases are not tested yet
358 const pg_log_entry_t
&entry
) override
{}
361 TEST_F(PGLogTest
, rewind_divergent_log
) {
362 /* +----------------+
371 | (1,4) | x9 < newhead
384 list
<hobject_t
> remove_snap
;
385 bool dirty_info
= false;
386 bool dirty_big_info
= false;
388 hobject_t divergent_object
;
389 eversion_t divergent_version
;
393 divergent
.set_hash(0x9);
397 e
.mark_unrollbackable();
399 e
.version
= eversion_t(1, 1);
400 e
.soid
.set_hash(0x5);
401 log
.tail
= e
.version
;
402 log
.log
.push_back(e
);
403 e
.version
= newhead
= eversion_t(1, 4);
405 e
.op
= pg_log_entry_t::MODIFY
;
406 log
.log
.push_back(e
);
407 e
.version
= divergent_version
= eversion_t(1, 5);
408 e
.prior_version
= eversion_t(1, 4);
410 divergent_object
= e
.soid
;
411 e
.op
= pg_log_entry_t::DELETE
;
412 log
.log
.push_back(e
);
413 log
.head
= e
.version
;
416 info
.last_update
= log
.head
;
417 info
.last_complete
= log
.head
;
420 EXPECT_FALSE(missing
.have_missing());
421 EXPECT_EQ(3U, log
.log
.size());
422 EXPECT_TRUE(remove_snap
.empty());
423 EXPECT_EQ(log
.head
, info
.last_update
);
424 EXPECT_EQ(log
.head
, info
.last_complete
);
425 EXPECT_FALSE(is_dirty());
426 EXPECT_FALSE(dirty_info
);
427 EXPECT_FALSE(dirty_big_info
);
429 TestHandler
h(remove_snap
);
430 rewind_divergent_log(newhead
, info
, &h
,
431 dirty_info
, dirty_big_info
);
433 EXPECT_TRUE(log
.objects
.count(divergent
));
434 EXPECT_TRUE(missing
.is_missing(divergent_object
));
435 EXPECT_EQ(1U, log
.objects
.count(divergent_object
));
436 EXPECT_EQ(2U, log
.log
.size());
437 EXPECT_TRUE(remove_snap
.empty());
438 EXPECT_EQ(newhead
, info
.last_update
);
439 EXPECT_EQ(newhead
, info
.last_complete
);
440 EXPECT_TRUE(is_dirty());
441 EXPECT_TRUE(dirty_info
);
442 EXPECT_TRUE(dirty_big_info
);
445 /* +----------------+
451 tail > (1,1) | NULL |
453 | (1,4) | NULL < newhead
464 list
<hobject_t
> remove_snap
;
465 bool dirty_info
= false;
466 bool dirty_big_info
= false;
468 hobject_t divergent_object
;
469 eversion_t divergent_version
;
470 eversion_t prior_version
;
474 e
.mark_unrollbackable();
476 info
.log_tail
= log
.tail
= eversion_t(1, 1);
477 newhead
= eversion_t(1, 3);
478 e
.version
= divergent_version
= eversion_t(1, 5);
479 e
.soid
.set_hash(0x9);
480 divergent_object
= e
.soid
;
481 e
.op
= pg_log_entry_t::DELETE
;
482 e
.prior_version
= prior_version
= eversion_t(0, 2);
483 log
.log
.push_back(e
);
484 log
.head
= e
.version
;
487 EXPECT_FALSE(missing
.have_missing());
488 EXPECT_EQ(1U, log
.log
.size());
489 EXPECT_TRUE(remove_snap
.empty());
490 EXPECT_FALSE(is_dirty());
491 EXPECT_FALSE(dirty_info
);
492 EXPECT_FALSE(dirty_big_info
);
494 TestHandler
h(remove_snap
);
495 rewind_divergent_log(newhead
, info
, &h
,
496 dirty_info
, dirty_big_info
);
498 EXPECT_TRUE(missing
.is_missing(divergent_object
));
499 EXPECT_EQ(0U, log
.objects
.count(divergent_object
));
500 EXPECT_TRUE(log
.empty());
501 EXPECT_TRUE(remove_snap
.empty());
502 EXPECT_TRUE(is_dirty());
503 EXPECT_TRUE(dirty_info
);
504 EXPECT_TRUE(dirty_big_info
);
511 list
<hobject_t
> remove_snap
;
513 info
.log_tail
= log
.tail
= eversion_t(1, 5);
514 info
.last_update
= eversion_t(1, 6);
515 bool dirty_info
= false;
516 bool dirty_big_info
= false;
520 e
.mark_unrollbackable();
521 e
.version
= eversion_t(1, 5);
522 e
.soid
.set_hash(0x9);
527 e
.mark_unrollbackable();
528 e
.version
= eversion_t(1, 6);
529 e
.soid
.set_hash(0x10);
532 TestHandler
h(remove_snap
);
533 roll_forward_to(eversion_t(1, 6), &h
);
534 rewind_divergent_log(eversion_t(1, 5), info
, &h
,
535 dirty_info
, dirty_big_info
);
537 reset_backfill_claim_log(log
, &h
);
541 TEST_F(PGLogTest
, merge_old_entry
) {
542 // entries > last_backfill are silently ignored
546 ObjectStore::Transaction t
;
548 oe
.mark_unrollbackable();
550 list
<hobject_t
> remove_snap
;
552 info
.last_backfill
= hobject_t();
553 info
.last_backfill
.set_hash(100);
555 ASSERT_GT(oe
.soid
, info
.last_backfill
);
557 EXPECT_FALSE(is_dirty());
558 EXPECT_TRUE(remove_snap
.empty());
559 EXPECT_TRUE(t
.empty());
560 EXPECT_FALSE(missing
.have_missing());
561 EXPECT_TRUE(log
.empty());
563 TestHandler
h(remove_snap
);
564 merge_old_entry(t
, oe
, info
, &h
);
566 EXPECT_FALSE(is_dirty());
567 EXPECT_TRUE(remove_snap
.empty());
568 EXPECT_TRUE(t
.empty());
569 EXPECT_FALSE(missing
.have_missing());
570 EXPECT_TRUE(log
.empty());
573 // the new entry (from the logs) has a version that is higher than
574 // the old entry (from the log entry given in argument) : do
575 // nothing and return false
579 ObjectStore::Transaction t
;
581 list
<hobject_t
> remove_snap
;
584 ne
.mark_unrollbackable();
585 ne
.version
= eversion_t(2,1);
588 EXPECT_FALSE(is_dirty());
589 EXPECT_TRUE(remove_snap
.empty());
590 EXPECT_TRUE(t
.empty());
591 EXPECT_FALSE(missing
.have_missing());
592 EXPECT_EQ(1U, log
.log
.size());
593 EXPECT_EQ(ne
.version
, log
.log
.front().version
);
595 // the newer entry ( from the logs ) can be DELETE
597 log
.log
.front().op
= pg_log_entry_t::DELETE
;
599 oe
.mark_unrollbackable();
600 oe
.version
= eversion_t(1,1);
602 TestHandler
h(remove_snap
);
603 merge_old_entry(t
, oe
, info
, &h
);
606 // if the newer entry is not DELETE, the object must be in missing
608 pg_log_entry_t
&ne
= log
.log
.front();
609 ne
.op
= pg_log_entry_t::MODIFY
;
610 missing
.add_next_event(ne
);
612 oe
.mark_unrollbackable();
613 oe
.version
= eversion_t(1,1);
615 TestHandler
h(remove_snap
);
616 merge_old_entry(t
, oe
, info
, &h
);
618 missing
.rm(ne
.soid
, ne
.version
);
622 EXPECT_FALSE(is_dirty());
623 EXPECT_FALSE(remove_snap
.empty());
624 EXPECT_TRUE(t
.empty());
625 EXPECT_FALSE(missing
.have_missing());
626 EXPECT_EQ(1U, log
.log
.size());
627 EXPECT_EQ(ne
.version
, log
.log
.front().version
);
631 // the new entry (from the logs) has a version that is lower than
632 // the old entry (from the log entry given in argument) and
633 // old and new are delete : do nothing and return false
637 ObjectStore::Transaction t
;
639 oe
.mark_unrollbackable();
641 list
<hobject_t
> remove_snap
;
644 ne
.mark_unrollbackable();
645 ne
.version
= eversion_t(1,1);
646 ne
.op
= pg_log_entry_t::DELETE
;
649 oe
.version
= eversion_t(2,1);
650 oe
.op
= pg_log_entry_t::DELETE
;
652 EXPECT_FALSE(is_dirty());
653 EXPECT_TRUE(remove_snap
.empty());
654 EXPECT_TRUE(t
.empty());
655 EXPECT_FALSE(missing
.have_missing());
656 EXPECT_EQ(1U, log
.log
.size());
658 TestHandler
h(remove_snap
);
659 merge_old_entry(t
, oe
, info
, &h
);
661 EXPECT_FALSE(is_dirty());
662 EXPECT_TRUE(remove_snap
.empty());
663 EXPECT_TRUE(t
.empty());
664 EXPECT_FALSE(missing
.have_missing());
665 EXPECT_EQ(1U, log
.log
.size());
668 // the new entry (from the logs) has a version that is lower than
669 // the old entry (from the log entry given in argument) and
670 // old is update and new is DELETE :
671 // if the object is in missing, it is removed
675 ObjectStore::Transaction t
;
677 oe
.mark_unrollbackable();
679 list
<hobject_t
> remove_snap
;
682 ne
.mark_unrollbackable();
683 ne
.version
= eversion_t(1,1);
684 ne
.op
= pg_log_entry_t::DELETE
;
687 oe
.version
= eversion_t(2,1);
688 oe
.op
= pg_log_entry_t::MODIFY
;
689 missing
.add_next_event(oe
);
692 EXPECT_FALSE(is_dirty());
693 EXPECT_TRUE(remove_snap
.empty());
694 EXPECT_TRUE(t
.empty());
695 EXPECT_TRUE(missing
.is_missing(oe
.soid
));
696 EXPECT_EQ(1U, log
.log
.size());
698 TestHandler
h(remove_snap
);
699 merge_old_entry(t
, oe
, info
, &h
);
702 EXPECT_FALSE(is_dirty());
703 EXPECT_TRUE(remove_snap
.size() > 0);
704 EXPECT_TRUE(t
.empty());
705 EXPECT_FALSE(missing
.have_missing());
706 EXPECT_EQ(1U, log
.log
.size());
709 // there is no new entry (from the logs) and
710 // the old entry (from the log entry given in argument) is not a CLONE and
711 // the old entry prior_version is greater than the tail of the log :
712 // do nothing and return false
716 ObjectStore::Transaction t
;
718 oe
.mark_unrollbackable();
720 list
<hobject_t
> remove_snap
;
722 info
.log_tail
= eversion_t(1,1);
723 oe
.op
= pg_log_entry_t::MODIFY
;
724 oe
.prior_version
= eversion_t(2,1);
725 missing_add(oe
.soid
, oe
.prior_version
, eversion_t());
728 EXPECT_FALSE(is_dirty());
729 EXPECT_TRUE(remove_snap
.empty());
730 EXPECT_TRUE(t
.empty());
731 EXPECT_TRUE(log
.empty());
733 TestHandler
h(remove_snap
);
734 merge_old_entry(t
, oe
, info
, &h
);
737 EXPECT_FALSE(is_dirty());
738 EXPECT_TRUE(remove_snap
.empty());
739 EXPECT_TRUE(t
.empty());
740 EXPECT_TRUE(log
.empty());
743 // there is no new entry (from the logs) and
744 // the old entry (from the log entry given in argument) is not a CLONE and
745 // the old entry (from the log entry given in argument) is not a DELETE and
746 // the old entry prior_version is lower than the tail of the log :
747 // add the old object to the remove_snap list and
748 // add the old object to divergent priors and
749 // add or update the prior_version of the object to missing and
754 ObjectStore::Transaction t
;
756 oe
.mark_unrollbackable();
758 list
<hobject_t
> remove_snap
;
760 info
.log_tail
= eversion_t(2,1);
762 oe
.op
= pg_log_entry_t::MODIFY
;
763 oe
.prior_version
= eversion_t(1,1);
765 EXPECT_FALSE(is_dirty());
766 EXPECT_TRUE(remove_snap
.empty());
767 EXPECT_TRUE(t
.empty());
768 EXPECT_FALSE(missing
.have_missing());
769 EXPECT_TRUE(log
.empty());
771 TestHandler
h(remove_snap
);
772 merge_old_entry(t
, oe
, info
, &h
);
774 EXPECT_TRUE(is_dirty());
775 EXPECT_EQ(oe
.soid
, remove_snap
.front());
776 EXPECT_TRUE(t
.empty());
777 EXPECT_TRUE(missing
.is_missing(oe
.soid
));
778 EXPECT_TRUE(log
.empty());
781 // there is no new entry (from the logs) and
782 // the old entry (from the log entry given in argument) is not a CLONE and
783 // the old entry (from the log entry given in argument) is a DELETE and
784 // the old entry prior_version is lower than the tail of the log :
785 // add the old object to divergent priors and
786 // add or update the prior_version of the object to missing and
791 ObjectStore::Transaction t
;
793 oe
.mark_unrollbackable();
795 list
<hobject_t
> remove_snap
;
797 info
.log_tail
= eversion_t(2,1);
799 oe
.op
= pg_log_entry_t::DELETE
;
800 oe
.prior_version
= eversion_t(1,1);
802 EXPECT_FALSE(is_dirty());
803 EXPECT_TRUE(remove_snap
.empty());
804 EXPECT_TRUE(t
.empty());
805 EXPECT_FALSE(missing
.have_missing());
806 EXPECT_TRUE(log
.empty());
808 TestHandler
h(remove_snap
);
809 merge_old_entry(t
, oe
, info
, &h
);
811 EXPECT_TRUE(is_dirty());
812 EXPECT_TRUE(remove_snap
.empty());
813 EXPECT_TRUE(t
.empty());
814 EXPECT_TRUE(missing
.is_missing(oe
.soid
));
815 EXPECT_TRUE(log
.empty());
819 // there is no new entry (from the logs) and
820 // the old entry (from the log entry given in argument) is not a CLONE and
821 // the old entry (from the log entry given in argument) is not a DELETE and
822 // the old entry prior_version is eversion_t() :
823 // add the old object to the remove_snap list and
824 // remove the prior_version of the object from missing, if any and
829 ObjectStore::Transaction t
;
831 oe
.mark_unrollbackable();
833 list
<hobject_t
> remove_snap
;
835 info
.log_tail
= eversion_t(10,1);
837 oe
.op
= pg_log_entry_t::MODIFY
;
838 oe
.prior_version
= eversion_t();
840 missing
.add(oe
.soid
, eversion_t(1,1), eversion_t(), false);
843 EXPECT_FALSE(is_dirty());
844 EXPECT_TRUE(remove_snap
.empty());
845 EXPECT_TRUE(t
.empty());
846 EXPECT_TRUE(missing
.is_missing(oe
.soid
));
847 EXPECT_TRUE(log
.empty());
849 TestHandler
h(remove_snap
);
850 merge_old_entry(t
, oe
, info
, &h
);
853 EXPECT_FALSE(is_dirty());
854 EXPECT_EQ(oe
.soid
, remove_snap
.front());
855 EXPECT_TRUE(t
.empty());
856 EXPECT_FALSE(missing
.have_missing());
857 EXPECT_TRUE(log
.empty());
862 TEST_F(PGLogTest
, merge_log
) {
863 // head and tail match, last_backfill is set:
872 list
<hobject_t
> remove_snap
;
873 bool dirty_info
= false;
874 bool dirty_big_info
= false;
876 hobject_t
last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
877 info
.last_backfill
= last_backfill
;
878 eversion_t
stat_version(10, 1);
879 info
.stats
.version
= stat_version
;
880 log
.tail
= olog
.tail
= eversion_t(1, 1);
881 log
.head
= olog
.head
= eversion_t(2, 1);
883 EXPECT_FALSE(missing
.have_missing());
884 EXPECT_EQ(0U, log
.log
.size());
885 EXPECT_EQ(stat_version
, info
.stats
.version
);
886 EXPECT_TRUE(remove_snap
.empty());
887 EXPECT_EQ(last_backfill
, info
.last_backfill
);
888 EXPECT_TRUE(info
.purged_snaps
.empty());
889 EXPECT_FALSE(is_dirty());
890 EXPECT_FALSE(dirty_info
);
891 EXPECT_FALSE(dirty_big_info
);
893 TestHandler
h(remove_snap
);
894 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
895 dirty_info
, dirty_big_info
);
897 EXPECT_FALSE(missing
.have_missing());
898 EXPECT_EQ(0U, log
.log
.size());
899 EXPECT_EQ(stat_version
, info
.stats
.version
);
900 EXPECT_TRUE(remove_snap
.empty());
901 EXPECT_TRUE(info
.purged_snaps
.empty());
902 EXPECT_FALSE(is_dirty());
903 EXPECT_FALSE(dirty_info
);
904 EXPECT_FALSE(dirty_big_info
);
907 // head and tail match, last_backfill is not set: info.stats is
908 // copied from oinfo.stats but info.stats.reported_* is guaranteed to
909 // never be replaced by a lower version
917 list
<hobject_t
> remove_snap
;
918 bool dirty_info
= false;
919 bool dirty_big_info
= false;
921 eversion_t
stat_version(10, 1);
922 oinfo
.stats
.version
= stat_version
;
923 info
.stats
.reported_seq
= 1;
924 info
.stats
.reported_epoch
= 10;
925 oinfo
.stats
.reported_seq
= 1;
926 oinfo
.stats
.reported_epoch
= 1;
927 log
.tail
= olog
.tail
= eversion_t(1, 1);
928 log
.head
= olog
.head
= eversion_t(2, 1);
929 missing
.may_include_deletes
= false;
931 EXPECT_FALSE(missing
.have_missing());
932 EXPECT_EQ(0U, log
.log
.size());
933 EXPECT_EQ(eversion_t(), info
.stats
.version
);
934 EXPECT_EQ(1ull, info
.stats
.reported_seq
);
935 EXPECT_EQ(10u, info
.stats
.reported_epoch
);
936 EXPECT_TRUE(remove_snap
.empty());
937 EXPECT_TRUE(info
.last_backfill
.is_max());
938 EXPECT_TRUE(info
.purged_snaps
.empty());
939 EXPECT_FALSE(is_dirty());
940 EXPECT_FALSE(dirty_info
);
941 EXPECT_FALSE(dirty_big_info
);
943 TestHandler
h(remove_snap
);
944 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
945 dirty_info
, dirty_big_info
);
947 EXPECT_FALSE(missing
.have_missing());
948 EXPECT_EQ(0U, log
.log
.size());
949 EXPECT_EQ(stat_version
, info
.stats
.version
);
950 EXPECT_EQ(1ull, info
.stats
.reported_seq
);
951 EXPECT_EQ(10u, info
.stats
.reported_epoch
);
952 EXPECT_TRUE(remove_snap
.empty());
953 EXPECT_TRUE(info
.purged_snaps
.empty());
954 EXPECT_FALSE(is_dirty());
955 EXPECT_FALSE(dirty_info
);
956 EXPECT_FALSE(dirty_big_info
);
960 +--------------------------+
962 +--------+-------+---------+
964 |version | hash | version |
966 | | x5 | (1,1) < tail
969 tail > (1,4) | x7 | |
972 head > (1,5) | x9 | (1,5) < head
975 +--------+-------+---------+
1002 list
<hobject_t
> remove_snap
;
1003 bool dirty_info
= false;
1004 bool dirty_big_info
= false;
1005 missing
.may_include_deletes
= false;
1009 e
.mark_unrollbackable();
1011 e
.version
= eversion_t(1, 4);
1012 e
.soid
.set_hash(0x5);
1013 log
.tail
= e
.version
;
1014 log
.log
.push_back(e
);
1015 e
.version
= eversion_t(1, 5);
1016 e
.soid
.set_hash(0x9);
1017 log
.log
.push_back(e
);
1018 log
.head
= e
.version
;
1021 info
.last_update
= log
.head
;
1023 e
.version
= eversion_t(1, 1);
1024 e
.soid
.set_hash(0x5);
1025 olog
.tail
= e
.version
;
1026 olog
.log
.push_back(e
);
1027 e
.version
= eversion_t(1, 5);
1028 e
.soid
.set_hash(0x9);
1029 olog
.log
.push_back(e
);
1030 olog
.head
= e
.version
;
1033 hobject_t
last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
1034 info
.last_backfill
= last_backfill
;
1035 eversion_t
stat_version(10, 1);
1036 info
.stats
.version
= stat_version
;
1038 EXPECT_FALSE(missing
.have_missing());
1039 EXPECT_EQ(2U, log
.log
.size());
1040 EXPECT_EQ(stat_version
, info
.stats
.version
);
1041 EXPECT_TRUE(remove_snap
.empty());
1042 EXPECT_EQ(last_backfill
, info
.last_backfill
);
1043 EXPECT_TRUE(info
.purged_snaps
.empty());
1044 EXPECT_FALSE(is_dirty());
1045 EXPECT_FALSE(dirty_info
);
1046 EXPECT_FALSE(dirty_big_info
);
1048 TestHandler
h(remove_snap
);
1049 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
1050 dirty_info
, dirty_big_info
);
1052 EXPECT_FALSE(missing
.have_missing());
1053 EXPECT_EQ(3U, log
.log
.size());
1054 EXPECT_EQ(stat_version
, info
.stats
.version
);
1055 EXPECT_TRUE(remove_snap
.empty());
1056 EXPECT_TRUE(info
.purged_snaps
.empty());
1057 EXPECT_TRUE(is_dirty());
1058 EXPECT_TRUE(dirty_info
);
1059 EXPECT_TRUE(dirty_big_info
);
1062 /* +--------------------------+
1064 +--------+-------+---------+
1066 |version | hash | version |
1068 tail > (1,1) | x5 | (1,1) < tail
1071 | (1,2) | x3 | (1,2) < lower_bound
1074 head > (1,3) | x9 | |
1080 | | x7 | (2,4) < head
1082 +--------+-------+---------+
1084 The log entry (1,3) deletes the object x9 but the olog entry (2,3) modifies
1085 it and is authoritative : the log entry (1,3) is divergent.
1095 list
<hobject_t
> remove_snap
;
1096 bool dirty_info
= false;
1097 bool dirty_big_info
= false;
1099 hobject_t divergent_object
;
1100 missing
.may_include_deletes
= true;
1104 e
.mark_unrollbackable();
1106 e
.version
= eversion_t(1, 1);
1107 e
.soid
.set_hash(0x5);
1108 log
.tail
= e
.version
;
1109 log
.log
.push_back(e
);
1110 e
.version
= eversion_t(1, 2);
1111 e
.soid
.set_hash(0x3);
1112 log
.log
.push_back(e
);
1113 e
.version
= eversion_t(1,3);
1114 e
.soid
.set_hash(0x9);
1115 divergent_object
= e
.soid
;
1116 e
.op
= pg_log_entry_t::DELETE
;
1117 log
.log
.push_back(e
);
1118 log
.head
= e
.version
;
1121 info
.last_update
= log
.head
;
1123 e
.version
= eversion_t(1, 1);
1124 e
.soid
.set_hash(0x5);
1125 olog
.tail
= e
.version
;
1126 olog
.log
.push_back(e
);
1127 e
.version
= eversion_t(1, 2);
1128 e
.soid
.set_hash(0x3);
1129 olog
.log
.push_back(e
);
1130 e
.version
= eversion_t(2, 3);
1131 e
.soid
.set_hash(0x9);
1132 e
.op
= pg_log_entry_t::MODIFY
;
1133 olog
.log
.push_back(e
);
1134 e
.version
= eversion_t(2, 4);
1135 e
.soid
.set_hash(0x7);
1136 e
.op
= pg_log_entry_t::DELETE
;
1137 olog
.log
.push_back(e
);
1138 olog
.head
= e
.version
;
1141 snapid_t
purged_snap(1);
1143 oinfo
.last_update
= olog
.head
;
1144 oinfo
.purged_snaps
.insert(purged_snap
);
1147 EXPECT_FALSE(missing
.have_missing());
1148 EXPECT_EQ(1U, log
.objects
.count(divergent_object
));
1149 EXPECT_EQ(3U, log
.log
.size());
1150 EXPECT_TRUE(remove_snap
.empty());
1151 EXPECT_EQ(log
.head
, info
.last_update
);
1152 EXPECT_TRUE(info
.purged_snaps
.empty());
1153 EXPECT_FALSE(is_dirty());
1154 EXPECT_FALSE(dirty_info
);
1155 EXPECT_FALSE(dirty_big_info
);
1157 TestHandler
h(remove_snap
);
1158 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
1159 dirty_info
, dirty_big_info
);
1161 /* When the divergent entry is a DELETE and the authoritative
1162 entry is a MODIFY, the object will be added to missing : it is
1163 a verifiable side effect proving the entry was identified
1166 EXPECT_TRUE(missing
.is_missing(divergent_object
));
1167 EXPECT_EQ(1U, log
.objects
.count(divergent_object
));
1168 EXPECT_EQ(4U, log
.log
.size());
1169 /* DELETE entries from olog that are appended to the hed of the
1170 log, and the divergent version of the object is removed (added
1173 EXPECT_EQ(0x9U
, remove_snap
.front().get_hash());
1174 EXPECT_EQ(log
.head
, info
.last_update
);
1175 EXPECT_TRUE(info
.purged_snaps
.contains(purged_snap
));
1176 EXPECT_TRUE(is_dirty());
1177 EXPECT_TRUE(dirty_info
);
1178 EXPECT_TRUE(dirty_big_info
);
1181 /* +--------------------------+
1183 +--------+-------+---------+
1185 |version | hash | version |
1187 tail > (1,1) | x5 | (1,1) < tail
1190 | (1,2) | x3 | (1,2) < lower_bound
1193 head > (1,3) | x9 | |
1199 | | x7 | (2,4) < head
1201 +--------+-------+---------+
1203 The log entry (1,3) deletes the object x9 but the olog entry (2,3) modifies
1204 it and is authoritative : the log entry (1,3) is divergent.
1214 list
<hobject_t
> remove_snap
;
1215 bool dirty_info
= false;
1216 bool dirty_big_info
= false;
1218 hobject_t divergent_object
;
1222 e
.mark_unrollbackable();
1224 e
.version
= eversion_t(1, 1);
1225 e
.soid
.set_hash(0x5);
1226 log
.tail
= e
.version
;
1227 log
.log
.push_back(e
);
1228 e
.version
= eversion_t(1, 2);
1229 e
.soid
.set_hash(0x3);
1230 log
.log
.push_back(e
);
1231 e
.version
= eversion_t(1,3);
1232 e
.soid
.set_hash(0x9);
1233 divergent_object
= e
.soid
;
1234 e
.op
= pg_log_entry_t::DELETE
;
1235 log
.log
.push_back(e
);
1236 log
.head
= e
.version
;
1239 info
.last_update
= log
.head
;
1241 e
.version
= eversion_t(1, 1);
1242 e
.soid
.set_hash(0x5);
1243 olog
.tail
= e
.version
;
1244 olog
.log
.push_back(e
);
1245 e
.version
= eversion_t(1, 2);
1246 e
.soid
.set_hash(0x3);
1247 olog
.log
.push_back(e
);
1248 e
.version
= eversion_t(2, 3);
1249 e
.soid
.set_hash(0x9);
1250 e
.op
= pg_log_entry_t::MODIFY
;
1251 olog
.log
.push_back(e
);
1252 e
.version
= eversion_t(2, 4);
1253 e
.soid
.set_hash(0x7);
1254 e
.op
= pg_log_entry_t::DELETE
;
1255 olog
.log
.push_back(e
);
1256 olog
.head
= e
.version
;
1259 snapid_t
purged_snap(1);
1261 oinfo
.last_update
= olog
.head
;
1262 oinfo
.purged_snaps
.insert(purged_snap
);
1265 EXPECT_FALSE(missing
.have_missing());
1266 EXPECT_EQ(1U, log
.objects
.count(divergent_object
));
1267 EXPECT_EQ(3U, log
.log
.size());
1268 EXPECT_TRUE(remove_snap
.empty());
1269 EXPECT_EQ(log
.head
, info
.last_update
);
1270 EXPECT_TRUE(info
.purged_snaps
.empty());
1271 EXPECT_FALSE(is_dirty());
1272 EXPECT_FALSE(dirty_info
);
1273 EXPECT_FALSE(dirty_big_info
);
1275 TestHandler
h(remove_snap
);
1276 missing
.may_include_deletes
= false;
1277 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
1278 dirty_info
, dirty_big_info
);
1280 /* When the divergent entry is a DELETE and the authoritative
1281 entry is a MODIFY, the object will be added to missing : it is
1282 a verifiable side effect proving the entry was identified
1285 EXPECT_TRUE(missing
.is_missing(divergent_object
));
1286 EXPECT_EQ(1U, log
.objects
.count(divergent_object
));
1287 EXPECT_EQ(4U, log
.log
.size());
1288 /* DELETE entries from olog that are appended to the hed of the
1289 log, and the divergent version of the object is removed (added
1290 to remove_snap). When peering handles deletes, it is the earlier
1291 version that is in the removed list.
1293 EXPECT_EQ(0x7U
, remove_snap
.front().get_hash());
1294 EXPECT_EQ(log
.head
, info
.last_update
);
1295 EXPECT_TRUE(info
.purged_snaps
.contains(purged_snap
));
1296 EXPECT_TRUE(is_dirty());
1297 EXPECT_TRUE(dirty_info
);
1298 EXPECT_TRUE(dirty_big_info
);
1301 /* +--------------------------+
1303 +--------+-------+---------+
1305 |version | hash | version |
1307 tail > (1,1) | x5 | (1,1) < tail
1310 | (1,4) | x7 | (1,4) < head
1313 head > (1,5) | x9 | |
1316 +--------+-------+---------+
1318 The head of the log entry (1,5) is divergent because it is greater than the
1329 list
<hobject_t
> remove_snap
;
1330 bool dirty_info
= false;
1331 bool dirty_big_info
= false;
1335 e
.mark_unrollbackable();
1337 e
.version
= eversion_t(1, 1);
1338 e
.soid
.set_hash(0x5);
1339 log
.tail
= e
.version
;
1340 log
.log
.push_back(e
);
1341 e
.version
= eversion_t(1, 4);
1342 e
.soid
.set_hash(0x7);
1343 log
.log
.push_back(e
);
1344 e
.version
= eversion_t(1, 5);
1345 e
.soid
.set_hash(0x9);
1346 log
.log
.push_back(e
);
1347 log
.head
= e
.version
;
1350 info
.last_update
= log
.head
;
1352 e
.version
= eversion_t(1, 1);
1353 e
.soid
.set_hash(0x5);
1354 olog
.tail
= e
.version
;
1355 olog
.log
.push_back(e
);
1356 e
.version
= eversion_t(1, 4);
1357 e
.soid
.set_hash(0x7);
1358 olog
.log
.push_back(e
);
1359 olog
.head
= e
.version
;
1362 hobject_t
last_backfill(object_t("oname"), string("key"), 1, 234, 1, "");
1363 info
.last_backfill
= last_backfill
;
1364 eversion_t
stat_version(10, 1);
1365 info
.stats
.version
= stat_version
;
1367 EXPECT_FALSE(missing
.have_missing());
1368 EXPECT_EQ(3U, log
.log
.size());
1369 EXPECT_EQ(stat_version
, info
.stats
.version
);
1370 EXPECT_TRUE(remove_snap
.empty());
1371 EXPECT_EQ(last_backfill
, info
.last_backfill
);
1372 EXPECT_TRUE(info
.purged_snaps
.empty());
1373 EXPECT_FALSE(is_dirty());
1374 EXPECT_FALSE(dirty_info
);
1375 EXPECT_FALSE(dirty_big_info
);
1377 TestHandler
h(remove_snap
);
1378 missing
.may_include_deletes
= false;
1379 merge_log(oinfo
, std::move(olog
), fromosd
, info
, &h
,
1380 dirty_info
, dirty_big_info
);
1382 EXPECT_FALSE(missing
.have_missing());
1383 EXPECT_EQ(2U, log
.log
.size());
1384 EXPECT_EQ(stat_version
, info
.stats
.version
);
1385 EXPECT_EQ(0x9U
, remove_snap
.front().get_hash());
1386 EXPECT_TRUE(info
.purged_snaps
.empty());
1387 EXPECT_TRUE(is_dirty());
1388 EXPECT_TRUE(dirty_info
);
1389 EXPECT_TRUE(dirty_big_info
);
1394 TEST_F(PGLogTest
, proc_replica_log
) {
1395 // empty log : no side effect
1401 pg_missing_t omissing
;
1404 eversion_t
last_update(1, 1);
1405 log
.head
= olog
.head
= oinfo
.last_update
= last_update
;
1406 eversion_t
last_complete(1, 1);
1407 oinfo
.last_complete
= last_complete
;
1409 EXPECT_FALSE(omissing
.have_missing());
1410 EXPECT_EQ(last_update
, oinfo
.last_update
);
1411 EXPECT_EQ(last_complete
, oinfo
.last_complete
);
1413 missing
.may_include_deletes
= false;
1414 proc_replica_log(oinfo
, olog
, omissing
, from
);
1416 EXPECT_FALSE(omissing
.have_missing());
1417 EXPECT_EQ(last_update
, oinfo
.last_update
);
1418 EXPECT_EQ(last_update
, oinfo
.last_complete
);
1421 /* +--------------------------+
1423 +--------+-------+---------+
1425 |version | hash | version |
1427 | | x3 | (1,1) < tail
1430 tail > (1,2) | x5 | |
1433 head > (1,3) | x9 | |
1436 | | x9 | (2,3) < head
1439 +--------+-------+---------+
1441 The log entry (1,3) deletes the object x9 and the olog entry
1442 (2,3) also deletes it : do nothing. The olog tail is ignored
1443 because it is before the log tail.
1451 pg_missing_t omissing
;
1456 e
.mark_unrollbackable();
1458 e
.version
= eversion_t(1, 2);
1459 e
.soid
.set_hash(0x5);
1460 log
.tail
= e
.version
;
1461 log
.log
.push_back(e
);
1462 e
.version
= eversion_t(1, 3);
1463 e
.soid
.set_hash(0x9);
1464 e
.op
= pg_log_entry_t::DELETE
;
1465 log
.log
.push_back(e
);
1466 log
.head
= e
.version
;
1469 e
.version
= eversion_t(1, 1);
1470 e
.soid
.set_hash(0x3);
1471 olog
.tail
= e
.version
;
1472 olog
.log
.push_back(e
);
1473 e
.version
= eversion_t(2, 3);
1474 e
.soid
.set_hash(0x9);
1475 e
.op
= pg_log_entry_t::DELETE
;
1476 olog
.log
.push_back(e
);
1477 olog
.head
= e
.version
;
1479 oinfo
.last_update
= olog
.head
;
1480 oinfo
.last_complete
= olog
.head
;
1483 EXPECT_FALSE(omissing
.have_missing());
1484 EXPECT_EQ(olog
.head
, oinfo
.last_update
);
1485 EXPECT_EQ(olog
.head
, oinfo
.last_complete
);
1487 missing
.may_include_deletes
= false;
1488 proc_replica_log(oinfo
, olog
, omissing
, from
);
1490 EXPECT_FALSE(omissing
.have_missing());
1498 pg_missing_t omissing
;
1501 hobject_t divergent_object
;
1505 e
.mark_unrollbackable();
1508 e
.soid
= divergent_object
;
1509 e
.soid
.set_hash(0x1);
1510 e
.version
= eversion_t(1, 1);
1511 log
.tail
= e
.version
;
1512 log
.log
.push_back(e
);
1514 e
.soid
= divergent_object
;
1515 e
.prior_version
= eversion_t(1, 1);
1516 e
.version
= eversion_t(1, 2);
1517 log
.tail
= e
.version
;
1518 log
.log
.push_back(e
);
1520 e
.soid
.set_hash(0x3);
1521 e
.version
= eversion_t(1, 4);
1522 log
.log
.push_back(e
);
1524 e
.soid
.set_hash(0x7);
1525 e
.version
= eversion_t(1, 5);
1526 log
.log
.push_back(e
);
1528 e
.soid
.set_hash(0x8);
1529 e
.version
= eversion_t(1, 6);
1530 log
.log
.push_back(e
);
1532 e
.soid
.set_hash(0x9);
1533 e
.op
= pg_log_entry_t::DELETE
;
1534 e
.version
= eversion_t(2, 7);
1535 log
.log
.push_back(e
);
1537 e
.soid
.set_hash(0xa);
1538 e
.version
= eversion_t(2, 8);
1539 log
.head
= e
.version
;
1540 log
.log
.push_back(e
);
1545 e
.soid
= divergent_object
;
1546 e
.soid
.set_hash(0x1);
1547 e
.version
= eversion_t(1, 1);
1548 olog
.tail
= e
.version
;
1549 olog
.log
.push_back(e
);
1551 e
.soid
= divergent_object
;
1552 e
.prior_version
= eversion_t(1, 1);
1553 e
.version
= eversion_t(1, 2);
1554 olog
.log
.push_back(e
);
1556 e
.prior_version
= eversion_t(0, 0);
1557 e
.soid
.set_hash(0x3);
1558 e
.version
= eversion_t(1, 4);
1559 olog
.log
.push_back(e
);
1561 e
.soid
.set_hash(0x7);
1562 e
.version
= eversion_t(1, 5);
1563 olog
.log
.push_back(e
);
1565 e
.soid
.set_hash(0x8);
1566 e
.version
= eversion_t(1, 6);
1567 olog
.log
.push_back(e
);
1569 e
.soid
.set_hash(0x9); // should not be added to missing, create
1570 e
.op
= pg_log_entry_t::MODIFY
;
1571 e
.version
= eversion_t(1, 7);
1572 olog
.log
.push_back(e
);
1574 e
.soid
= divergent_object
; // should be added to missing at 1,2
1575 e
.op
= pg_log_entry_t::MODIFY
;
1576 e
.version
= eversion_t(1, 8);
1577 e
.prior_version
= eversion_t(1, 2);
1578 olog
.log
.push_back(e
);
1579 olog
.head
= e
.version
;
1581 oinfo
.last_update
= olog
.head
;
1582 oinfo
.last_complete
= olog
.head
;
1585 EXPECT_FALSE(omissing
.have_missing());
1586 EXPECT_EQ(olog
.head
, oinfo
.last_update
);
1587 EXPECT_EQ(olog
.head
, oinfo
.last_complete
);
1589 missing
.may_include_deletes
= false;
1590 proc_replica_log(oinfo
, olog
, omissing
, from
);
1592 EXPECT_TRUE(omissing
.have_missing());
1593 EXPECT_TRUE(omissing
.is_missing(divergent_object
));
1594 EXPECT_EQ(eversion_t(1, 2), omissing
.get_items().at(divergent_object
).need
);
1595 EXPECT_EQ(eversion_t(1, 6), oinfo
.last_update
);
1596 EXPECT_EQ(eversion_t(1, 1), oinfo
.last_complete
);
1599 /* +--------------------------+
1601 +--------+-------+---------+
1603 |version | hash | version |
1605 tail > (1,1) | x9 | (1,1) < tail
1608 | (1,2) | x3 | (1,2) |
1611 head > (1,3) | x9 | |
1614 | | x9 | (2,3) < head
1617 +--------+-------+---------+
1619 The log entry (1,3) deletes the object x9 and the olog entry
1620 (2,3) also deletes it : do nothing.
1628 pg_missing_t omissing
;
1631 eversion_t
last_update(1, 2);
1632 hobject_t divergent_object
;
1633 divergent_object
.set_hash(0x9);
1637 e
.mark_unrollbackable();
1639 e
.version
= eversion_t(1, 1);
1640 e
.soid
= divergent_object
;
1641 log
.tail
= e
.version
;
1642 log
.log
.push_back(e
);
1643 e
.version
= last_update
;
1644 e
.soid
.set_hash(0x3);
1645 log
.log
.push_back(e
);
1646 e
.version
= eversion_t(2, 3);
1647 e
.prior_version
= eversion_t(1, 1);
1648 e
.soid
= divergent_object
;
1649 e
.op
= pg_log_entry_t::DELETE
;
1650 log
.log
.push_back(e
);
1651 log
.head
= e
.version
;
1654 e
.version
= eversion_t(1, 1);
1655 e
.soid
= divergent_object
;
1656 olog
.tail
= e
.version
;
1657 olog
.log
.push_back(e
);
1658 e
.version
= last_update
;
1659 e
.soid
.set_hash(0x3);
1660 olog
.log
.push_back(e
);
1661 e
.version
= eversion_t(1, 3);
1662 e
.prior_version
= eversion_t(1, 1);
1663 e
.soid
= divergent_object
;
1664 e
.op
= pg_log_entry_t::DELETE
;
1665 olog
.log
.push_back(e
);
1666 olog
.head
= e
.version
;
1668 oinfo
.last_update
= olog
.head
;
1669 oinfo
.last_complete
= olog
.head
;
1672 EXPECT_FALSE(omissing
.have_missing());
1673 EXPECT_EQ(olog
.head
, oinfo
.last_update
);
1674 EXPECT_EQ(olog
.head
, oinfo
.last_complete
);
1676 missing
.may_include_deletes
= false;
1677 proc_replica_log(oinfo
, olog
, omissing
, from
);
1679 EXPECT_TRUE(omissing
.have_missing());
1680 EXPECT_TRUE(omissing
.is_missing(divergent_object
));
1681 EXPECT_EQ(omissing
.get_items().at(divergent_object
).have
, eversion_t(0, 0));
1682 EXPECT_EQ(omissing
.get_items().at(divergent_object
).need
, eversion_t(1, 1));
1683 EXPECT_EQ(last_update
, oinfo
.last_update
);
1686 /* +--------------------------+
1688 +--------+-------+---------+
1690 |version | hash | version |
1692 tail > (1,1) | x9 | (1,1) < tail
1695 | (1,2) | x3 | (1,2) |
1698 head > (1,3) | x9 | |
1701 | | x9 | (2,3) < head
1704 +--------+-------+---------+
1706 The log entry (1,3) deletes the object x9 but the olog entry
1707 (2,3) modifies it : remove it from omissing.
1715 pg_missing_t omissing
;
1718 eversion_t
last_update(1, 2);
1719 hobject_t divergent_object
;
1723 e
.mark_unrollbackable();
1725 e
.version
= eversion_t(1, 1);
1726 e
.soid
= divergent_object
;
1727 log
.tail
= e
.version
;
1728 log
.log
.push_back(e
);
1729 e
.version
= last_update
;
1730 e
.soid
.set_hash(0x3);
1731 log
.log
.push_back(e
);
1732 e
.version
= eversion_t(2, 3);
1733 e
.prior_version
= eversion_t(1, 1);
1734 e
.soid
= divergent_object
;
1735 e
.op
= pg_log_entry_t::DELETE
;
1736 log
.log
.push_back(e
);
1737 log
.head
= e
.version
;
1740 e
.version
= eversion_t(1, 1);
1741 e
.soid
= divergent_object
;
1742 olog
.tail
= e
.version
;
1743 olog
.log
.push_back(e
);
1744 e
.version
= last_update
;
1745 e
.soid
.set_hash(0x3);
1746 olog
.log
.push_back(e
);
1747 e
.version
= eversion_t(1, 3);
1748 e
.prior_version
= eversion_t(1, 1);
1749 e
.soid
= divergent_object
;
1750 divergent_object
= e
.soid
;
1751 omissing
.add(divergent_object
, e
.version
, eversion_t(), false);
1752 e
.op
= pg_log_entry_t::MODIFY
;
1753 olog
.log
.push_back(e
);
1754 olog
.head
= e
.version
;
1756 oinfo
.last_update
= olog
.head
;
1757 oinfo
.last_complete
= olog
.head
;
1760 EXPECT_TRUE(omissing
.have_missing());
1761 EXPECT_TRUE(omissing
.is_missing(divergent_object
));
1762 EXPECT_EQ(eversion_t(1, 3), omissing
.get_items().at(divergent_object
).need
);
1763 EXPECT_EQ(olog
.head
, oinfo
.last_update
);
1764 EXPECT_EQ(olog
.head
, oinfo
.last_complete
);
1766 missing
.may_include_deletes
= false;
1767 proc_replica_log(oinfo
, olog
, omissing
, from
);
1769 EXPECT_TRUE(omissing
.have_missing());
1770 EXPECT_TRUE(omissing
.is_missing(divergent_object
));
1771 EXPECT_EQ(omissing
.get_items().at(divergent_object
).have
, eversion_t(0, 0));
1772 EXPECT_EQ(omissing
.get_items().at(divergent_object
).need
, eversion_t(1, 1));
1773 EXPECT_EQ(last_update
, oinfo
.last_update
);
1776 /* +--------------------------+
1778 +--------+-------+---------+
1780 |version | hash | version |
1782 tail > (1,1) | x9 | (1,1) < tail
1785 | (1,2) | x3 | (1,2) |
1788 | | x9 | (1,3) < head
1791 head > (2,3) | x9 | |
1794 +--------+-------+---------+
1796 The log entry (2,3) deletes the object x9 but the olog entry
1797 (1,3) modifies it : proc_replica_log should adjust missing to
1798 1,1 for that object until add_next_event in PG::activate processes
1806 pg_missing_t omissing
;
1809 eversion_t
last_update(1, 2);
1810 hobject_t divergent_object
;
1811 eversion_t
new_version(2, 3);
1812 eversion_t
divergent_version(1, 3);
1816 e
.mark_unrollbackable();
1818 e
.version
= eversion_t(1, 1);
1819 e
.soid
.set_hash(0x9);
1820 log
.tail
= e
.version
;
1821 log
.log
.push_back(e
);
1822 e
.version
= last_update
;
1823 e
.soid
.set_hash(0x3);
1824 log
.log
.push_back(e
);
1825 e
.version
= new_version
;
1826 e
.prior_version
= eversion_t(1, 1);
1827 e
.soid
.set_hash(0x9);
1828 e
.op
= pg_log_entry_t::DELETE
;
1829 log
.log
.push_back(e
);
1830 log
.head
= e
.version
;
1833 e
.op
= pg_log_entry_t::MODIFY
;
1834 e
.version
= eversion_t(1, 1);
1835 e
.soid
.set_hash(0x9);
1836 olog
.tail
= e
.version
;
1837 olog
.log
.push_back(e
);
1838 e
.version
= last_update
;
1839 e
.soid
.set_hash(0x3);
1840 olog
.log
.push_back(e
);
1841 e
.version
= divergent_version
;
1842 e
.prior_version
= eversion_t(1, 1);
1843 e
.soid
.set_hash(0x9);
1844 divergent_object
= e
.soid
;
1845 omissing
.add(divergent_object
, e
.version
, eversion_t(), false);
1846 e
.op
= pg_log_entry_t::MODIFY
;
1847 olog
.log
.push_back(e
);
1848 olog
.head
= e
.version
;
1850 oinfo
.last_update
= olog
.head
;
1851 oinfo
.last_complete
= olog
.head
;
1854 EXPECT_TRUE(omissing
.have_missing());
1855 EXPECT_TRUE(omissing
.is_missing(divergent_object
));
1856 EXPECT_EQ(divergent_version
, omissing
.get_items().at(divergent_object
).need
);
1857 EXPECT_EQ(olog
.head
, oinfo
.last_update
);
1858 EXPECT_EQ(olog
.head
, oinfo
.last_complete
);
1860 missing
.may_include_deletes
= false;
1861 proc_replica_log(oinfo
, olog
, omissing
, from
);
1863 EXPECT_TRUE(omissing
.have_missing());
1864 EXPECT_TRUE(omissing
.get_items().begin()->second
.need
== eversion_t(1, 1));
1865 EXPECT_EQ(last_update
, oinfo
.last_update
);
1866 EXPECT_EQ(eversion_t(0, 0), oinfo
.last_complete
);
1871 TEST_F(PGLogTest
, merge_log_1
) {
1873 t
.base
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1875 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1877 t
.final
.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1879 t
.toremove
.insert(mk_obj(1));
1885 TEST_F(PGLogTest
, merge_log_2
) {
1887 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1889 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1890 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1892 t
.torollback
.insert(
1893 t
.torollback
.begin(), t
.div
.rbegin(), t
.div
.rend());
1899 TEST_F(PGLogTest
, merge_log_3
) {
1901 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1903 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1904 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1906 t
.final
.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1908 t
.toremove
.insert(mk_obj(1));
1914 TEST_F(PGLogTest
, merge_log_4
) {
1916 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1918 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1919 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1921 t
.init
.add(mk_obj(1), mk_evt(10, 102), mk_evt(0, 0), false);
1922 t
.final
.add(mk_obj(1), mk_evt(10, 100), mk_evt(0, 0), false);
1928 TEST_F(PGLogTest
, merge_log_5
) {
1930 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1932 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
1933 t
.div
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 102), mk_evt(10, 101)));
1935 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1937 t
.final
.add(mk_obj(1), mk_evt(11, 101), mk_evt(0, 0), false);
1939 t
.toremove
.insert(mk_obj(1));
1945 TEST_F(PGLogTest
, merge_log_6
) {
1947 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1949 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1951 t
.final
.add(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100), false);
1957 TEST_F(PGLogTest
, merge_log_7
) {
1959 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1961 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1963 t
.init
.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
1964 t
.final
.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), false);
1970 TEST_F(PGLogTest
, merge_log_8
) {
1972 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1974 t
.auth
.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1976 t
.init
.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
1977 t
.final
.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), true);
1983 TEST_F(PGLogTest
, merge_log_9
) {
1985 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
1987 t
.auth
.push_back(mk_ple_dt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
1989 t
.init
.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
1990 t
.toremove
.insert(mk_obj(1));
1991 t
.deletes_during_peering
= true;
1997 TEST_F(PGLogTest
, merge_log_10
) {
1999 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
2001 t
.auth
.push_back(mk_ple_ldt(mk_obj(1), mk_evt(11, 101), mk_evt(10, 100)));
2003 t
.init
.add(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80), false);
2004 t
.final
.add(mk_obj(1), mk_evt(11, 101), mk_evt(8, 80), true);
2010 TEST_F(PGLogTest
, merge_log_prior_version_have
) {
2012 t
.base
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 80)));
2014 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100)));
2016 t
.init
.add(mk_obj(1), mk_evt(10, 101), mk_evt(10, 100), false);
2022 TEST_F(PGLogTest
, merge_log_split_missing_entries_at_head
) {
2024 t
.auth
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2025 t
.auth
.push_back(mk_ple_mod_rb(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2027 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(8, 70), mk_evt(8, 65)));
2030 t
.set_div_bounds(mk_evt(9, 79), mk_evt(8, 69));
2031 t
.set_auth_bounds(mk_evt(15, 160), mk_evt(9, 77));
2032 t
.final
.add(mk_obj(1), mk_evt(15, 150), mk_evt(8, 70), false);
2036 TEST_F(PGLogTest
, olog_tail_gt_log_tail_split
) {
2038 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2039 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2040 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 155), mk_evt(15, 150)));
2043 t
.set_div_bounds(mk_evt(15, 153), mk_evt(15, 151));
2044 t
.set_auth_bounds(mk_evt(15, 156), mk_evt(10, 99));
2045 t
.final
.add(mk_obj(1), mk_evt(15, 155), mk_evt(15, 150), false);
2049 TEST_F(PGLogTest
, olog_tail_gt_log_tail_split2
) {
2051 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2052 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 150), mk_evt(10, 100)));
2053 t
.auth
.push_back(mk_ple_mod(mk_obj(1), mk_evt(16, 155), mk_evt(15, 150)));
2054 t
.div
.push_back(mk_ple_mod(mk_obj(1), mk_evt(15, 153), mk_evt(15, 150)));
2057 t
.set_div_bounds(mk_evt(15, 153), mk_evt(15, 151));
2058 t
.set_auth_bounds(mk_evt(16, 156), mk_evt(10, 99));
2059 t
.final
.add(mk_obj(1), mk_evt(16, 155), mk_evt(0, 0), false);
2060 t
.toremove
.insert(mk_obj(1));
2064 TEST_F(PGLogTest
, filter_log_1
) {
2070 int64_t pool_id
= 1;
2073 int pg_num
= max_osd
<< bits
;
2074 int num_objects
= 1000;
2075 int num_internal
= 10;
2077 // Set up splitting map
2078 OSDMap
*osdmap
= new OSDMap
;
2080 test_uuid
.generate_random();
2081 osdmap
->build_simple_with_pool(g_ceph_context
, epoch
, test_uuid
, max_osd
, bits
, bits
);
2082 osdmap
->set_state(osd_id
, CEPH_OSD_EXISTS
);
2084 const string
hit_set_namespace("internal");
2088 e
.mark_unrollbackable();
2089 e
.op
= pg_log_entry_t::MODIFY
;
2090 e
.soid
.pool
= pool_id
;
2094 for (i
= 1; i
<= num_objects
; ++i
) {
2095 e
.version
= eversion_t(epoch
, i
);
2096 // Use this to generate random file names
2097 uuid_name
.generate_random();
2100 e
.soid
.oid
.name
= name
.str();
2101 // First has no namespace
2103 // num_internal have the internal namspace
2104 if (i
<= num_internal
+ 1) {
2105 e
.soid
.nspace
= hit_set_namespace
;
2106 } else { // rest have different namespaces
2109 e
.soid
.nspace
= ns
.str();
2112 log
.log
.push_back(e
);
2114 log
.tail
= e
.version
;
2116 log
.head
= e
.version
;
2120 spg_t
pgid(pg_t(2, pool_id
), shard_id_t::NO_SHARD
);
2122 // See if we created the right number of entries
2123 int total
= log
.log
.size();
2124 ASSERT_EQ(total
, num_objects
);
2126 // Some should be removed
2128 pg_log_t filtered
, reject
;
2129 pg_log_t::filter_log(
2130 pgid
, *osdmap
, hit_set_namespace
, log
, filtered
, reject
);
2131 log
= IndexedLog(filtered
);
2133 EXPECT_LE(log
.log
.size(), (size_t)total
);
2135 // If we filter a second time, there should be the same total
2136 total
= log
.log
.size();
2138 pg_log_t filtered
, reject
;
2139 pg_log_t::filter_log(
2140 pgid
, *osdmap
, hit_set_namespace
, log
, filtered
, reject
);
2141 log
= IndexedLog(filtered
);
2143 EXPECT_EQ(log
.log
.size(), (size_t)total
);
2145 // Increase pg_num as if there would be a split
2146 int new_pg_num
= pg_num
* 16;
2147 OSDMap::Incremental
inc(epoch
+ 1);
2148 inc
.fsid
= test_uuid
;
2149 const pg_pool_t
*pool
= osdmap
->get_pg_pool(pool_id
);
2152 newpool
.set_pg_num(new_pg_num
);
2153 newpool
.set_pgp_num(new_pg_num
);
2154 inc
.new_pools
[pool_id
] = newpool
;
2155 int ret
= osdmap
->apply_incremental(inc
);
2158 // We should have fewer entries after a filter
2160 pg_log_t filtered
, reject
;
2161 pg_log_t::filter_log(
2162 pgid
, *osdmap
, hit_set_namespace
, log
, filtered
, reject
);
2163 log
= IndexedLog(filtered
);
2165 EXPECT_LE(log
.log
.size(), (size_t)total
);
2167 // Make sure all internal entries are retained
2169 for (list
<pg_log_entry_t
>::iterator i
= log
.log
.begin();
2170 i
!= log
.log
.end(); ++i
) {
2171 if (i
->soid
.nspace
== hit_set_namespace
) count
++;
2173 EXPECT_EQ(count
, num_internal
);
2177 TEST_F(PGLogTest
, get_request
) {
2180 // make sure writes, deletes, and errors are found
2181 vector
<pg_log_entry_t
> entries
;
2182 hobject_t
oid(object_t("objname"), "key", 123, 456, 0, "");
2184 pg_log_entry_t(pg_log_entry_t::ERROR
, oid
, eversion_t(6,2), eversion_t(3,4),
2185 1, osd_reqid_t(entity_name_t::CLIENT(777), 8, 1),
2186 utime_t(0,1), -ENOENT
));
2188 pg_log_entry_t(pg_log_entry_t::MODIFY
, oid
, eversion_t(6,3), eversion_t(3,4),
2189 2, osd_reqid_t(entity_name_t::CLIENT(777), 8, 2),
2192 pg_log_entry_t(pg_log_entry_t::DELETE
, oid
, eversion_t(7,4), eversion_t(7,4),
2193 3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 3),
2196 pg_log_entry_t(pg_log_entry_t::ERROR
, oid
, eversion_t(7,5), eversion_t(7,4),
2197 3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 4),
2198 utime_t(20,1), -ENOENT
));
2200 for (auto &entry
: entries
) {
2204 for (auto &entry
: entries
) {
2205 eversion_t replay_version
;
2206 version_t user_version
;
2207 int return_code
= 0;
2208 vector
<pg_log_op_return_item_t
> op_returns
;
2209 bool got
= log
.get_request(
2210 entry
.reqid
, &replay_version
, &user_version
, &return_code
, &op_returns
);
2212 EXPECT_EQ(entry
.return_code
, return_code
);
2213 EXPECT_EQ(entry
.version
, replay_version
);
2214 EXPECT_EQ(entry
.user_version
, user_version
);
2218 TEST_F(PGLogTest
, ErrorNotIndexedByObject
) {
2221 // make sure writes, deletes, and errors are found
2222 hobject_t
oid(object_t("objname"), "key", 123, 456, 0, "");
2224 pg_log_entry_t(pg_log_entry_t::ERROR
, oid
, eversion_t(6,2), eversion_t(3,4),
2225 1, osd_reqid_t(entity_name_t::CLIENT(777), 8, 1),
2226 utime_t(0,1), -ENOENT
));
2228 EXPECT_FALSE(log
.logged_object(oid
));
2230 pg_log_entry_t
modify(pg_log_entry_t::MODIFY
, oid
, eversion_t(6,3),
2232 osd_reqid_t(entity_name_t::CLIENT(777), 8, 2),
2236 EXPECT_TRUE(log
.logged_object(oid
));
2237 pg_log_entry_t
*entry
= log
.objects
[oid
];
2238 EXPECT_EQ(modify
.op
, entry
->op
);
2239 EXPECT_EQ(modify
.version
, entry
->version
);
2240 EXPECT_EQ(modify
.prior_version
, entry
->prior_version
);
2241 EXPECT_EQ(modify
.user_version
, entry
->user_version
);
2242 EXPECT_EQ(modify
.reqid
, entry
->reqid
);
2244 pg_log_entry_t
del(pg_log_entry_t::DELETE
, oid
, eversion_t(7,4),
2246 osd_reqid_t(entity_name_t::CLIENT(777), 8, 3),
2250 EXPECT_TRUE(log
.logged_object(oid
));
2251 entry
= log
.objects
[oid
];
2252 EXPECT_EQ(del
.op
, entry
->op
);
2253 EXPECT_EQ(del
.version
, entry
->version
);
2254 EXPECT_EQ(del
.prior_version
, entry
->prior_version
);
2255 EXPECT_EQ(del
.user_version
, entry
->user_version
);
2256 EXPECT_EQ(del
.reqid
, entry
->reqid
);
2259 pg_log_entry_t(pg_log_entry_t::ERROR
, oid
, eversion_t(7,5), eversion_t(7,4),
2260 3, osd_reqid_t(entity_name_t::CLIENT(777), 8, 4),
2261 utime_t(20,1), -ENOENT
));
2263 EXPECT_TRUE(log
.logged_object(oid
));
2264 entry
= log
.objects
[oid
];
2265 EXPECT_EQ(del
.op
, entry
->op
);
2266 EXPECT_EQ(del
.version
, entry
->version
);
2267 EXPECT_EQ(del
.prior_version
, entry
->prior_version
);
2268 EXPECT_EQ(del
.user_version
, entry
->user_version
);
2269 EXPECT_EQ(del
.reqid
, entry
->reqid
);
2272 TEST_F(PGLogTest
, split_into_preserves_may_include_deletes
) {
2276 may_include_deletes_in_missing_dirty
= false;
2277 missing
.may_include_deletes
= true;
2278 PGLog
child_log(cct
);
2280 split_into(child_pg
, 6, &child_log
);
2281 ASSERT_TRUE(child_log
.get_missing().may_include_deletes
);
2282 ASSERT_TRUE(child_log
.get_may_include_deletes_in_missing_dirty());
2286 may_include_deletes_in_missing_dirty
= false;
2287 missing
.may_include_deletes
= false;
2288 PGLog
child_log(cct
);
2290 split_into(child_pg
, 6, &child_log
);
2291 ASSERT_FALSE(child_log
.get_missing().may_include_deletes
);
2292 ASSERT_FALSE(child_log
.get_may_include_deletes_in_missing_dirty());
2296 class PGLogTestRebuildMissing
: public PGLogTest
, public StoreTestFixture
{
2298 PGLogTestRebuildMissing() : PGLogTest(), StoreTestFixture("memstore") {}
2299 void SetUp() override
{
2300 StoreTestFixture::SetUp();
2301 ObjectStore::Transaction t
;
2302 test_coll
= coll_t(spg_t(pg_t(1, 1)));
2303 ch
= store
->create_new_collection(test_coll
);
2304 t
.create_collection(test_coll
, 0);
2305 store
->queue_transaction(ch
, std::move(t
));
2306 existing_oid
= mk_obj(0);
2307 nonexistent_oid
= mk_obj(1);
2308 ghobject_t
existing_ghobj(existing_oid
);
2309 object_info_t existing_info
;
2310 existing_info
.version
= eversion_t(6, 2);
2312 encode(existing_info
, enc_oi
, 0);
2313 ObjectStore::Transaction t2
;
2314 t2
.touch(test_coll
, ghobject_t(existing_oid
));
2315 t2
.setattr(test_coll
, ghobject_t(existing_oid
), OI_ATTR
, enc_oi
);
2316 ASSERT_EQ(0, store
->queue_transaction(ch
, std::move(t2
)));
2317 info
.last_backfill
= hobject_t::get_max();
2318 info
.last_complete
= eversion_t();
2321 void TearDown() override
{
2323 missing
.may_include_deletes
= false;
2324 StoreTestFixture::TearDown();
2329 hobject_t existing_oid
, nonexistent_oid
;
2331 void run_rebuild_missing_test(const map
<hobject_t
, pg_missing_item
> &expected_missing_items
) {
2332 rebuild_missing_set_with_deletes(store
.get(), ch
, info
);
2333 ASSERT_EQ(expected_missing_items
, missing
.get_items());
2337 TEST_F(PGLogTestRebuildMissing
, EmptyLog
) {
2338 missing
.add(existing_oid
, mk_evt(6, 2), mk_evt(6, 3), false);
2339 missing
.add(nonexistent_oid
, mk_evt(7, 4), mk_evt(0, 0), false);
2340 map
<hobject_t
, pg_missing_item
> orig_missing
= missing
.get_items();
2341 run_rebuild_missing_test(orig_missing
);
2344 TEST_F(PGLogTestRebuildMissing
, SameVersionMod
) {
2345 missing
.add(existing_oid
, mk_evt(6, 2), mk_evt(6, 1), false);
2346 log
.add(mk_ple_mod(existing_oid
, mk_evt(6, 2), mk_evt(6, 1)));
2347 map
<hobject_t
, pg_missing_item
> empty_missing
;
2348 run_rebuild_missing_test(empty_missing
);
2351 TEST_F(PGLogTestRebuildMissing
, DelExisting
) {
2352 missing
.add(existing_oid
, mk_evt(6, 3), mk_evt(6, 2), false);
2353 log
.add(mk_ple_dt(existing_oid
, mk_evt(7, 5), mk_evt(7, 4)));
2354 map
<hobject_t
, pg_missing_item
> expected
;
2355 expected
[existing_oid
] = pg_missing_item(mk_evt(7, 5), mk_evt(6, 2), true);
2356 run_rebuild_missing_test(expected
);
2359 TEST_F(PGLogTestRebuildMissing
, DelNonexistent
) {
2360 log
.add(mk_ple_dt(nonexistent_oid
, mk_evt(7, 5), mk_evt(7, 4)));
2361 map
<hobject_t
, pg_missing_item
> expected
;
2362 expected
[nonexistent_oid
] = pg_missing_item(mk_evt(7, 5), mk_evt(0, 0), true);
2363 run_rebuild_missing_test(expected
);
2366 TEST_F(PGLogTestRebuildMissing
, MissingNotInLog
) {
2367 missing
.add(mk_obj(10), mk_evt(8, 12), mk_evt(8, 10), false);
2368 log
.add(mk_ple_dt(nonexistent_oid
, mk_evt(7, 5), mk_evt(7, 4)));
2369 map
<hobject_t
, pg_missing_item
> expected
;
2370 expected
[nonexistent_oid
] = pg_missing_item(mk_evt(7, 5), mk_evt(0, 0), true);
2371 expected
[mk_obj(10)] = pg_missing_item(mk_evt(8, 12), mk_evt(8, 10), false);
2372 run_rebuild_missing_test(expected
);
2376 class PGLogMergeDupsTest
: protected PGLog
, public StoreTestFixture
{
2380 PGLogMergeDupsTest() : PGLog(g_ceph_context
), StoreTestFixture("memstore") { }
2382 void SetUp() override
{
2383 StoreTestFixture::SetUp();
2384 ObjectStore::Transaction t
;
2385 test_coll
= coll_t(spg_t(pg_t(1, 1)));
2386 auto ch
= store
->create_new_collection(test_coll
);
2387 t
.create_collection(test_coll
, 0);
2388 store
->queue_transaction(ch
, std::move(t
));
2391 void TearDown() override
{
2392 test_disk_roundtrip();
2394 StoreTestFixture::TearDown();
2397 static pg_log_dup_t
create_dup_entry(uint a
, uint b
) {
2398 // make each dup_entry unique by using different client id's
2399 static uint client_id
= 777;
2400 return pg_log_dup_t(eversion_t(a
, b
),
2402 osd_reqid_t(entity_name_t::CLIENT(client_id
++), 8, 1),
2406 static std::vector
<pg_log_dup_t
> example_dups_1() {
2407 std::vector
<pg_log_dup_t
> result
= {
2408 create_dup_entry(10, 11),
2409 create_dup_entry(10, 12),
2410 create_dup_entry(11, 1),
2411 create_dup_entry(12, 3),
2412 create_dup_entry(13, 99)
2417 static std::vector
<pg_log_dup_t
> example_dups_2() {
2418 std::vector
<pg_log_dup_t
> result
= {
2419 create_dup_entry(12, 3),
2420 create_dup_entry(13, 99),
2421 create_dup_entry(15, 11),
2422 create_dup_entry(16, 14),
2423 create_dup_entry(16, 32)
2428 void add_dups(uint a
, uint b
) {
2429 log
.dups
.push_back(create_dup_entry(a
, b
));
2430 write_from_dups
= std::min(write_from_dups
, log
.dups
.back().version
);
2433 void add_dups(const std::vector
<pg_log_dup_t
>& l
) {
2435 log
.dups
.push_back(i
);
2436 write_from_dups
= std::min(write_from_dups
, log
.dups
.back().version
);
2440 static void add_dups(IndexedLog
& log
, const std::vector
<pg_log_dup_t
>& dups
) {
2441 for (auto& i
: dups
) {
2442 log
.dups
.push_back(i
);
2446 void check_order() {
2447 eversion_t
prev(0, 0);
2449 for (auto& i
: log
.dups
) {
2450 EXPECT_LT(prev
, i
.version
) << "verify versions monotonically increase";
2455 void check_index() {
2456 EXPECT_EQ(log
.dups
.size(), log
.dup_index
.size());
2457 for (auto& i
: log
.dups
) {
2458 EXPECT_EQ(1u, log
.dup_index
.count(i
.reqid
));
2462 void test_disk_roundtrip() {
2463 ObjectStore::Transaction t
;
2467 ghobject_t
log_oid(hoid
);
2468 map
<string
, bufferlist
> km
;
2469 write_log_and_missing(t
, &km
, test_coll
, log_oid
, false);
2471 t
.omap_setkeys(test_coll
, log_oid
, km
);
2473 auto ch
= store
->open_collection(test_coll
);
2474 ASSERT_EQ(0, store
->queue_transaction(ch
, std::move(t
)));
2476 auto orig_dups
= log
.dups
;
2479 read_log_and_missing(store
.get(), ch
, log_oid
,
2480 pg_info_t(), err
, false);
2481 ASSERT_EQ(orig_dups
.size(), log
.dups
.size());
2482 ASSERT_EQ(orig_dups
, log
.dups
);
2483 auto dups_it
= log
.dups
.begin();
2484 for (auto orig_dup
: orig_dups
) {
2485 ASSERT_EQ(orig_dup
, *dups_it
);
2493 TEST_F(PGLogMergeDupsTest
, OtherEmpty
) {
2494 log
.tail
= eversion_t(14, 5);
2498 add_dups(example_dups_1());
2501 bool changed
= merge_log_dups(olog
);
2503 EXPECT_FALSE(changed
);
2504 EXPECT_EQ(5u, log
.dups
.size());
2506 if (5 == log
.dups
.size()) {
2507 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2508 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2509 EXPECT_EQ(13u, log
.dups
.back().version
.epoch
);
2510 EXPECT_EQ(99u, log
.dups
.back().version
.version
);
2517 TEST_F(PGLogMergeDupsTest
, AmEmpty
) {
2518 log
.tail
= eversion_t(14, 5);
2523 add_dups(olog
, example_dups_1());
2525 bool changed
= merge_log_dups(olog
);
2527 EXPECT_TRUE(changed
);
2528 EXPECT_EQ(5u, log
.dups
.size());
2530 if (5 == log
.dups
.size()) {
2531 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2532 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2534 EXPECT_EQ(13u, log
.dups
.back().version
.epoch
);
2535 EXPECT_EQ(99u, log
.dups
.back().version
.version
);
2542 TEST_F(PGLogMergeDupsTest
, AmEmptyOverlap
) {
2543 log
.tail
= eversion_t(12, 3);
2548 add_dups(olog
, example_dups_1());
2550 bool changed
= merge_log_dups(olog
);
2552 EXPECT_TRUE(changed
);
2553 EXPECT_EQ(4u, log
.dups
.size());
2555 if (4 == log
.dups
.size()) {
2556 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2557 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2559 EXPECT_EQ(12u, log
.dups
.back().version
.epoch
);
2560 EXPECT_EQ(3u, log
.dups
.back().version
.version
);
2567 TEST_F(PGLogMergeDupsTest
, Same
) {
2568 log
.tail
= eversion_t(14, 1);
2572 add_dups(example_dups_1());
2574 add_dups(olog
, example_dups_1());
2576 bool changed
= merge_log_dups(olog
);
2578 EXPECT_FALSE(changed
);
2579 EXPECT_EQ(5u, log
.dups
.size());
2581 if (5 == log
.dups
.size()) {
2582 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2583 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2585 EXPECT_EQ(13u, log
.dups
.back().version
.epoch
);
2586 EXPECT_EQ(99u, log
.dups
.back().version
.version
);
2594 TEST_F(PGLogMergeDupsTest
, Later
) {
2595 log
.tail
= eversion_t(16, 14);
2599 add_dups(example_dups_1());
2601 add_dups(olog
, example_dups_2());
2603 bool changed
= merge_log_dups(olog
);
2605 EXPECT_TRUE(changed
);
2606 EXPECT_EQ(7u, log
.dups
.size());
2608 if (7 == log
.dups
.size()) {
2609 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2610 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2612 EXPECT_EQ(16u, log
.dups
.back().version
.epoch
);
2613 EXPECT_EQ(14u, log
.dups
.back().version
.version
);
2621 TEST_F(PGLogMergeDupsTest
, Earlier
) {
2622 log
.tail
= eversion_t(17, 2);
2626 add_dups(example_dups_2());
2628 add_dups(olog
, example_dups_1());
2630 bool changed
= merge_log_dups(olog
);
2632 EXPECT_TRUE(changed
);
2633 EXPECT_EQ(8u, log
.dups
.size());
2635 if (6 == log
.dups
.size()) {
2636 EXPECT_EQ(10u, log
.dups
.front().version
.epoch
);
2637 EXPECT_EQ(11u, log
.dups
.front().version
.version
);
2639 EXPECT_EQ(16u, log
.dups
.back().version
.epoch
);
2640 EXPECT_EQ(32u, log
.dups
.back().version
.version
);
2648 TEST_F(PGLogMergeDupsTest
, Superset
) {
2649 log
.tail
= eversion_t(17, 2);
2653 add_dups(example_dups_1());
2656 olog
.dups
.push_back(create_dup_entry(9, 5));
2657 olog
.dups
.push_back(create_dup_entry(15, 11));
2659 bool changed
= merge_log_dups(olog
);
2661 EXPECT_TRUE(changed
);
2662 EXPECT_EQ(7u, log
.dups
.size());
2664 if (7 == log
.dups
.size()) {
2665 EXPECT_EQ(9u, log
.dups
.front().version
.epoch
);
2666 EXPECT_EQ(5u, log
.dups
.front().version
.version
);
2668 EXPECT_EQ(15u, log
.dups
.back().version
.epoch
);
2669 EXPECT_EQ(11u, log
.dups
.back().version
.version
);
2677 struct PGLogTrimTest
:
2678 public ::testing::Test
,
2679 public PGLogTestBase
,
2680 public PGLog::IndexedLog
2682 CephContext
*cct
= g_ceph_context
;
2684 using ::testing::Test::SetUp
;
2685 void SetUp(unsigned dup_track
) {
2686 constexpr size_t size
= 10;
2688 char dup_track_s
[size
];
2690 snprintf(dup_track_s
, size
, "%u", dup_track
);
2692 cct
->_conf
.set_val_or_die("osd_pg_log_dups_tracked", dup_track_s
);
2694 }; // struct PGLogTrimTest
2697 TEST_F(PGLogTrimTest
, TestMakingCephContext
)
2701 EXPECT_EQ(5u, cct
->_conf
->osd_pg_log_dups_tracked
);
2705 TEST_F(PGLogTrimTest
, TestPartialTrim
)
2708 PGLog::IndexedLog log
;
2709 log
.head
= mk_evt(24, 0);
2710 log
.skip_can_rollback_to_to_head();
2711 log
.head
= mk_evt(9, 0);
2713 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2714 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2715 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2716 log
.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2717 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2718 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2720 std::set
<eversion_t
> trimmed
;
2721 std::set
<std::string
> trimmed_dups
;
2722 eversion_t write_from_dups
= eversion_t::max();
2724 log
.trim(cct
, mk_evt(19, 157), &trimmed
, &trimmed_dups
, &write_from_dups
);
2726 EXPECT_EQ(eversion_t(15, 150), write_from_dups
);
2727 EXPECT_EQ(3u, log
.log
.size());
2728 EXPECT_EQ(3u, trimmed
.size());
2729 EXPECT_EQ(2u, log
.dups
.size());
2730 EXPECT_EQ(0u, trimmed_dups
.size());
2734 std::set
<eversion_t
> trimmed2
;
2735 std::set
<std::string
> trimmed_dups2
;
2736 eversion_t write_from_dups2
= eversion_t::max();
2738 log
.trim(cct
, mk_evt(20, 164), &trimmed2
, &trimmed_dups2
, &write_from_dups2
);
2740 EXPECT_EQ(eversion_t(19, 160), write_from_dups2
);
2741 EXPECT_EQ(2u, log
.log
.size());
2742 EXPECT_EQ(1u, trimmed2
.size());
2743 EXPECT_EQ(2u, log
.dups
.size());
2744 EXPECT_EQ(1u, trimmed_dups2
.size());
2748 TEST_F(PGLogTrimTest
, TestTrimNoTrimmed
) {
2750 PGLog::IndexedLog log
;
2751 log
.head
= mk_evt(20, 0);
2752 log
.skip_can_rollback_to_to_head();
2753 log
.head
= mk_evt(9, 0);
2755 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2756 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2757 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2758 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152)));
2759 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2760 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2762 eversion_t write_from_dups
= eversion_t::max();
2764 log
.trim(cct
, mk_evt(19, 157), nullptr, nullptr, &write_from_dups
);
2766 EXPECT_EQ(eversion_t(15, 150), write_from_dups
);
2767 EXPECT_EQ(3u, log
.log
.size());
2768 EXPECT_EQ(2u, log
.dups
.size());
2772 TEST_F(PGLogTrimTest
, TestTrimNoDups
)
2775 PGLog::IndexedLog log
;
2776 log
.head
= mk_evt(20, 0);
2777 log
.skip_can_rollback_to_to_head();
2778 log
.head
= mk_evt(9, 0);
2780 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2781 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2782 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2783 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152)));
2784 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2785 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2787 std::set
<eversion_t
> trimmed
;
2788 std::set
<std::string
> trimmed_dups
;
2789 eversion_t write_from_dups
= eversion_t::max();
2791 log
.trim(cct
, mk_evt(19, 157), &trimmed
, &trimmed_dups
, &write_from_dups
);
2793 EXPECT_EQ(eversion_t::max(), write_from_dups
);
2794 EXPECT_EQ(3u, log
.log
.size());
2795 EXPECT_EQ(3u, trimmed
.size());
2796 EXPECT_EQ(0u, log
.dups
.size());
2797 EXPECT_EQ(0u, trimmed_dups
.size());
2800 TEST_F(PGLogTrimTest
, TestNoTrim
)
2803 PGLog::IndexedLog log
;
2804 log
.head
= mk_evt(24, 0);
2805 log
.skip_can_rollback_to_to_head();
2806 log
.head
= mk_evt(9, 0);
2808 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2809 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2810 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2811 log
.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2812 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2813 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2815 std::set
<eversion_t
> trimmed
;
2816 std::set
<std::string
> trimmed_dups
;
2817 eversion_t write_from_dups
= eversion_t::max();
2819 log
.trim(cct
, mk_evt(9, 99), &trimmed
, &trimmed_dups
, &write_from_dups
);
2821 EXPECT_EQ(eversion_t::max(), write_from_dups
);
2822 EXPECT_EQ(6u, log
.log
.size());
2823 EXPECT_EQ(0u, trimmed
.size());
2824 EXPECT_EQ(0u, log
.dups
.size());
2825 EXPECT_EQ(0u, trimmed_dups
.size());
2828 TEST_F(PGLogTrimTest
, TestTrimAll
)
2831 PGLog::IndexedLog log
;
2832 EXPECT_EQ(0u, log
.dup_index
.size()); // Sanity check
2833 log
.head
= mk_evt(24, 0);
2834 log
.skip_can_rollback_to_to_head();
2835 log
.head
= mk_evt(9, 0);
2837 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70)));
2838 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100)));
2839 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150)));
2840 log
.add(mk_ple_mod(mk_obj(1), mk_evt(19, 160), mk_evt(25, 152)));
2841 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160)));
2842 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166)));
2844 std::set
<eversion_t
> trimmed
;
2845 std::set
<std::string
> trimmed_dups
;
2846 eversion_t write_from_dups
= eversion_t::max();
2848 log
.trim(cct
, mk_evt(22, 180), &trimmed
, &trimmed_dups
, &write_from_dups
);
2850 EXPECT_EQ(eversion_t(15, 150), write_from_dups
);
2851 EXPECT_EQ(0u, log
.log
.size());
2852 EXPECT_EQ(6u, trimmed
.size());
2853 EXPECT_EQ(5u, log
.dups
.size());
2854 EXPECT_EQ(0u, trimmed_dups
.size());
2855 EXPECT_EQ(0u, log
.dup_index
.size()); // dup_index entry should be trimmed
2859 TEST_F(PGLogTrimTest
, TestGetRequest
) {
2861 PGLog::IndexedLog log
;
2862 log
.head
= mk_evt(20, 0);
2863 log
.skip_can_rollback_to_to_head();
2864 log
.head
= mk_evt(9, 0);
2866 entity_name_t client
= entity_name_t::CLIENT(777);
2868 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(8, 70),
2869 osd_reqid_t(client
, 8, 1)));
2870 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 150), mk_evt(10, 100),
2871 osd_reqid_t(client
, 8, 2)));
2872 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 155), mk_evt(15, 150),
2873 osd_reqid_t(client
, 8, 3)));
2874 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 160), mk_evt(25, 152),
2875 osd_reqid_t(client
, 8, 4)));
2876 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 165), mk_evt(26, 160),
2877 osd_reqid_t(client
, 8, 5)));
2878 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 167), mk_evt(31, 166),
2879 osd_reqid_t(client
, 8, 6)));
2881 eversion_t write_from_dups
= eversion_t::max();
2883 log
.trim(cct
, mk_evt(19, 157), nullptr, nullptr, &write_from_dups
);
2885 EXPECT_EQ(eversion_t(15, 150), write_from_dups
);
2886 EXPECT_EQ(3u, log
.log
.size());
2887 EXPECT_EQ(2u, log
.dups
.size());
2890 version_t user_version
;
2892 vector
<pg_log_op_return_item_t
> op_returns
;
2894 osd_reqid_t log_reqid
= osd_reqid_t(client
, 8, 5);
2895 osd_reqid_t dup_reqid
= osd_reqid_t(client
, 8, 3);
2896 osd_reqid_t bad_reqid
= osd_reqid_t(client
, 8, 1);
2900 result
= log
.get_request(log_reqid
, &version
, &user_version
, &return_code
,
2902 EXPECT_EQ(true, result
);
2903 EXPECT_EQ(mk_evt(21, 165), version
);
2905 result
= log
.get_request(dup_reqid
, &version
, &user_version
, &return_code
,
2907 EXPECT_EQ(true, result
);
2908 EXPECT_EQ(mk_evt(15, 155), version
);
2910 result
= log
.get_request(bad_reqid
, &version
, &user_version
, &return_code
,
2912 EXPECT_FALSE(result
);
2915 TEST_F(PGLogTest
, _merge_object_divergent_entries
) {
2917 // Test for issue 20843
2919 hobject_t
hoid(object_t(/*name*/"notify.7"),
2924 /*nspace*/string(""));
2925 mempool::osd_pglog::list
<pg_log_entry_t
> orig_entries
;
2926 orig_entries
.push_back(mk_ple_mod(hoid
, eversion_t(8336, 957), eversion_t(8336, 952)));
2927 orig_entries
.push_back(mk_ple_err(hoid
, eversion_t(8336, 958)));
2928 orig_entries
.push_back(mk_ple_err(hoid
, eversion_t(8336, 959)));
2929 orig_entries
.push_back(mk_ple_mod(hoid
, eversion_t(8336, 960), eversion_t(8336, 957)));
2930 log
.add(mk_ple_mod(hoid
, eversion_t(8973, 1075), eversion_t(8971, 1070)));
2932 /*need*/eversion_t(8971, 1070),
2933 /*have*/eversion_t(8336, 952),
2936 LogHandler rollbacker
;
2937 _merge_object_divergent_entries(log
, hoid
,
2938 orig_entries
, oinfo
,
2939 log
.get_can_rollback_to(),
2940 missing
, &rollbacker
,
2945 // skip leading error entries
2947 hobject_t
hoid(object_t(/*name*/"notify.7"),
2952 /*nspace*/string(""));
2953 mempool::osd_pglog::list
<pg_log_entry_t
> orig_entries
;
2954 orig_entries
.push_back(mk_ple_err(hoid
, eversion_t(8336, 956)));
2955 orig_entries
.push_back(mk_ple_mod(hoid
, eversion_t(8336, 957), eversion_t(8336, 952)));
2956 log
.add(mk_ple_mod(hoid
, eversion_t(8973, 1075), eversion_t(8971, 1070)));
2958 /*need*/eversion_t(8971, 1070),
2959 /*have*/eversion_t(8336, 952),
2962 LogHandler rollbacker
;
2963 _merge_object_divergent_entries(log
, hoid
,
2964 orig_entries
, oinfo
,
2965 log
.get_can_rollback_to(),
2966 missing
, &rollbacker
,
2972 TEST(eversion_t
, get_key_name
) {
2973 eversion_t
a(1234, 5678);
2974 std::string a_key_name
= a
.get_key_name();
2975 EXPECT_EQ("0000001234.00000000000000005678", a_key_name
);
2978 TEST(pg_log_dup_t
, get_key_name
) {
2979 pg_log_dup_t
a(eversion_t(1234, 5678),
2981 osd_reqid_t(entity_name_t::CLIENT(777), 8, 999),
2983 std::string a_key_name
= a
.get_key_name();
2984 EXPECT_EQ("dup_0000001234.00000000000000005678", a_key_name
);
2988 // This tests trim() to make copies of
2989 // 2 log entries (107, 106) and 3 additional for a total
2990 // of 5 dups. Nothing from the original dups is copied.
2991 TEST_F(PGLogTrimTest
, TestTrimDups
) {
2993 PGLog::IndexedLog log
;
2994 log
.head
= mk_evt(21, 107);
2995 log
.skip_can_rollback_to_to_head();
2996 log
.tail
= mk_evt(9, 99);
2997 log
.head
= mk_evt(9, 99);
2999 entity_name_t client
= entity_name_t::CLIENT(777);
3001 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3002 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3004 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3005 osd_reqid_t(client
, 8, 1)));
3006 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3007 osd_reqid_t(client
, 8, 2)));
3008 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3009 osd_reqid_t(client
, 8, 3)));
3010 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3011 osd_reqid_t(client
, 8, 4)));
3012 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3013 osd_reqid_t(client
, 8, 5)));
3014 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3015 osd_reqid_t(client
, 8, 6)));
3016 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3017 osd_reqid_t(client
, 8, 6)));
3018 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3019 osd_reqid_t(client
, 8, 6)));
3021 eversion_t write_from_dups
= eversion_t::max();
3023 log
.trim(cct
, mk_evt(21, 105), nullptr, nullptr, &write_from_dups
);
3025 EXPECT_EQ(eversion_t(20, 103), write_from_dups
) << log
;
3026 EXPECT_EQ(2u, log
.log
.size()) << log
;
3027 EXPECT_EQ(3u, log
.dups
.size()) << log
;
3030 // This tests trim() to make copies of
3031 // 4 log entries (107, 106, 105, 104) and 5 additional for a total
3032 // of 9 dups. Only 1 of 2 existing dups are copied.
3033 TEST_F(PGLogTrimTest
, TestTrimDups2
) {
3035 PGLog::IndexedLog log
;
3036 log
.head
= mk_evt(21, 107);
3037 log
.skip_can_rollback_to_to_head();
3038 log
.tail
= mk_evt(9, 99);
3039 log
.head
= mk_evt(9, 99);
3041 entity_name_t client
= entity_name_t::CLIENT(777);
3043 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3044 mk_evt(9, 98), mk_evt(8, 97), osd_reqid_t(client
, 8, 1))));
3045 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3046 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3048 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3049 osd_reqid_t(client
, 8, 1)));
3050 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3051 osd_reqid_t(client
, 8, 2)));
3052 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3053 osd_reqid_t(client
, 8, 3)));
3054 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3055 osd_reqid_t(client
, 8, 4)));
3056 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3057 osd_reqid_t(client
, 8, 5)));
3058 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3059 osd_reqid_t(client
, 8, 6)));
3060 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3061 osd_reqid_t(client
, 8, 6)));
3062 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3063 osd_reqid_t(client
, 8, 6)));
3065 eversion_t write_from_dups
= eversion_t::max();
3067 log
.trim(cct
, mk_evt(20, 103), nullptr, nullptr, &write_from_dups
);
3069 EXPECT_EQ(eversion_t(10, 100), write_from_dups
) << log
;
3070 EXPECT_EQ(4u, log
.log
.size()) << log
;
3071 EXPECT_EQ(5u, log
.dups
.size()) << log
;
3074 // This tests copy_up_to() to make copies of
3075 // 2 log entries (107, 106) and 3 additional for a total
3076 // of 5 dups. Nothing from the original dups is copied.
3077 TEST_F(PGLogTrimTest
, TestCopyUpTo
) {
3079 PGLog::IndexedLog log
, copy
;
3080 log
.tail
= mk_evt(9, 99);
3081 log
.head
= mk_evt(9, 99);
3083 entity_name_t client
= entity_name_t::CLIENT(777);
3085 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3086 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3088 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3089 osd_reqid_t(client
, 8, 1)));
3090 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3091 osd_reqid_t(client
, 8, 2)));
3092 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3093 osd_reqid_t(client
, 8, 3)));
3094 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3095 osd_reqid_t(client
, 8, 4)));
3096 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3097 osd_reqid_t(client
, 8, 5)));
3098 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3099 osd_reqid_t(client
, 8, 6)));
3100 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3101 osd_reqid_t(client
, 8, 6)));
3102 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3103 osd_reqid_t(client
, 8, 6)));
3105 copy
.copy_up_to(cct
, log
, 2);
3107 EXPECT_EQ(2u, copy
.log
.size()) << copy
;
3108 EXPECT_EQ(copy
.head
, mk_evt(21, 107)) << copy
;
3109 EXPECT_EQ(copy
.tail
, mk_evt(21, 105)) << copy
;
3110 // Tracking 5 means 3 additional as dups
3111 EXPECT_EQ(3u, copy
.dups
.size()) << copy
;
3114 // This tests copy_up_to() to make copies of
3115 // 4 log entries (107, 106, 105, 104) and 5 additional for a total
3116 // of 5 dups. Only 1 of 2 existing dups are copied.
3117 TEST_F(PGLogTrimTest
, TestCopyUpTo2
) {
3119 PGLog::IndexedLog log
, copy
;
3120 log
.tail
= mk_evt(9, 99);
3121 log
.head
= mk_evt(9, 99);
3123 entity_name_t client
= entity_name_t::CLIENT(777);
3125 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3126 mk_evt(8, 98), mk_evt(8, 97), osd_reqid_t(client
, 8, 1))));
3127 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3128 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3130 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3131 osd_reqid_t(client
, 8, 1)));
3132 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3133 osd_reqid_t(client
, 8, 2)));
3134 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3135 osd_reqid_t(client
, 8, 3)));
3136 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3137 osd_reqid_t(client
, 8, 4)));
3138 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3139 osd_reqid_t(client
, 8, 5)));
3140 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3141 osd_reqid_t(client
, 8, 6)));
3142 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3143 osd_reqid_t(client
, 8, 6)));
3144 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3145 osd_reqid_t(client
, 8, 6)));
3147 copy
.copy_up_to(cct
, log
, 4);
3149 EXPECT_EQ(4u, copy
.log
.size()) << copy
;
3150 EXPECT_EQ(copy
.head
, mk_evt(21, 107)) << copy
;
3151 EXPECT_EQ(copy
.tail
, mk_evt(20, 103)) << copy
;
3152 // Tracking 5 means 3 additional as dups
3153 EXPECT_EQ(5u, copy
.dups
.size()) << copy
;
3156 // This tests copy_after() by specifying a version that copies
3157 // 2 log entries (107, 106) and 3 additional for a total
3158 // of 5 dups. Nothing of the original dups is copied.
3159 TEST_F(PGLogTrimTest
, TestCopyAfter
) {
3161 PGLog::IndexedLog log
, copy
;
3162 log
.tail
= mk_evt(9, 99);
3163 log
.head
= mk_evt(9, 99);
3165 entity_name_t client
= entity_name_t::CLIENT(777);
3167 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3168 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3170 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3171 osd_reqid_t(client
, 8, 1)));
3172 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3173 osd_reqid_t(client
, 8, 2)));
3174 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3175 osd_reqid_t(client
, 8, 3)));
3176 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3177 osd_reqid_t(client
, 8, 4)));
3178 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3179 osd_reqid_t(client
, 8, 5)));
3180 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3181 osd_reqid_t(client
, 8, 6)));
3182 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3183 osd_reqid_t(client
, 8, 6)));
3184 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3185 osd_reqid_t(client
, 8, 6)));
3187 copy
.copy_after(cct
, log
, mk_evt(21, 105));
3189 EXPECT_EQ(2u, copy
.log
.size()) << copy
;
3190 EXPECT_EQ(copy
.head
, mk_evt(21, 107)) << copy
;
3191 EXPECT_EQ(copy
.tail
, mk_evt(21, 105)) << copy
;
3192 // Tracking 5 means 3 additional as dups
3193 EXPECT_EQ(3u, copy
.dups
.size()) << copy
;
3196 // This copies everything dups and log because of the large max dups
3197 // and value passed to copy_after().
3198 TEST_F(PGLogTrimTest
, TestCopyAfter2
) {
3200 PGLog::IndexedLog log
, copy
;
3201 log
.tail
= mk_evt(9, 99);
3202 log
.head
= mk_evt(9, 99);
3204 entity_name_t client
= entity_name_t::CLIENT(777);
3206 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3207 mk_evt(8, 93), mk_evt(8, 92), osd_reqid_t(client
, 8, 1))));
3208 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3209 mk_evt(8, 94), mk_evt(8, 93), osd_reqid_t(client
, 8, 1))));
3210 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3211 mk_evt(8, 95), mk_evt(8, 94), osd_reqid_t(client
, 8, 1))));
3212 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3213 mk_evt(8, 96), mk_evt(8, 95), osd_reqid_t(client
, 8, 1))));
3214 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3215 mk_evt(8, 97), mk_evt(8, 96), osd_reqid_t(client
, 8, 1))));
3216 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3217 mk_evt(8, 98), mk_evt(8, 97), osd_reqid_t(client
, 8, 1))));
3218 log
.dups
.push_back(pg_log_dup_t(mk_ple_mod(mk_obj(1),
3219 mk_evt(9, 99), mk_evt(8, 98), osd_reqid_t(client
, 8, 1))));
3221 log
.add(mk_ple_mod(mk_obj(1), mk_evt(10, 100), mk_evt(9, 99),
3222 osd_reqid_t(client
, 8, 1)));
3223 log
.add(mk_ple_dt(mk_obj(2), mk_evt(15, 101), mk_evt(10, 100),
3224 osd_reqid_t(client
, 8, 2)));
3225 log
.add(mk_ple_mod_rb(mk_obj(3), mk_evt(15, 102), mk_evt(15, 101),
3226 osd_reqid_t(client
, 8, 3)));
3227 log
.add(mk_ple_mod(mk_obj(1), mk_evt(20, 103), mk_evt(15, 102),
3228 osd_reqid_t(client
, 8, 4)));
3229 log
.add(mk_ple_mod(mk_obj(4), mk_evt(21, 104), mk_evt(20, 103),
3230 osd_reqid_t(client
, 8, 5)));
3231 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 105), mk_evt(21, 104),
3232 osd_reqid_t(client
, 8, 6)));
3233 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 106), mk_evt(21, 105),
3234 osd_reqid_t(client
, 8, 6)));
3235 log
.add(mk_ple_dt_rb(mk_obj(5), mk_evt(21, 107), mk_evt(21, 106),
3236 osd_reqid_t(client
, 8, 6)));
3238 copy
.copy_after(cct
, log
, mk_evt(9, 99));
3240 EXPECT_EQ(8u, copy
.log
.size()) << copy
;
3241 EXPECT_EQ(copy
.head
, mk_evt(21, 107)) << copy
;
3242 EXPECT_EQ(copy
.tail
, mk_evt(9, 99)) << copy
;
3243 // Tracking 3000 is larger than all entries, so all dups copied
3244 EXPECT_EQ(7u, copy
.dups
.size()) << copy
;
3248 // compile-command: "cd ../.. ; make unittest_pglog ; ./unittest_pglog --log-to-stderr=true --debug-osd=20 # --gtest_filter=*.* "