1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "test/librados_test_stub/TestMemIoCtxImpl.h"
5 #include "test/librados_test_stub/TestMemRadosClient.h"
6 #include "common/Clock.h"
7 #include "common/RWLock.h"
8 #include "include/err.h"
9 #include <boost/algorithm/string/predicate.hpp>
10 #include <boost/bind.hpp>
12 #include <include/compat.h>
14 static void to_vector(const interval_set
<uint64_t> &set
,
15 std::vector
<std::pair
<uint64_t, uint64_t> > *vec
) {
17 for (interval_set
<uint64_t>::const_iterator it
= set
.begin();
18 it
!= set
.end(); ++it
) {
23 // see PrimaryLogPG::finish_extent_cmp()
24 static int cmpext_compare(const bufferlist
&bl
, const bufferlist
&read_bl
) {
25 for (uint64_t idx
= 0; idx
< bl
.length(); ++idx
) {
26 char read_byte
= (idx
< read_bl
.length() ? read_bl
[idx
] : 0);
27 if (bl
[idx
] != read_byte
) {
28 return -MAX_ERRNO
- idx
;
36 TestMemIoCtxImpl::TestMemIoCtxImpl() {
39 TestMemIoCtxImpl::TestMemIoCtxImpl(const TestMemIoCtxImpl
& rhs
)
40 : TestIoCtxImpl(rhs
), m_client(rhs
.m_client
), m_pool(rhs
.m_pool
) {
44 TestMemIoCtxImpl::TestMemIoCtxImpl(TestMemRadosClient
*client
, int64_t pool_id
,
45 const std::string
& pool_name
,
46 TestMemCluster::Pool
*pool
)
47 : TestIoCtxImpl(client
, pool_id
, pool_name
), m_client(client
),
52 TestMemIoCtxImpl::~TestMemIoCtxImpl() {
56 TestIoCtxImpl
*TestMemIoCtxImpl::clone() {
57 return new TestMemIoCtxImpl(*this);
60 int TestMemIoCtxImpl::aio_remove(const std::string
& oid
, AioCompletionImpl
*c
, int flags
) {
61 m_client
->add_aio_operation(oid
, true,
62 boost::bind(&TestMemIoCtxImpl::remove
, this, oid
,
68 int TestMemIoCtxImpl::append(const std::string
& oid
, const bufferlist
&bl
,
69 const SnapContext
&snapc
) {
70 if (get_snap_read() != CEPH_NOSNAP
) {
72 } else if (m_client
->is_blacklisted()) {
76 TestMemCluster::SharedFile file
;
78 RWLock::WLocker
l(m_pool
->file_lock
);
79 file
= get_file(oid
, true, snapc
);
82 RWLock::WLocker
l(file
->lock
);
83 auto off
= file
->data
.length();
84 ensure_minimum_length(off
+ bl
.length(), &file
->data
);
85 file
->data
.copy_in(off
, bl
.length(), bl
);
89 int TestMemIoCtxImpl::assert_exists(const std::string
&oid
) {
90 if (m_client
->is_blacklisted()) {
94 RWLock::RLocker
l(m_pool
->file_lock
);
95 TestMemCluster::SharedFile file
= get_file(oid
, false, get_snap_context());
102 int TestMemIoCtxImpl::create(const std::string
& oid
, bool exclusive
) {
103 if (get_snap_read() != CEPH_NOSNAP
) {
105 } else if (m_client
->is_blacklisted()) {
106 return -EBLACKLISTED
;
109 RWLock::WLocker
l(m_pool
->file_lock
);
110 get_file(oid
, true, get_snap_context());
114 int TestMemIoCtxImpl::list_snaps(const std::string
& oid
, snap_set_t
*out_snaps
) {
115 if (m_client
->is_blacklisted()) {
116 return -EBLACKLISTED
;
120 out_snaps
->clones
.clear();
122 RWLock::RLocker
l(m_pool
->file_lock
);
123 TestMemCluster::Files::iterator it
= m_pool
->files
.find(
124 {get_namespace(), oid
});
125 if (it
== m_pool
->files
.end()) {
129 bool include_head
= false;
130 TestMemCluster::FileSnapshots
&file_snaps
= it
->second
;
131 for (TestMemCluster::FileSnapshots::iterator s_it
= file_snaps
.begin();
132 s_it
!= file_snaps
.end(); ++s_it
) {
133 TestMemCluster::File
&file
= *s_it
->get();
135 if (file_snaps
.size() > 1) {
136 out_snaps
->seq
= file
.snap_id
;
137 TestMemCluster::FileSnapshots::iterator
next_it(s_it
);
139 if (next_it
== file_snaps
.end()) {
149 // update the overlap with the next version's overlap metadata
150 TestMemCluster::File
&next_file
= *next_it
->get();
151 interval_set
<uint64_t> overlap
;
152 if (next_file
.exists
) {
153 overlap
= next_file
.snap_overlap
;
157 clone
.cloneid
= file
.snap_id
;
158 clone
.snaps
= file
.snaps
;
159 to_vector(overlap
, &clone
.overlap
);
160 clone
.size
= file
.data
.length();
161 out_snaps
->clones
.push_back(clone
);
165 if ((file_snaps
.size() == 1 && file_snaps
.back()->data
.length() > 0) ||
168 // Include the SNAP_HEAD
169 TestMemCluster::File
&file
= *file_snaps
.back();
171 RWLock::RLocker
l2(file
.lock
);
172 if (out_snaps
->seq
== 0 && !include_head
) {
173 out_snaps
->seq
= file
.snap_id
;
175 clone_info_t head_clone
;
176 head_clone
.cloneid
= librados::SNAP_HEAD
;
177 head_clone
.size
= file
.data
.length();
178 out_snaps
->clones
.push_back(head_clone
);
185 int TestMemIoCtxImpl::omap_get_vals2(const std::string
& oid
,
186 const std::string
& start_after
,
187 const std::string
&filter_prefix
,
189 std::map
<std::string
, bufferlist
> *out_vals
,
191 if (out_vals
== NULL
) {
193 } else if (m_client
->is_blacklisted()) {
194 return -EBLACKLISTED
;
197 TestMemCluster::SharedFile file
;
199 RWLock::RLocker
l(m_pool
->file_lock
);
200 file
= get_file(oid
, false, get_snap_context());
208 RWLock::RLocker
l(file
->lock
);
209 TestMemCluster::FileOMaps::iterator o_it
= m_pool
->file_omaps
.find(
210 {get_namespace(), oid
});
211 if (o_it
== m_pool
->file_omaps
.end()) {
218 TestMemCluster::OMap
&omap
= o_it
->second
;
219 TestMemCluster::OMap::iterator it
= omap
.begin();
220 if (!start_after
.empty()) {
221 it
= omap
.upper_bound(start_after
);
224 while (it
!= omap
.end() && max_return
> 0) {
225 if (filter_prefix
.empty() ||
226 boost::algorithm::starts_with(it
->first
, filter_prefix
)) {
227 (*out_vals
)[it
->first
] = it
->second
;
233 *pmore
= (it
!= omap
.end());
238 int TestMemIoCtxImpl::omap_get_vals(const std::string
& oid
,
239 const std::string
& start_after
,
240 const std::string
&filter_prefix
,
242 std::map
<std::string
, bufferlist
> *out_vals
) {
243 return omap_get_vals2(oid
, start_after
, filter_prefix
, max_return
, out_vals
, nullptr);
246 int TestMemIoCtxImpl::omap_rm_keys(const std::string
& oid
,
247 const std::set
<std::string
>& keys
) {
248 if (get_snap_read() != CEPH_NOSNAP
) {
250 } else if (m_client
->is_blacklisted()) {
251 return -EBLACKLISTED
;
254 TestMemCluster::SharedFile file
;
256 RWLock::WLocker
l(m_pool
->file_lock
);
257 file
= get_file(oid
, true, get_snap_context());
263 RWLock::WLocker
l(file
->lock
);
264 for (std::set
<std::string
>::iterator it
= keys
.begin();
265 it
!= keys
.end(); ++it
) {
266 m_pool
->file_omaps
[{get_namespace(), oid
}].erase(*it
);
271 int TestMemIoCtxImpl::omap_set(const std::string
& oid
,
272 const std::map
<std::string
, bufferlist
> &map
) {
273 if (get_snap_read() != CEPH_NOSNAP
) {
275 } else if (m_client
->is_blacklisted()) {
276 return -EBLACKLISTED
;
279 TestMemCluster::SharedFile file
;
281 RWLock::WLocker
l(m_pool
->file_lock
);
282 file
= get_file(oid
, true, get_snap_context());
288 RWLock::WLocker
l(file
->lock
);
289 for (std::map
<std::string
, bufferlist
>::const_iterator it
= map
.begin();
290 it
!= map
.end(); ++it
) {
292 bl
.append(it
->second
);
293 m_pool
->file_omaps
[{get_namespace(), oid
}][it
->first
] = bl
;
299 int TestMemIoCtxImpl::read(const std::string
& oid
, size_t len
, uint64_t off
,
301 if (m_client
->is_blacklisted()) {
302 return -EBLACKLISTED
;
305 TestMemCluster::SharedFile file
;
307 RWLock::RLocker
l(m_pool
->file_lock
);
308 file
= get_file(oid
, false, get_snap_context());
314 RWLock::RLocker
l(file
->lock
);
316 len
= file
->data
.length();
318 len
= clip_io(off
, len
, file
->data
.length());
319 if (bl
!= NULL
&& len
> 0) {
321 bit
.substr_of(file
->data
, off
, len
);
322 append_clone(bit
, bl
);
327 int TestMemIoCtxImpl::remove(const std::string
& oid
, const SnapContext
&snapc
) {
328 if (get_snap_read() != CEPH_NOSNAP
) {
330 } else if (m_client
->is_blacklisted()) {
331 return -EBLACKLISTED
;
334 RWLock::WLocker
l(m_pool
->file_lock
);
335 TestMemCluster::SharedFile file
= get_file(oid
, false, snapc
);
339 file
= get_file(oid
, true, snapc
);
342 RWLock::WLocker
l2(file
->lock
);
343 file
->exists
= false;
346 TestCluster::ObjectLocator
locator(get_namespace(), oid
);
347 TestMemCluster::Files::iterator it
= m_pool
->files
.find(locator
);
348 ceph_assert(it
!= m_pool
->files
.end());
350 if (*it
->second
.rbegin() == file
) {
351 TestMemCluster::ObjectHandlers object_handlers
;
352 std::swap(object_handlers
, m_pool
->file_handlers
[locator
]);
353 m_pool
->file_handlers
.erase(locator
);
355 for (auto object_handler
: object_handlers
) {
356 object_handler
->handle_removed(m_client
);
360 if (it
->second
.size() == 1) {
361 m_pool
->files
.erase(it
);
362 m_pool
->file_omaps
.erase(locator
);
367 int TestMemIoCtxImpl::selfmanaged_snap_create(uint64_t *snapid
) {
368 if (m_client
->is_blacklisted()) {
369 return -EBLACKLISTED
;
372 RWLock::WLocker
l(m_pool
->file_lock
);
373 *snapid
= ++m_pool
->snap_id
;
374 m_pool
->snap_seqs
.insert(*snapid
);
378 int TestMemIoCtxImpl::selfmanaged_snap_remove(uint64_t snapid
) {
379 if (m_client
->is_blacklisted()) {
380 return -EBLACKLISTED
;
383 RWLock::WLocker
l(m_pool
->file_lock
);
384 TestMemCluster::SnapSeqs::iterator it
=
385 m_pool
->snap_seqs
.find(snapid
);
386 if (it
== m_pool
->snap_seqs
.end()) {
390 // TODO clean up all file snapshots
391 m_pool
->snap_seqs
.erase(it
);
395 int TestMemIoCtxImpl::selfmanaged_snap_rollback(const std::string
& oid
,
397 if (m_client
->is_blacklisted()) {
398 return -EBLACKLISTED
;
401 RWLock::WLocker
l(m_pool
->file_lock
);
403 TestMemCluster::SharedFile file
;
404 TestMemCluster::Files::iterator f_it
= m_pool
->files
.find(
405 {get_namespace(), oid
});
406 if (f_it
== m_pool
->files
.end()) {
410 TestMemCluster::FileSnapshots
&snaps
= f_it
->second
;
414 for (TestMemCluster::FileSnapshots::reverse_iterator it
= snaps
.rbegin();
415 it
!= snaps
.rend(); ++it
) {
416 TestMemCluster::SharedFile file
= *it
;
417 if (file
->snap_id
< get_snap_read()) {
419 // already at the snapshot version
421 } else if (file
->snap_id
== CEPH_NOSNAP
) {
423 // delete it current HEAD, next one is correct version
424 snaps
.erase(it
.base());
426 // overwrite contents of current HEAD
427 file
= TestMemCluster::SharedFile (new TestMemCluster::File(**it
));
428 file
->snap_id
= CEPH_NOSNAP
;
432 // create new head version
433 file
= TestMemCluster::SharedFile (new TestMemCluster::File(**it
));
434 file
->snap_id
= m_pool
->snap_id
;
435 snaps
.push_back(file
);
444 int TestMemIoCtxImpl::set_alloc_hint(const std::string
& oid
,
445 uint64_t expected_object_size
,
446 uint64_t expected_write_size
,
447 const SnapContext
&snapc
) {
448 if (get_snap_read() != CEPH_NOSNAP
) {
450 } else if (m_client
->is_blacklisted()) {
451 return -EBLACKLISTED
;
455 RWLock::WLocker
l(m_pool
->file_lock
);
456 get_file(oid
, true, snapc
);
462 int TestMemIoCtxImpl::sparse_read(const std::string
& oid
, uint64_t off
,
464 std::map
<uint64_t,uint64_t> *m
,
465 bufferlist
*data_bl
) {
466 if (m_client
->is_blacklisted()) {
467 return -EBLACKLISTED
;
470 // TODO verify correctness
471 TestMemCluster::SharedFile file
;
473 RWLock::RLocker
l(m_pool
->file_lock
);
474 file
= get_file(oid
, false, get_snap_context());
480 RWLock::RLocker
l(file
->lock
);
481 len
= clip_io(off
, len
, file
->data
.length());
482 // TODO support sparse read
489 if (data_bl
!= NULL
&& len
> 0) {
491 bit
.substr_of(file
->data
, off
, len
);
492 append_clone(bit
, data_bl
);
494 return len
> 0 ? 1 : 0;
497 int TestMemIoCtxImpl::stat(const std::string
& oid
, uint64_t *psize
,
499 if (m_client
->is_blacklisted()) {
500 return -EBLACKLISTED
;
503 TestMemCluster::SharedFile file
;
505 RWLock::RLocker
l(m_pool
->file_lock
);
506 file
= get_file(oid
, false, get_snap_context());
512 RWLock::RLocker
l(file
->lock
);
514 *psize
= file
->data
.length();
516 if (pmtime
!= NULL
) {
517 *pmtime
= file
->mtime
;
522 int TestMemIoCtxImpl::truncate(const std::string
& oid
, uint64_t size
,
523 const SnapContext
&snapc
) {
524 if (get_snap_read() != CEPH_NOSNAP
) {
526 } else if (m_client
->is_blacklisted()) {
527 return -EBLACKLISTED
;
530 TestMemCluster::SharedFile file
;
532 RWLock::WLocker
l(m_pool
->file_lock
);
533 file
= get_file(oid
, true, snapc
);
536 RWLock::WLocker
l(file
->lock
);
539 interval_set
<uint64_t> is
;
540 if (file
->data
.length() > size
) {
541 is
.insert(size
, file
->data
.length() - size
);
543 bl
.substr_of(file
->data
, 0, size
);
545 } else if (file
->data
.length() != size
) {
551 bl
.append_zero(size
- file
->data
.length());
552 file
->data
.append(bl
);
555 is
.intersection_of(file
->snap_overlap
);
556 file
->snap_overlap
.subtract(is
);
560 int TestMemIoCtxImpl::write(const std::string
& oid
, bufferlist
& bl
, size_t len
,
561 uint64_t off
, const SnapContext
&snapc
) {
562 if (get_snap_read() != CEPH_NOSNAP
) {
564 } else if (m_client
->is_blacklisted()) {
565 return -EBLACKLISTED
;
568 TestMemCluster::SharedFile file
;
570 RWLock::WLocker
l(m_pool
->file_lock
);
571 file
= get_file(oid
, true, snapc
);
574 RWLock::WLocker
l(file
->lock
);
576 interval_set
<uint64_t> is
;
578 is
.intersection_of(file
->snap_overlap
);
579 file
->snap_overlap
.subtract(is
);
582 ensure_minimum_length(off
+ len
, &file
->data
);
583 file
->data
.copy_in(off
, len
, bl
);
587 int TestMemIoCtxImpl::write_full(const std::string
& oid
, bufferlist
& bl
,
588 const SnapContext
&snapc
) {
589 if (get_snap_read() != CEPH_NOSNAP
) {
591 } else if (m_client
->is_blacklisted()) {
592 return -EBLACKLISTED
;
595 TestMemCluster::SharedFile file
;
597 RWLock::WLocker
l(m_pool
->file_lock
);
598 file
= get_file(oid
, true, snapc
);
604 RWLock::WLocker
l(file
->lock
);
605 if (bl
.length() > 0) {
606 interval_set
<uint64_t> is
;
607 is
.insert(0, bl
.length());
608 is
.intersection_of(file
->snap_overlap
);
609 file
->snap_overlap
.subtract(is
);
613 ensure_minimum_length(bl
.length(), &file
->data
);
614 file
->data
.copy_in(0, bl
.length(), bl
);
618 int TestMemIoCtxImpl::writesame(const std::string
& oid
, bufferlist
& bl
, size_t len
,
619 uint64_t off
, const SnapContext
&snapc
) {
620 if (get_snap_read() != CEPH_NOSNAP
) {
622 } else if (m_client
->is_blacklisted()) {
623 return -EBLACKLISTED
;
626 if (len
== 0 || (len
% bl
.length())) {
630 TestMemCluster::SharedFile file
;
632 RWLock::WLocker
l(m_pool
->file_lock
);
633 file
= get_file(oid
, true, snapc
);
636 RWLock::WLocker
l(file
->lock
);
638 interval_set
<uint64_t> is
;
640 is
.intersection_of(file
->snap_overlap
);
641 file
->snap_overlap
.subtract(is
);
644 ensure_minimum_length(off
+ len
, &file
->data
);
646 file
->data
.copy_in(off
, bl
.length(), bl
);
653 int TestMemIoCtxImpl::cmpext(const std::string
& oid
, uint64_t off
,
654 bufferlist
& cmp_bl
) {
655 if (m_client
->is_blacklisted()) {
656 return -EBLACKLISTED
;
660 uint64_t len
= cmp_bl
.length();
662 TestMemCluster::SharedFile file
;
664 RWLock::RLocker
l(m_pool
->file_lock
);
665 file
= get_file(oid
, false, get_snap_context());
667 return cmpext_compare(cmp_bl
, read_bl
);
671 RWLock::RLocker
l(file
->lock
);
672 if (off
>= file
->data
.length()) {
674 } else if (off
+ len
> file
->data
.length()) {
675 len
= file
->data
.length() - off
;
677 read_bl
.substr_of(file
->data
, off
, len
);
678 return cmpext_compare(cmp_bl
, read_bl
);
681 int TestMemIoCtxImpl::xattr_get(const std::string
& oid
,
682 std::map
<std::string
, bufferlist
>* attrset
) {
683 if (m_client
->is_blacklisted()) {
684 return -EBLACKLISTED
;
687 TestMemCluster::SharedFile file
;
688 RWLock::RLocker
l(m_pool
->file_lock
);
689 TestMemCluster::FileXAttrs::iterator it
= m_pool
->file_xattrs
.find(
690 {get_namespace(), oid
});
691 if (it
== m_pool
->file_xattrs
.end()) {
694 *attrset
= it
->second
;
698 int TestMemIoCtxImpl::xattr_set(const std::string
& oid
, const std::string
&name
,
700 if (m_client
->is_blacklisted()) {
701 return -EBLACKLISTED
;
704 RWLock::WLocker
l(m_pool
->file_lock
);
705 m_pool
->file_xattrs
[{get_namespace(), oid
}][name
] = bl
;
709 int TestMemIoCtxImpl::zero(const std::string
& oid
, uint64_t off
, uint64_t len
,
710 const SnapContext
&snapc
) {
711 if (m_client
->is_blacklisted()) {
712 return -EBLACKLISTED
;
715 bool truncate_redirect
= false;
716 TestMemCluster::SharedFile file
;
718 RWLock::WLocker
l(m_pool
->file_lock
);
719 file
= get_file(oid
, false, snapc
);
723 file
= get_file(oid
, true, snapc
);
725 RWLock::RLocker
l2(file
->lock
);
726 if (len
> 0 && off
+ len
>= file
->data
.length()) {
727 // Zero -> Truncate logic embedded in OSD
728 truncate_redirect
= true;
731 if (truncate_redirect
) {
732 return truncate(oid
, off
, snapc
);
737 return write(oid
, bl
, len
, off
, snapc
);
740 void TestMemIoCtxImpl::append_clone(bufferlist
& src
, bufferlist
* dest
) {
741 // deep-copy the src to ensure our memory-based mock RADOS data cannot
742 // be modified by callers
743 if (src
.length() > 0) {
744 bufferlist::iterator iter
= src
.begin();
746 iter
.copy_deep(src
.length(), ptr
);
751 size_t TestMemIoCtxImpl::clip_io(size_t off
, size_t len
, size_t bl_len
) {
754 } else if (off
+ len
> bl_len
) {
760 void TestMemIoCtxImpl::ensure_minimum_length(size_t len
, bufferlist
*bl
) {
761 if (len
> bl
->length()) {
762 bufferptr
ptr(buffer::create(len
- bl
->length()));
768 TestMemCluster::SharedFile
TestMemIoCtxImpl::get_file(
769 const std::string
&oid
, bool write
, const SnapContext
&snapc
) {
770 ceph_assert(m_pool
->file_lock
.is_locked() || m_pool
->file_lock
.is_wlocked());
771 ceph_assert(!write
|| m_pool
->file_lock
.is_wlocked());
773 TestMemCluster::SharedFile file
;
774 TestMemCluster::Files::iterator it
= m_pool
->files
.find(
775 {get_namespace(), oid
});
776 if (it
!= m_pool
->files
.end()) {
777 file
= it
->second
.back();
779 return TestMemCluster::SharedFile();
783 bool new_version
= false;
784 if (!file
|| !file
->exists
) {
785 file
= TestMemCluster::SharedFile(new TestMemCluster::File());
788 if (!snapc
.snaps
.empty() && file
->snap_id
< snapc
.seq
) {
789 for (std::vector
<snapid_t
>::const_reverse_iterator seq_it
=
790 snapc
.snaps
.rbegin();
791 seq_it
!= snapc
.snaps
.rend(); ++seq_it
) {
792 if (*seq_it
> file
->snap_id
&& *seq_it
<= snapc
.seq
) {
793 file
->snaps
.push_back(*seq_it
);
797 bufferlist prev_data
= file
->data
;
798 file
= TestMemCluster::SharedFile(
799 new TestMemCluster::File(*file
));
801 append_clone(prev_data
, &file
->data
);
802 if (prev_data
.length() > 0) {
803 file
->snap_overlap
.insert(0, prev_data
.length());
810 file
->snap_id
= snapc
.seq
;
811 file
->mtime
= ceph_clock_now().sec();
812 m_pool
->files
[{get_namespace(), oid
}].push_back(file
);
817 if (get_snap_read() == CEPH_NOSNAP
) {
819 ceph_assert(it
->second
.size() > 1);
820 return TestMemCluster::SharedFile();
825 TestMemCluster::FileSnapshots
&snaps
= it
->second
;
826 for (TestMemCluster::FileSnapshots::reverse_iterator it
= snaps
.rbegin();
827 it
!= snaps
.rend(); ++it
) {
828 TestMemCluster::SharedFile file
= *it
;
829 if (file
->snap_id
< get_snap_read()) {
831 return TestMemCluster::SharedFile();
836 return TestMemCluster::SharedFile();
839 } // namespace librados