1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 #include "test/librados_test_stub/TestMemIoCtxImpl.h"
5 #include "test/librados_test_stub/TestMemRadosClient.h"
6 #include "common/Clock.h"
7 #include "common/RWLock.h"
8 #include "include/err.h"
9 #include <boost/algorithm/string/predicate.hpp>
10 #include <boost/bind.hpp>
12 #include <include/compat.h>
14 static void to_vector(const interval_set
<uint64_t> &set
,
15 std::vector
<std::pair
<uint64_t, uint64_t> > *vec
) {
17 for (interval_set
<uint64_t>::const_iterator it
= set
.begin();
18 it
!= set
.end(); ++it
) {
23 // see PrimaryLogPG::finish_extent_cmp()
24 static int cmpext_compare(const bufferlist
&bl
, const bufferlist
&read_bl
) {
25 for (uint64_t idx
= 0; idx
< bl
.length(); ++idx
) {
26 char read_byte
= (idx
< read_bl
.length() ? read_bl
[idx
] : 0);
27 if (bl
[idx
] != read_byte
) {
28 return -MAX_ERRNO
- idx
;
36 TestMemIoCtxImpl::TestMemIoCtxImpl() {
39 TestMemIoCtxImpl::TestMemIoCtxImpl(const TestMemIoCtxImpl
& rhs
)
40 : TestIoCtxImpl(rhs
), m_client(rhs
.m_client
), m_pool(rhs
.m_pool
) {
44 TestMemIoCtxImpl::TestMemIoCtxImpl(TestMemRadosClient
*client
, int64_t pool_id
,
45 const std::string
& pool_name
,
46 TestMemCluster::Pool
*pool
)
47 : TestIoCtxImpl(client
, pool_id
, pool_name
), m_client(client
),
52 TestMemIoCtxImpl::~TestMemIoCtxImpl() {
56 TestIoCtxImpl
*TestMemIoCtxImpl::clone() {
57 return new TestMemIoCtxImpl(*this);
60 int TestMemIoCtxImpl::aio_remove(const std::string
& oid
, AioCompletionImpl
*c
, int flags
) {
61 m_client
->add_aio_operation(oid
, true,
62 boost::bind(&TestMemIoCtxImpl::remove
, this, oid
,
68 int TestMemIoCtxImpl::append(const std::string
& oid
, const bufferlist
&bl
,
69 const SnapContext
&snapc
) {
70 if (get_snap_read() != CEPH_NOSNAP
) {
72 } else if (m_client
->is_blacklisted()) {
76 TestMemCluster::SharedFile file
;
78 std::unique_lock l
{m_pool
->file_lock
};
79 file
= get_file(oid
, true, snapc
);
82 std::unique_lock l
{file
->lock
};
83 auto off
= file
->data
.length();
84 ensure_minimum_length(off
+ bl
.length(), &file
->data
);
85 file
->data
.begin(off
).copy_in(bl
.length(), bl
);
89 int TestMemIoCtxImpl::assert_exists(const std::string
&oid
) {
90 if (m_client
->is_blacklisted()) {
94 std::shared_lock l
{m_pool
->file_lock
};
95 TestMemCluster::SharedFile file
= get_file(oid
, false, get_snap_context());
102 int TestMemIoCtxImpl::create(const std::string
& oid
, bool exclusive
,
103 const SnapContext
&snapc
) {
104 if (get_snap_read() != CEPH_NOSNAP
) {
106 } else if (m_client
->is_blacklisted()) {
107 return -EBLACKLISTED
;
110 std::unique_lock l
{m_pool
->file_lock
};
111 get_file(oid
, true, snapc
);
115 int TestMemIoCtxImpl::list_snaps(const std::string
& oid
, snap_set_t
*out_snaps
) {
116 if (m_client
->is_blacklisted()) {
117 return -EBLACKLISTED
;
121 out_snaps
->clones
.clear();
123 std::shared_lock l
{m_pool
->file_lock
};
124 TestMemCluster::Files::iterator it
= m_pool
->files
.find(
125 {get_namespace(), oid
});
126 if (it
== m_pool
->files
.end()) {
130 bool include_head
= false;
131 TestMemCluster::FileSnapshots
&file_snaps
= it
->second
;
132 for (TestMemCluster::FileSnapshots::iterator s_it
= file_snaps
.begin();
133 s_it
!= file_snaps
.end(); ++s_it
) {
134 TestMemCluster::File
&file
= *s_it
->get();
136 if (file_snaps
.size() > 1) {
137 out_snaps
->seq
= file
.snap_id
;
138 TestMemCluster::FileSnapshots::iterator
next_it(s_it
);
140 if (next_it
== file_snaps
.end()) {
150 // update the overlap with the next version's overlap metadata
151 TestMemCluster::File
&next_file
= *next_it
->get();
152 interval_set
<uint64_t> overlap
;
153 if (next_file
.exists
) {
154 overlap
= next_file
.snap_overlap
;
158 clone
.cloneid
= file
.snap_id
;
159 clone
.snaps
= file
.snaps
;
160 to_vector(overlap
, &clone
.overlap
);
161 clone
.size
= file
.data
.length();
162 out_snaps
->clones
.push_back(clone
);
166 if ((file_snaps
.size() == 1 && file_snaps
.back()->data
.length() > 0) ||
169 // Include the SNAP_HEAD
170 TestMemCluster::File
&file
= *file_snaps
.back();
172 std::shared_lock l2
{file
.lock
};
173 if (out_snaps
->seq
== 0 && !include_head
) {
174 out_snaps
->seq
= file
.snap_id
;
176 clone_info_t head_clone
;
177 head_clone
.cloneid
= librados::SNAP_HEAD
;
178 head_clone
.size
= file
.data
.length();
179 out_snaps
->clones
.push_back(head_clone
);
186 int TestMemIoCtxImpl::omap_get_vals2(const std::string
& oid
,
187 const std::string
& start_after
,
188 const std::string
&filter_prefix
,
190 std::map
<std::string
, bufferlist
> *out_vals
,
192 if (out_vals
== NULL
) {
194 } else if (m_client
->is_blacklisted()) {
195 return -EBLACKLISTED
;
198 TestMemCluster::SharedFile file
;
200 std::shared_lock l
{m_pool
->file_lock
};
201 file
= get_file(oid
, false, get_snap_context());
209 std::shared_lock l
{file
->lock
};
210 TestMemCluster::FileOMaps::iterator o_it
= m_pool
->file_omaps
.find(
211 {get_namespace(), oid
});
212 if (o_it
== m_pool
->file_omaps
.end()) {
219 TestMemCluster::OMap
&omap
= o_it
->second
;
220 TestMemCluster::OMap::iterator it
= omap
.begin();
221 if (!start_after
.empty()) {
222 it
= omap
.upper_bound(start_after
);
225 while (it
!= omap
.end() && max_return
> 0) {
226 if (filter_prefix
.empty() ||
227 boost::algorithm::starts_with(it
->first
, filter_prefix
)) {
228 (*out_vals
)[it
->first
] = it
->second
;
234 *pmore
= (it
!= omap
.end());
239 int TestMemIoCtxImpl::omap_get_vals(const std::string
& oid
,
240 const std::string
& start_after
,
241 const std::string
&filter_prefix
,
243 std::map
<std::string
, bufferlist
> *out_vals
) {
244 return omap_get_vals2(oid
, start_after
, filter_prefix
, max_return
, out_vals
, nullptr);
247 int TestMemIoCtxImpl::omap_rm_keys(const std::string
& oid
,
248 const std::set
<std::string
>& keys
) {
249 if (get_snap_read() != CEPH_NOSNAP
) {
251 } else if (m_client
->is_blacklisted()) {
252 return -EBLACKLISTED
;
255 TestMemCluster::SharedFile file
;
257 std::unique_lock l
{m_pool
->file_lock
};
258 file
= get_file(oid
, true, get_snap_context());
264 std::unique_lock l
{file
->lock
};
265 for (std::set
<std::string
>::iterator it
= keys
.begin();
266 it
!= keys
.end(); ++it
) {
267 m_pool
->file_omaps
[{get_namespace(), oid
}].erase(*it
);
272 int TestMemIoCtxImpl::omap_set(const std::string
& oid
,
273 const std::map
<std::string
, bufferlist
> &map
) {
274 if (get_snap_read() != CEPH_NOSNAP
) {
276 } else if (m_client
->is_blacklisted()) {
277 return -EBLACKLISTED
;
280 TestMemCluster::SharedFile file
;
282 std::unique_lock l
{m_pool
->file_lock
};
283 file
= get_file(oid
, true, get_snap_context());
289 std::unique_lock l
{file
->lock
};
290 for (std::map
<std::string
, bufferlist
>::const_iterator it
= map
.begin();
291 it
!= map
.end(); ++it
) {
293 bl
.append(it
->second
);
294 m_pool
->file_omaps
[{get_namespace(), oid
}][it
->first
] = bl
;
300 int TestMemIoCtxImpl::read(const std::string
& oid
, size_t len
, uint64_t off
,
302 if (m_client
->is_blacklisted()) {
303 return -EBLACKLISTED
;
306 TestMemCluster::SharedFile file
;
308 std::shared_lock l
{m_pool
->file_lock
};
309 file
= get_file(oid
, false, get_snap_context());
315 std::shared_lock l
{file
->lock
};
317 len
= file
->data
.length();
319 len
= clip_io(off
, len
, file
->data
.length());
320 if (bl
!= NULL
&& len
> 0) {
322 bit
.substr_of(file
->data
, off
, len
);
323 append_clone(bit
, bl
);
328 int TestMemIoCtxImpl::remove(const std::string
& oid
, const SnapContext
&snapc
) {
329 if (get_snap_read() != CEPH_NOSNAP
) {
331 } else if (m_client
->is_blacklisted()) {
332 return -EBLACKLISTED
;
335 std::unique_lock l
{m_pool
->file_lock
};
336 TestMemCluster::SharedFile file
= get_file(oid
, false, snapc
);
340 file
= get_file(oid
, true, snapc
);
343 std::unique_lock l2
{file
->lock
};
344 file
->exists
= false;
347 TestCluster::ObjectLocator
locator(get_namespace(), oid
);
348 TestMemCluster::Files::iterator it
= m_pool
->files
.find(locator
);
349 ceph_assert(it
!= m_pool
->files
.end());
351 if (*it
->second
.rbegin() == file
) {
352 TestMemCluster::ObjectHandlers object_handlers
;
353 std::swap(object_handlers
, m_pool
->file_handlers
[locator
]);
354 m_pool
->file_handlers
.erase(locator
);
356 for (auto object_handler
: object_handlers
) {
357 object_handler
->handle_removed(m_client
);
361 if (it
->second
.size() == 1) {
362 m_pool
->files
.erase(it
);
363 m_pool
->file_omaps
.erase(locator
);
368 int TestMemIoCtxImpl::selfmanaged_snap_create(uint64_t *snapid
) {
369 if (m_client
->is_blacklisted()) {
370 return -EBLACKLISTED
;
373 std::unique_lock l
{m_pool
->file_lock
};
374 *snapid
= ++m_pool
->snap_id
;
375 m_pool
->snap_seqs
.insert(*snapid
);
379 int TestMemIoCtxImpl::selfmanaged_snap_remove(uint64_t snapid
) {
380 if (m_client
->is_blacklisted()) {
381 return -EBLACKLISTED
;
384 std::unique_lock l
{m_pool
->file_lock
};
385 TestMemCluster::SnapSeqs::iterator it
=
386 m_pool
->snap_seqs
.find(snapid
);
387 if (it
== m_pool
->snap_seqs
.end()) {
391 // TODO clean up all file snapshots
392 m_pool
->snap_seqs
.erase(it
);
396 int TestMemIoCtxImpl::selfmanaged_snap_rollback(const std::string
& oid
,
398 if (m_client
->is_blacklisted()) {
399 return -EBLACKLISTED
;
402 std::unique_lock l
{m_pool
->file_lock
};
404 TestMemCluster::SharedFile file
;
405 TestMemCluster::Files::iterator f_it
= m_pool
->files
.find(
406 {get_namespace(), oid
});
407 if (f_it
== m_pool
->files
.end()) {
411 TestMemCluster::FileSnapshots
&snaps
= f_it
->second
;
415 for (TestMemCluster::FileSnapshots::reverse_iterator it
= snaps
.rbegin();
416 it
!= snaps
.rend(); ++it
) {
417 TestMemCluster::SharedFile file
= *it
;
418 if (file
->snap_id
< get_snap_read()) {
420 // already at the snapshot version
422 } else if (file
->snap_id
== CEPH_NOSNAP
) {
424 // delete it current HEAD, next one is correct version
425 snaps
.erase(it
.base());
427 // overwrite contents of current HEAD
428 file
= TestMemCluster::SharedFile (new TestMemCluster::File(**it
));
429 file
->snap_id
= CEPH_NOSNAP
;
433 // create new head version
434 file
= TestMemCluster::SharedFile (new TestMemCluster::File(**it
));
435 file
->snap_id
= m_pool
->snap_id
;
436 snaps
.push_back(file
);
445 int TestMemIoCtxImpl::set_alloc_hint(const std::string
& oid
,
446 uint64_t expected_object_size
,
447 uint64_t expected_write_size
,
449 const SnapContext
&snapc
) {
450 if (get_snap_read() != CEPH_NOSNAP
) {
452 } else if (m_client
->is_blacklisted()) {
453 return -EBLACKLISTED
;
457 std::unique_lock l
{m_pool
->file_lock
};
458 get_file(oid
, true, snapc
);
464 int TestMemIoCtxImpl::sparse_read(const std::string
& oid
, uint64_t off
,
466 std::map
<uint64_t,uint64_t> *m
,
467 bufferlist
*data_bl
) {
468 if (m_client
->is_blacklisted()) {
469 return -EBLACKLISTED
;
472 // TODO verify correctness
473 TestMemCluster::SharedFile file
;
475 std::shared_lock l
{m_pool
->file_lock
};
476 file
= get_file(oid
, false, get_snap_context());
482 std::shared_lock l
{file
->lock
};
483 len
= clip_io(off
, len
, file
->data
.length());
484 // TODO support sparse read
491 if (data_bl
!= NULL
&& len
> 0) {
493 bit
.substr_of(file
->data
, off
, len
);
494 append_clone(bit
, data_bl
);
496 return len
> 0 ? 1 : 0;
499 int TestMemIoCtxImpl::stat(const std::string
& oid
, uint64_t *psize
,
501 if (m_client
->is_blacklisted()) {
502 return -EBLACKLISTED
;
505 TestMemCluster::SharedFile file
;
507 std::shared_lock l
{m_pool
->file_lock
};
508 file
= get_file(oid
, false, get_snap_context());
514 std::shared_lock l
{file
->lock
};
516 *psize
= file
->data
.length();
518 if (pmtime
!= NULL
) {
519 *pmtime
= file
->mtime
;
524 int TestMemIoCtxImpl::truncate(const std::string
& oid
, uint64_t size
,
525 const SnapContext
&snapc
) {
526 if (get_snap_read() != CEPH_NOSNAP
) {
528 } else if (m_client
->is_blacklisted()) {
529 return -EBLACKLISTED
;
532 TestMemCluster::SharedFile file
;
534 std::unique_lock l
{m_pool
->file_lock
};
535 file
= get_file(oid
, true, snapc
);
538 std::unique_lock l
{file
->lock
};
541 interval_set
<uint64_t> is
;
542 if (file
->data
.length() > size
) {
543 is
.insert(size
, file
->data
.length() - size
);
545 bl
.substr_of(file
->data
, 0, size
);
547 } else if (file
->data
.length() != size
) {
553 bl
.append_zero(size
- file
->data
.length());
554 file
->data
.append(bl
);
557 is
.intersection_of(file
->snap_overlap
);
558 file
->snap_overlap
.subtract(is
);
562 int TestMemIoCtxImpl::write(const std::string
& oid
, bufferlist
& bl
, size_t len
,
563 uint64_t off
, const SnapContext
&snapc
) {
564 if (get_snap_read() != CEPH_NOSNAP
) {
566 } else if (m_client
->is_blacklisted()) {
567 return -EBLACKLISTED
;
570 TestMemCluster::SharedFile file
;
572 std::unique_lock l
{m_pool
->file_lock
};
573 file
= get_file(oid
, true, snapc
);
576 std::unique_lock l
{file
->lock
};
578 interval_set
<uint64_t> is
;
580 is
.intersection_of(file
->snap_overlap
);
581 file
->snap_overlap
.subtract(is
);
584 ensure_minimum_length(off
+ len
, &file
->data
);
585 file
->data
.begin(off
).copy_in(len
, bl
);
589 int TestMemIoCtxImpl::write_full(const std::string
& oid
, bufferlist
& bl
,
590 const SnapContext
&snapc
) {
591 if (get_snap_read() != CEPH_NOSNAP
) {
593 } else if (m_client
->is_blacklisted()) {
594 return -EBLACKLISTED
;
597 TestMemCluster::SharedFile file
;
599 std::unique_lock l
{m_pool
->file_lock
};
600 file
= get_file(oid
, true, snapc
);
606 std::unique_lock l
{file
->lock
};
607 if (bl
.length() > 0) {
608 interval_set
<uint64_t> is
;
609 is
.insert(0, bl
.length());
610 is
.intersection_of(file
->snap_overlap
);
611 file
->snap_overlap
.subtract(is
);
615 ensure_minimum_length(bl
.length(), &file
->data
);
616 file
->data
.begin().copy_in(bl
.length(), bl
);
620 int TestMemIoCtxImpl::writesame(const std::string
& oid
, bufferlist
& bl
, size_t len
,
621 uint64_t off
, const SnapContext
&snapc
) {
622 if (get_snap_read() != CEPH_NOSNAP
) {
624 } else if (m_client
->is_blacklisted()) {
625 return -EBLACKLISTED
;
628 if (len
== 0 || (len
% bl
.length())) {
632 TestMemCluster::SharedFile file
;
634 std::unique_lock l
{m_pool
->file_lock
};
635 file
= get_file(oid
, true, snapc
);
638 std::unique_lock l
{file
->lock
};
640 interval_set
<uint64_t> is
;
642 is
.intersection_of(file
->snap_overlap
);
643 file
->snap_overlap
.subtract(is
);
646 ensure_minimum_length(off
+ len
, &file
->data
);
648 file
->data
.begin(off
).copy_in(bl
.length(), bl
);
655 int TestMemIoCtxImpl::cmpext(const std::string
& oid
, uint64_t off
,
656 bufferlist
& cmp_bl
) {
657 if (m_client
->is_blacklisted()) {
658 return -EBLACKLISTED
;
662 uint64_t len
= cmp_bl
.length();
664 TestMemCluster::SharedFile file
;
666 std::shared_lock l
{m_pool
->file_lock
};
667 file
= get_file(oid
, false, get_snap_context());
669 return cmpext_compare(cmp_bl
, read_bl
);
673 std::shared_lock l
{file
->lock
};
674 if (off
>= file
->data
.length()) {
676 } else if (off
+ len
> file
->data
.length()) {
677 len
= file
->data
.length() - off
;
679 read_bl
.substr_of(file
->data
, off
, len
);
680 return cmpext_compare(cmp_bl
, read_bl
);
683 int TestMemIoCtxImpl::xattr_get(const std::string
& oid
,
684 std::map
<std::string
, bufferlist
>* attrset
) {
685 if (m_client
->is_blacklisted()) {
686 return -EBLACKLISTED
;
689 TestMemCluster::SharedFile file
;
690 std::shared_lock l
{m_pool
->file_lock
};
691 TestMemCluster::FileXAttrs::iterator it
= m_pool
->file_xattrs
.find(
692 {get_namespace(), oid
});
693 if (it
== m_pool
->file_xattrs
.end()) {
696 *attrset
= it
->second
;
700 int TestMemIoCtxImpl::xattr_set(const std::string
& oid
, const std::string
&name
,
702 if (m_client
->is_blacklisted()) {
703 return -EBLACKLISTED
;
706 std::unique_lock l
{m_pool
->file_lock
};
707 m_pool
->file_xattrs
[{get_namespace(), oid
}][name
] = bl
;
711 int TestMemIoCtxImpl::zero(const std::string
& oid
, uint64_t off
, uint64_t len
,
712 const SnapContext
&snapc
) {
713 if (m_client
->is_blacklisted()) {
714 return -EBLACKLISTED
;
717 bool truncate_redirect
= false;
718 TestMemCluster::SharedFile file
;
720 std::unique_lock l
{m_pool
->file_lock
};
721 file
= get_file(oid
, false, snapc
);
725 file
= get_file(oid
, true, snapc
);
727 std::shared_lock l2
{file
->lock
};
728 if (len
> 0 && off
+ len
>= file
->data
.length()) {
729 // Zero -> Truncate logic embedded in OSD
730 truncate_redirect
= true;
733 if (truncate_redirect
) {
734 return truncate(oid
, off
, snapc
);
739 return write(oid
, bl
, len
, off
, snapc
);
742 void TestMemIoCtxImpl::append_clone(bufferlist
& src
, bufferlist
* dest
) {
743 // deep-copy the src to ensure our memory-based mock RADOS data cannot
744 // be modified by callers
745 if (src
.length() > 0) {
746 bufferlist::iterator iter
= src
.begin();
748 iter
.copy_deep(src
.length(), ptr
);
753 size_t TestMemIoCtxImpl::clip_io(size_t off
, size_t len
, size_t bl_len
) {
756 } else if (off
+ len
> bl_len
) {
762 void TestMemIoCtxImpl::ensure_minimum_length(size_t len
, bufferlist
*bl
) {
763 if (len
> bl
->length()) {
764 bufferptr
ptr(buffer::create(len
- bl
->length()));
770 TestMemCluster::SharedFile
TestMemIoCtxImpl::get_file(
771 const std::string
&oid
, bool write
, const SnapContext
&snapc
) {
772 ceph_assert(ceph_mutex_is_locked(m_pool
->file_lock
) ||
773 ceph_mutex_is_wlocked(m_pool
->file_lock
));
774 ceph_assert(!write
|| ceph_mutex_is_wlocked(m_pool
->file_lock
));
776 TestMemCluster::SharedFile file
;
777 TestMemCluster::Files::iterator it
= m_pool
->files
.find(
778 {get_namespace(), oid
});
779 if (it
!= m_pool
->files
.end()) {
780 file
= it
->second
.back();
782 return TestMemCluster::SharedFile();
786 bool new_version
= false;
787 if (!file
|| !file
->exists
) {
788 file
= TestMemCluster::SharedFile(new TestMemCluster::File());
791 if (!snapc
.snaps
.empty() && file
->snap_id
< snapc
.seq
) {
792 for (std::vector
<snapid_t
>::const_reverse_iterator seq_it
=
793 snapc
.snaps
.rbegin();
794 seq_it
!= snapc
.snaps
.rend(); ++seq_it
) {
795 if (*seq_it
> file
->snap_id
&& *seq_it
<= snapc
.seq
) {
796 file
->snaps
.push_back(*seq_it
);
800 bufferlist prev_data
= file
->data
;
801 file
= TestMemCluster::SharedFile(
802 new TestMemCluster::File(*file
));
804 append_clone(prev_data
, &file
->data
);
805 if (prev_data
.length() > 0) {
806 file
->snap_overlap
.insert(0, prev_data
.length());
813 file
->snap_id
= snapc
.seq
;
814 file
->mtime
= ceph_clock_now().sec();
815 m_pool
->files
[{get_namespace(), oid
}].push_back(file
);
820 if (get_snap_read() == CEPH_NOSNAP
) {
822 ceph_assert(it
->second
.size() > 1);
823 return TestMemCluster::SharedFile();
828 TestMemCluster::FileSnapshots
&snaps
= it
->second
;
829 for (TestMemCluster::FileSnapshots::reverse_iterator it
= snaps
.rbegin();
830 it
!= snaps
.rend(); ++it
) {
831 TestMemCluster::SharedFile file
= *it
;
832 if (file
->snap_id
< get_snap_read()) {
834 return TestMemCluster::SharedFile();
839 return TestMemCluster::SharedFile();
842 } // namespace librados