]>
git.proxmox.com Git - ceph.git/blob - ceph/src/test/rgw/test_cls_fifo_legacy.cc
1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2019 Red Hat, Inc.
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
17 #include <string_view>
19 #include "include/scope_guard.h"
20 #include "include/types.h"
21 #include "include/rados/librados.hpp"
22 #include "common/ceph_context.h"
24 #include "cls/fifo/cls_fifo_ops.h"
25 #include "test/librados/test_cxx.h"
26 #include "global/global_context.h"
28 #include "rgw/rgw_tools.h"
29 #include "rgw/cls_fifo_legacy.h"
31 #include "gtest/gtest.h"
33 namespace R
= librados
;
34 namespace cb
= ceph::buffer
;
35 namespace fifo
= rados::cls::fifo
;
36 namespace RCf
= rgw::cls::fifo
;
38 auto cct
= new CephContext(CEPH_ENTITY_TYPE_CLIENT
);
39 const DoutPrefix
dp(cct
, 1, "test legacy cls fifo: ");
42 int fifo_create(const DoutPrefixProvider
*dpp
, R::IoCtx
& ioctx
,
43 const std::string
& oid
,
46 std::optional
<fifo::objv
> objv
= std::nullopt
,
47 std::optional
<std::string_view
> oid_prefix
= std::nullopt
,
48 bool exclusive
= false,
49 std::uint64_t max_part_size
= RCf::default_max_part_size
,
50 std::uint64_t max_entry_size
= RCf::default_max_entry_size
)
52 R::ObjectWriteOperation op
;
53 RCf::create_meta(&op
, id
, objv
, oid_prefix
, exclusive
, max_part_size
,
55 return rgw_rados_operate(dpp
, ioctx
, oid
, &op
, y
);
59 class LegacyFIFO
: public testing::Test
{
61 const std::string pool_name
= get_temp_pool_name();
62 const std::string fifo_id
= "fifo";
64 librados::IoCtx ioctx
;
66 void SetUp() override
{
67 ASSERT_EQ("", create_one_pool_pp(pool_name
, rados
));
68 ASSERT_EQ(0, rados
.ioctx_create(pool_name
.c_str(), ioctx
));
70 void TearDown() override
{
71 destroy_one_pool_pp(pool_name
, rados
);
75 using LegacyClsFIFO
= LegacyFIFO
;
76 using AioLegacyFIFO
= LegacyFIFO
;
79 TEST_F(LegacyClsFIFO
, TestCreate
)
81 auto r
= fifo_create(&dp
, ioctx
, fifo_id
, ""s
, null_yield
);
82 EXPECT_EQ(-EINVAL
, r
);
83 r
= fifo_create(&dp
, ioctx
, fifo_id
, fifo_id
, null_yield
, std::nullopt
,
84 std::nullopt
, false, 0);
85 EXPECT_EQ(-EINVAL
, r
);
86 r
= fifo_create(&dp
, ioctx
, fifo_id
, {}, null_yield
,
87 std::nullopt
, std::nullopt
,
88 false, RCf::default_max_part_size
, 0);
89 EXPECT_EQ(-EINVAL
, r
);
90 r
= fifo_create(&dp
, ioctx
, fifo_id
, fifo_id
, null_yield
);
93 ioctx
.stat(fifo_id
, &size
, nullptr);
95 /* test idempotency */
96 r
= fifo_create(&dp
, ioctx
, fifo_id
, fifo_id
, null_yield
);
98 r
= fifo_create(&dp
, ioctx
, fifo_id
, {}, null_yield
, std::nullopt
,
100 EXPECT_EQ(-EINVAL
, r
);
101 r
= fifo_create(&dp
, ioctx
, fifo_id
, {}, null_yield
, std::nullopt
,
102 "myprefix"sv
, false);
103 EXPECT_EQ(-EINVAL
, r
);
104 r
= fifo_create(&dp
, ioctx
, fifo_id
, "foo"sv
, null_yield
,
105 std::nullopt
, std::nullopt
, false);
106 EXPECT_EQ(-EEXIST
, r
);
109 TEST_F(LegacyClsFIFO
, TestGetInfo
)
111 auto r
= fifo_create(&dp
, ioctx
, fifo_id
, fifo_id
, null_yield
);
113 std::uint32_t part_header_size
;
114 std::uint32_t part_entry_overhead
;
115 r
= RCf::get_meta(&dp
, ioctx
, fifo_id
, std::nullopt
, &info
, &part_header_size
,
116 &part_entry_overhead
, 0, null_yield
);
118 EXPECT_GT(part_header_size
, 0);
119 EXPECT_GT(part_entry_overhead
, 0);
120 EXPECT_FALSE(info
.version
.instance
.empty());
122 r
= RCf::get_meta(&dp
, ioctx
, fifo_id
, info
.version
, &info
, &part_header_size
,
123 &part_entry_overhead
, 0, null_yield
);
126 objv
.instance
= "foo";
128 r
= RCf::get_meta(&dp
, ioctx
, fifo_id
, objv
, &info
, &part_header_size
,
129 &part_entry_overhead
, 0, null_yield
);
130 EXPECT_EQ(-ECANCELED
, r
);
133 TEST_F(LegacyFIFO
, TestOpenDefault
)
135 std::unique_ptr
<RCf::FIFO
> fifo
;
136 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &fifo
, null_yield
);
138 // force reading from backend
139 r
= fifo
->read_meta(&dp
, null_yield
);
141 auto info
= fifo
->meta();
142 EXPECT_EQ(info
.id
, fifo_id
);
145 TEST_F(LegacyFIFO
, TestOpenParams
)
147 const std::uint64_t max_part_size
= 10 * 1024;
148 const std::uint64_t max_entry_size
= 128;
149 auto oid_prefix
= "foo.123."sv
;
151 objv
.instance
= "fooz"s
;
154 /* first successful create */
155 std::unique_ptr
<RCf::FIFO
> f
;
156 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, objv
, oid_prefix
,
157 false, max_part_size
, max_entry_size
);
160 /* force reading from backend */
161 r
= f
->read_meta(&dp
, null_yield
);
162 auto info
= f
->meta();
163 EXPECT_EQ(info
.id
, fifo_id
);
164 EXPECT_EQ(info
.params
.max_part_size
, max_part_size
);
165 EXPECT_EQ(info
.params
.max_entry_size
, max_entry_size
);
166 EXPECT_EQ(info
.version
, objv
);
171 std::pair
<T
, std::string
> decode_entry(const RCf::list_entry
& entry
)
174 auto iter
= entry
.data
.cbegin();
176 return std::make_pair(std::move(val
), entry
.marker
);
181 TEST_F(LegacyFIFO
, TestPushListTrim
)
183 std::unique_ptr
<RCf::FIFO
> f
;
184 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
);
186 static constexpr auto max_entries
= 10u;
187 for (uint32_t i
= 0; i
< max_entries
; ++i
) {
190 r
= f
->push(&dp
, bl
, null_yield
);
194 std::optional
<std::string
> marker
;
195 /* get entries one by one */
196 std::vector
<RCf::list_entry
> result
;
198 for (auto i
= 0u; i
< max_entries
; ++i
) {
200 r
= f
->list(&dp
, 1, marker
, &result
, &more
, null_yield
);
203 bool expected_more
= (i
!= (max_entries
- 1));
204 ASSERT_EQ(expected_more
, more
);
205 ASSERT_EQ(1, result
.size());
208 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
214 /* get all entries at once */
215 std::string markers
[max_entries
];
216 std::uint32_t min_entry
= 0;
217 r
= f
->list(&dp
, max_entries
* 10, std::nullopt
, &result
, &more
, null_yield
);
221 ASSERT_EQ(max_entries
, result
.size());
222 for (auto i
= 0u; i
< max_entries
; ++i
) {
224 std::tie(val
, markers
[i
]) = decode_entry
<std::uint32_t>(result
[i
]);
229 r
= f
->trim(&dp
, markers
[min_entry
], false, null_yield
);
233 r
= f
->list(&dp
, max_entries
* 10, std::nullopt
, &result
, &more
, null_yield
);
236 ASSERT_EQ(max_entries
- min_entry
, result
.size());
238 for (auto i
= min_entry
; i
< max_entries
; ++i
) {
240 std::tie(val
, markers
[i
- min_entry
]) =
241 decode_entry
<std::uint32_t>(result
[i
- min_entry
]);
247 TEST_F(LegacyFIFO
, TestPushTooBig
)
249 static constexpr auto max_part_size
= 2048ull;
250 static constexpr auto max_entry_size
= 128ull;
252 std::unique_ptr
<RCf::FIFO
> f
;
253 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
254 std::nullopt
, false, max_part_size
, max_entry_size
);
257 char buf
[max_entry_size
+ 1];
258 memset(buf
, 0, sizeof(buf
));
261 bl
.append(buf
, sizeof(buf
));
263 r
= f
->push(&dp
, bl
, null_yield
);
264 EXPECT_EQ(-E2BIG
, r
);
268 TEST_F(LegacyFIFO
, TestMultipleParts
)
270 static constexpr auto max_part_size
= 2048ull;
271 static constexpr auto max_entry_size
= 128ull;
272 std::unique_ptr
<RCf::FIFO
> f
;
273 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
274 std::nullopt
, false, max_part_size
,
278 char buf
[max_entry_size
];
279 memset(buf
, 0, sizeof(buf
));
280 const auto [part_header_size
, part_entry_overhead
] =
281 f
->get_part_layout_info();
282 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
283 (max_entry_size
+ part_entry_overhead
));
284 const auto max_entries
= entries_per_part
* 4 + 1;
285 /* push enough entries */
286 for (auto i
= 0u; i
< max_entries
; ++i
) {
289 bl
.append(buf
, sizeof(buf
));
290 r
= f
->push(&dp
, bl
, null_yield
);
294 auto info
= f
->meta();
295 ASSERT_EQ(info
.id
, fifo_id
);
296 /* head should have advanced */
297 ASSERT_GT(info
.head_part_num
, 0);
299 /* list all at once */
300 std::vector
<RCf::list_entry
> result
;
302 r
= f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
304 EXPECT_EQ(false, more
);
305 ASSERT_EQ(max_entries
, result
.size());
307 for (auto i
= 0u; i
< max_entries
; ++i
) {
308 auto& bl
= result
[i
].data
;
309 ASSERT_EQ(i
, *(int *)bl
.c_str());
312 std::optional
<std::string
> marker
;
313 /* get entries one by one */
315 for (auto i
= 0u; i
< max_entries
; ++i
) {
316 r
= f
->list(&dp
, 1, marker
, &result
, &more
, null_yield
);
318 ASSERT_EQ(result
.size(), 1);
319 const bool expected_more
= (i
!= (max_entries
- 1));
320 ASSERT_EQ(expected_more
, more
);
323 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
325 auto& entry
= result
.front();
326 auto& bl
= entry
.data
;
327 ASSERT_EQ(i
, *(int *)bl
.c_str());
328 marker
= entry
.marker
;
331 /* trim one at a time */
333 for (auto i
= 0u; i
< max_entries
; ++i
) {
334 /* read single entry */
335 r
= f
->list(&dp
, 1, marker
, &result
, &more
, null_yield
);
337 ASSERT_EQ(result
.size(), 1);
338 const bool expected_more
= (i
!= (max_entries
- 1));
339 ASSERT_EQ(expected_more
, more
);
341 marker
= result
.front().marker
;
342 r
= f
->trim(&dp
, *marker
, false, null_yield
);
347 ASSERT_EQ(info
.tail_part_num
, i
/ entries_per_part
);
349 /* try to read all again, see how many entries left */
350 r
= f
->list(&dp
, max_entries
, marker
, &result
, &more
, null_yield
);
351 ASSERT_EQ(max_entries
- i
- 1, result
.size());
352 ASSERT_EQ(false, more
);
355 /* tail now should point at head */
357 ASSERT_EQ(info
.head_part_num
, info
.tail_part_num
);
359 RCf::part_info partinfo
;
360 /* check old tails are removed */
361 for (auto i
= 0; i
< info
.tail_part_num
; ++i
) {
362 r
= f
->get_part_info(&dp
, i
, &partinfo
, null_yield
);
363 ASSERT_EQ(-ENOENT
, r
);
365 /* check current tail exists */
366 r
= f
->get_part_info(&dp
, info
.tail_part_num
, &partinfo
, null_yield
);
370 TEST_F(LegacyFIFO
, TestTwoPushers
)
372 static constexpr auto max_part_size
= 2048ull;
373 static constexpr auto max_entry_size
= 128ull;
375 std::unique_ptr
<RCf::FIFO
> f
;
376 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
377 std::nullopt
, false, max_part_size
,
380 char buf
[max_entry_size
];
381 memset(buf
, 0, sizeof(buf
));
383 auto [part_header_size
, part_entry_overhead
] = f
->get_part_layout_info();
384 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
385 (max_entry_size
+ part_entry_overhead
));
386 const auto max_entries
= entries_per_part
* 4 + 1;
387 std::unique_ptr
<RCf::FIFO
> f2
;
388 r
= RCf::FIFO::open(&dp
, ioctx
, fifo_id
, &f2
, null_yield
);
389 std::vector fifos
{&f
, &f2
};
391 for (auto i
= 0u; i
< max_entries
; ++i
) {
394 bl
.append(buf
, sizeof(buf
));
395 auto& f
= *fifos
[i
% fifos
.size()];
396 r
= f
->push(&dp
, bl
, null_yield
);
400 /* list all by both */
401 std::vector
<RCf::list_entry
> result
;
403 r
= f2
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
405 ASSERT_EQ(false, more
);
406 ASSERT_EQ(max_entries
, result
.size());
408 r
= f2
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
410 ASSERT_EQ(false, more
);
411 ASSERT_EQ(max_entries
, result
.size());
413 for (auto i
= 0u; i
< max_entries
; ++i
) {
414 auto& bl
= result
[i
].data
;
415 ASSERT_EQ(i
, *(int *)bl
.c_str());
419 TEST_F(LegacyFIFO
, TestTwoPushersTrim
)
421 static constexpr auto max_part_size
= 2048ull;
422 static constexpr auto max_entry_size
= 128ull;
423 std::unique_ptr
<RCf::FIFO
> f1
;
424 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f1
, null_yield
, std::nullopt
,
425 std::nullopt
, false, max_part_size
,
429 char buf
[max_entry_size
];
430 memset(buf
, 0, sizeof(buf
));
432 auto [part_header_size
, part_entry_overhead
] = f1
->get_part_layout_info();
433 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
434 (max_entry_size
+ part_entry_overhead
));
435 const auto max_entries
= entries_per_part
* 4 + 1;
437 std::unique_ptr
<RCf::FIFO
> f2
;
438 r
= RCf::FIFO::open(&dp
, ioctx
, fifo_id
, &f2
, null_yield
);
441 /* push one entry to f2 and the rest to f1 */
442 for (auto i
= 0u; i
< max_entries
; ++i
) {
445 bl
.append(buf
, sizeof(buf
));
446 auto& f
= (i
< 1 ? f2
: f1
);
447 r
= f
->push(&dp
, bl
, null_yield
);
451 /* trim half by fifo1 */
452 auto num
= max_entries
/ 2;
454 std::vector
<RCf::list_entry
> result
;
456 r
= f1
->list(&dp
, num
, std::nullopt
, &result
, &more
, null_yield
);
458 ASSERT_EQ(true, more
);
459 ASSERT_EQ(num
, result
.size());
461 for (auto i
= 0u; i
< num
; ++i
) {
462 auto& bl
= result
[i
].data
;
463 ASSERT_EQ(i
, *(int *)bl
.c_str());
466 auto& entry
= result
[num
- 1];
467 marker
= entry
.marker
;
468 r
= f1
->trim(&dp
, marker
, false, null_yield
);
469 /* list what's left by fifo2 */
471 const auto left
= max_entries
- num
;
472 f2
->list(&dp
, left
, marker
, &result
, &more
, null_yield
);
473 ASSERT_EQ(left
, result
.size());
474 ASSERT_EQ(false, more
);
476 for (auto i
= num
; i
< max_entries
; ++i
) {
477 auto& bl
= result
[i
- num
].data
;
478 ASSERT_EQ(i
, *(int *)bl
.c_str());
482 TEST_F(LegacyFIFO
, TestPushBatch
)
484 static constexpr auto max_part_size
= 2048ull;
485 static constexpr auto max_entry_size
= 128ull;
487 std::unique_ptr
<RCf::FIFO
> f
;
488 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
489 std::nullopt
, false, max_part_size
,
493 char buf
[max_entry_size
];
494 memset(buf
, 0, sizeof(buf
));
495 auto [part_header_size
, part_entry_overhead
] = f
->get_part_layout_info();
496 auto entries_per_part
= ((max_part_size
- part_header_size
) /
497 (max_entry_size
+ part_entry_overhead
));
498 auto max_entries
= entries_per_part
* 4 + 1; /* enough entries to span multiple parts */
499 std::vector
<cb::list
> bufs
;
500 for (auto i
= 0u; i
< max_entries
; ++i
) {
503 bl
.append(buf
, sizeof(buf
));
506 ASSERT_EQ(max_entries
, bufs
.size());
508 r
= f
->push(&dp
, bufs
, null_yield
);
513 std::vector
<RCf::list_entry
> result
;
515 r
= f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
517 ASSERT_EQ(false, more
);
518 ASSERT_EQ(max_entries
, result
.size());
519 for (auto i
= 0u; i
< max_entries
; ++i
) {
520 auto& bl
= result
[i
].data
;
521 ASSERT_EQ(i
, *(int *)bl
.c_str());
523 auto& info
= f
->meta();
524 ASSERT_EQ(info
.head_part_num
, 4);
527 TEST_F(LegacyFIFO
, TestAioTrim
)
529 static constexpr auto max_part_size
= 2048ull;
530 static constexpr auto max_entry_size
= 128ull;
531 std::unique_ptr
<RCf::FIFO
> f
;
532 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
533 std::nullopt
, false, max_part_size
,
537 char buf
[max_entry_size
];
538 memset(buf
, 0, sizeof(buf
));
539 const auto [part_header_size
, part_entry_overhead
] =
540 f
->get_part_layout_info();
541 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
542 (max_entry_size
+ part_entry_overhead
));
543 const auto max_entries
= entries_per_part
* 4 + 1;
544 /* push enough entries */
545 std::vector
<cb::list
> bufs
;
546 for (auto i
= 0u; i
< max_entries
; ++i
) {
549 bl
.append(buf
, sizeof(buf
));
550 bufs
.push_back(std::move(bl
));
552 ASSERT_EQ(max_entries
, bufs
.size());
554 r
= f
->push(&dp
, bufs
, null_yield
);
557 auto info
= f
->meta();
558 ASSERT_EQ(info
.id
, fifo_id
);
559 /* head should have advanced */
560 ASSERT_GT(info
.head_part_num
, 0);
562 /* list all at once */
563 std::vector
<RCf::list_entry
> result
;
565 r
= f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
567 ASSERT_EQ(false, more
);
568 ASSERT_EQ(max_entries
, result
.size());
570 std::optional
<std::string
> marker
;
571 /* trim one at a time */
575 for (auto i
= 0u; i
< max_entries
; ++i
) {
576 /* read single entry */
577 r
= f
->list(&dp
, 1, marker
, &result
, &more
, null_yield
);
579 ASSERT_EQ(result
.size(), 1);
580 const bool expected_more
= (i
!= (max_entries
- 1));
581 ASSERT_EQ(expected_more
, more
);
583 marker
= result
.front().marker
;
584 std::unique_ptr
<R::AioCompletion
> c(rados
.aio_create_completion(nullptr,
586 f
->trim(&dp
, *marker
, false, c
.get());
587 c
->wait_for_complete();
588 r
= c
->get_return_value();
593 ASSERT_EQ(info
.tail_part_num
, i
/ entries_per_part
);
595 /* try to read all again, see how many entries left */
596 r
= f
->list(&dp
, max_entries
, marker
, &result
, &more
, null_yield
);
597 ASSERT_EQ(max_entries
- i
- 1, result
.size());
598 ASSERT_EQ(false, more
);
601 /* tail now should point at head */
603 ASSERT_EQ(info
.head_part_num
, info
.tail_part_num
);
605 RCf::part_info partinfo
;
606 /* check old tails are removed */
607 for (auto i
= 0; i
< info
.tail_part_num
; ++i
) {
608 r
= f
->get_part_info(&dp
, i
, &partinfo
, null_yield
);
609 ASSERT_EQ(-ENOENT
, r
);
611 /* check current tail exists */
612 r
= f
->get_part_info(&dp
, info
.tail_part_num
, &partinfo
, null_yield
);
616 TEST_F(LegacyFIFO
, TestTrimExclusive
) {
617 std::unique_ptr
<RCf::FIFO
> f
;
618 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
);
620 std::vector
<RCf::list_entry
> result
;
623 static constexpr auto max_entries
= 10u;
624 for (uint32_t i
= 0; i
< max_entries
; ++i
) {
627 f
->push(&dp
, bl
, null_yield
);
630 f
->list(&dp
, 1, std::nullopt
, &result
, &more
, null_yield
);
631 auto [val
, marker
] = decode_entry
<std::uint32_t>(result
.front());
633 f
->trim(&dp
, marker
, true, null_yield
);
636 f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
637 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
639 f
->trim(&dp
, result
[4].marker
, true, null_yield
);
642 f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
643 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
645 f
->trim(&dp
, result
.back().marker
, true, null_yield
);
648 f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, null_yield
);
649 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
650 ASSERT_EQ(result
.size(), 1);
651 ASSERT_EQ(max_entries
- 1, val
);
654 TEST_F(AioLegacyFIFO
, TestPushListTrim
)
656 std::unique_ptr
<RCf::FIFO
> f
;
657 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
);
659 static constexpr auto max_entries
= 10u;
660 for (uint32_t i
= 0; i
< max_entries
; ++i
) {
663 auto c
= R::Rados::aio_create_completion();
665 c
->wait_for_complete();
666 r
= c
->get_return_value();
671 std::optional
<std::string
> marker
;
672 /* get entries one by one */
673 std::vector
<RCf::list_entry
> result
;
675 for (auto i
= 0u; i
< max_entries
; ++i
) {
676 auto c
= R::Rados::aio_create_completion();
677 f
->list(&dp
, 1, marker
, &result
, &more
, c
);
678 c
->wait_for_complete();
679 r
= c
->get_return_value();
683 bool expected_more
= (i
!= (max_entries
- 1));
684 ASSERT_EQ(expected_more
, more
);
685 ASSERT_EQ(1, result
.size());
688 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
694 /* get all entries at once */
695 std::string markers
[max_entries
];
696 std::uint32_t min_entry
= 0;
697 auto c
= R::Rados::aio_create_completion();
698 f
->list(&dp
, max_entries
* 10, std::nullopt
, &result
, &more
, c
);
699 c
->wait_for_complete();
700 r
= c
->get_return_value();
705 ASSERT_EQ(max_entries
, result
.size());
706 for (auto i
= 0u; i
< max_entries
; ++i
) {
708 std::tie(val
, markers
[i
]) = decode_entry
<std::uint32_t>(result
[i
]);
713 c
= R::Rados::aio_create_completion();
714 f
->trim(&dp
, markers
[min_entry
], false, c
);
715 c
->wait_for_complete();
716 r
= c
->get_return_value();
721 c
= R::Rados::aio_create_completion();
722 f
->list(&dp
, max_entries
* 10, std::nullopt
, &result
, &more
, c
);
723 c
->wait_for_complete();
724 r
= c
->get_return_value();
728 ASSERT_EQ(max_entries
- min_entry
, result
.size());
730 for (auto i
= min_entry
; i
< max_entries
; ++i
) {
732 std::tie(val
, markers
[i
- min_entry
]) =
733 decode_entry
<std::uint32_t>(result
[i
- min_entry
]);
739 TEST_F(AioLegacyFIFO
, TestPushTooBig
)
741 static constexpr auto max_part_size
= 2048ull;
742 static constexpr auto max_entry_size
= 128ull;
744 std::unique_ptr
<RCf::FIFO
> f
;
745 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
746 std::nullopt
, false, max_part_size
, max_entry_size
);
749 char buf
[max_entry_size
+ 1];
750 memset(buf
, 0, sizeof(buf
));
753 bl
.append(buf
, sizeof(buf
));
755 auto c
= R::Rados::aio_create_completion();
757 c
->wait_for_complete();
758 r
= c
->get_return_value();
759 ASSERT_EQ(-E2BIG
, r
);
762 c
= R::Rados::aio_create_completion();
763 f
->push(&dp
, std::vector
<cb::list
>{}, c
);
764 c
->wait_for_complete();
765 r
= c
->get_return_value();
771 TEST_F(AioLegacyFIFO
, TestMultipleParts
)
773 static constexpr auto max_part_size
= 2048ull;
774 static constexpr auto max_entry_size
= 128ull;
775 std::unique_ptr
<RCf::FIFO
> f
;
776 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
777 std::nullopt
, false, max_part_size
,
782 auto c
= R::Rados::aio_create_completion();
783 f
->get_head_info(&dp
, [&](int r
, RCf::part_info
&& p
) {
784 ASSERT_TRUE(p
.tag
.empty());
785 ASSERT_EQ(0, p
.magic
);
786 ASSERT_EQ(0, p
.min_ofs
);
787 ASSERT_EQ(0, p
.last_ofs
);
788 ASSERT_EQ(0, p
.next_ofs
);
789 ASSERT_EQ(0, p
.min_index
);
790 ASSERT_EQ(0, p
.max_index
);
791 ASSERT_EQ(ceph::real_time
{}, p
.max_time
);
793 c
->wait_for_complete();
794 r
= c
->get_return_value();
798 char buf
[max_entry_size
];
799 memset(buf
, 0, sizeof(buf
));
800 const auto [part_header_size
, part_entry_overhead
] =
801 f
->get_part_layout_info();
802 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
803 (max_entry_size
+ part_entry_overhead
));
804 const auto max_entries
= entries_per_part
* 4 + 1;
805 /* push enough entries */
806 for (auto i
= 0u; i
< max_entries
; ++i
) {
809 bl
.append(buf
, sizeof(buf
));
810 auto c
= R::Rados::aio_create_completion();
812 c
->wait_for_complete();
813 r
= c
->get_return_value();
818 auto info
= f
->meta();
819 ASSERT_EQ(info
.id
, fifo_id
);
820 /* head should have advanced */
821 ASSERT_GT(info
.head_part_num
, 0);
823 /* list all at once */
824 std::vector
<RCf::list_entry
> result
;
826 auto c
= R::Rados::aio_create_completion();
827 f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, c
);
828 c
->wait_for_complete();
829 r
= c
->get_return_value();
832 EXPECT_EQ(false, more
);
833 ASSERT_EQ(max_entries
, result
.size());
835 for (auto i
= 0u; i
< max_entries
; ++i
) {
836 auto& bl
= result
[i
].data
;
837 ASSERT_EQ(i
, *(int *)bl
.c_str());
840 std::optional
<std::string
> marker
;
841 /* get entries one by one */
843 for (auto i
= 0u; i
< max_entries
; ++i
) {
844 c
= R::Rados::aio_create_completion();
845 f
->list(&dp
, 1, marker
, &result
, &more
, c
);
846 c
->wait_for_complete();
847 r
= c
->get_return_value();
850 ASSERT_EQ(result
.size(), 1);
851 const bool expected_more
= (i
!= (max_entries
- 1));
852 ASSERT_EQ(expected_more
, more
);
855 std::tie(val
, marker
) = decode_entry
<std::uint32_t>(result
.front());
857 auto& entry
= result
.front();
858 auto& bl
= entry
.data
;
859 ASSERT_EQ(i
, *(int *)bl
.c_str());
860 marker
= entry
.marker
;
863 /* trim one at a time */
865 for (auto i
= 0u; i
< max_entries
; ++i
) {
866 /* read single entry */
867 c
= R::Rados::aio_create_completion();
868 f
->list(&dp
, 1, marker
, &result
, &more
, c
);
869 c
->wait_for_complete();
870 r
= c
->get_return_value();
873 ASSERT_EQ(result
.size(), 1);
874 const bool expected_more
= (i
!= (max_entries
- 1));
875 ASSERT_EQ(expected_more
, more
);
877 marker
= result
.front().marker
;
878 c
= R::Rados::aio_create_completion();
879 f
->trim(&dp
, *marker
, false, c
);
880 c
->wait_for_complete();
881 r
= c
->get_return_value();
884 ASSERT_EQ(result
.size(), 1);
888 ASSERT_EQ(info
.tail_part_num
, i
/ entries_per_part
);
890 /* try to read all again, see how many entries left */
891 c
= R::Rados::aio_create_completion();
892 f
->list(&dp
, max_entries
, marker
, &result
, &more
, c
);
893 c
->wait_for_complete();
894 r
= c
->get_return_value();
897 ASSERT_EQ(max_entries
- i
- 1, result
.size());
898 ASSERT_EQ(false, more
);
901 /* tail now should point at head */
903 ASSERT_EQ(info
.head_part_num
, info
.tail_part_num
);
905 /* check old tails are removed */
906 for (auto i
= 0; i
< info
.tail_part_num
; ++i
) {
907 c
= R::Rados::aio_create_completion();
908 RCf::part_info partinfo
;
909 f
->get_part_info(i
, &partinfo
, c
);
910 c
->wait_for_complete();
911 r
= c
->get_return_value();
913 ASSERT_EQ(-ENOENT
, r
);
915 /* check current tail exists */
916 std::uint64_t next_ofs
;
918 c
= R::Rados::aio_create_completion();
919 RCf::part_info partinfo
;
920 f
->get_part_info(info
.tail_part_num
, &partinfo
, c
);
921 c
->wait_for_complete();
922 r
= c
->get_return_value();
924 next_ofs
= partinfo
.next_ofs
;
928 c
= R::Rados::aio_create_completion();
929 f
->get_head_info(&dp
, [&](int r
, RCf::part_info
&& p
) {
930 ASSERT_EQ(next_ofs
, p
.next_ofs
);
932 c
->wait_for_complete();
933 r
= c
->get_return_value();
938 TEST_F(AioLegacyFIFO
, TestTwoPushers
)
940 static constexpr auto max_part_size
= 2048ull;
941 static constexpr auto max_entry_size
= 128ull;
943 std::unique_ptr
<RCf::FIFO
> f
;
944 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
945 std::nullopt
, false, max_part_size
,
948 char buf
[max_entry_size
];
949 memset(buf
, 0, sizeof(buf
));
951 auto [part_header_size
, part_entry_overhead
] = f
->get_part_layout_info();
952 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
953 (max_entry_size
+ part_entry_overhead
));
954 const auto max_entries
= entries_per_part
* 4 + 1;
955 std::unique_ptr
<RCf::FIFO
> f2
;
956 r
= RCf::FIFO::open(&dp
, ioctx
, fifo_id
, &f2
, null_yield
);
957 std::vector fifos
{&f
, &f2
};
959 for (auto i
= 0u; i
< max_entries
; ++i
) {
962 bl
.append(buf
, sizeof(buf
));
963 auto& f
= *fifos
[i
% fifos
.size()];
964 auto c
= R::Rados::aio_create_completion();
966 c
->wait_for_complete();
967 r
= c
->get_return_value();
972 /* list all by both */
973 std::vector
<RCf::list_entry
> result
;
975 auto c
= R::Rados::aio_create_completion();
976 f2
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, c
);
977 c
->wait_for_complete();
978 r
= c
->get_return_value();
981 ASSERT_EQ(false, more
);
982 ASSERT_EQ(max_entries
, result
.size());
984 c
= R::Rados::aio_create_completion();
985 f2
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, c
);
986 c
->wait_for_complete();
987 r
= c
->get_return_value();
990 ASSERT_EQ(false, more
);
991 ASSERT_EQ(max_entries
, result
.size());
993 for (auto i
= 0u; i
< max_entries
; ++i
) {
994 auto& bl
= result
[i
].data
;
995 ASSERT_EQ(i
, *(int *)bl
.c_str());
999 TEST_F(AioLegacyFIFO
, TestTwoPushersTrim
)
1001 static constexpr auto max_part_size
= 2048ull;
1002 static constexpr auto max_entry_size
= 128ull;
1003 std::unique_ptr
<RCf::FIFO
> f1
;
1004 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f1
, null_yield
, std::nullopt
,
1005 std::nullopt
, false, max_part_size
,
1009 char buf
[max_entry_size
];
1010 memset(buf
, 0, sizeof(buf
));
1012 auto [part_header_size
, part_entry_overhead
] = f1
->get_part_layout_info();
1013 const auto entries_per_part
= ((max_part_size
- part_header_size
) /
1014 (max_entry_size
+ part_entry_overhead
));
1015 const auto max_entries
= entries_per_part
* 4 + 1;
1017 std::unique_ptr
<RCf::FIFO
> f2
;
1018 r
= RCf::FIFO::open(&dp
, ioctx
, fifo_id
, &f2
, null_yield
);
1021 /* push one entry to f2 and the rest to f1 */
1022 for (auto i
= 0u; i
< max_entries
; ++i
) {
1025 bl
.append(buf
, sizeof(buf
));
1026 auto& f
= (i
< 1 ? f2
: f1
);
1027 auto c
= R::Rados::aio_create_completion();
1028 f
->push(&dp
, bl
, c
);
1029 c
->wait_for_complete();
1030 r
= c
->get_return_value();
1035 /* trim half by fifo1 */
1036 auto num
= max_entries
/ 2;
1038 std::vector
<RCf::list_entry
> result
;
1040 auto c
= R::Rados::aio_create_completion();
1041 f1
->list(&dp
, num
, std::nullopt
, &result
, &more
, c
);
1042 c
->wait_for_complete();
1043 r
= c
->get_return_value();
1046 ASSERT_EQ(true, more
);
1047 ASSERT_EQ(num
, result
.size());
1049 for (auto i
= 0u; i
< num
; ++i
) {
1050 auto& bl
= result
[i
].data
;
1051 ASSERT_EQ(i
, *(int *)bl
.c_str());
1054 auto& entry
= result
[num
- 1];
1055 marker
= entry
.marker
;
1056 c
= R::Rados::aio_create_completion();
1057 f1
->trim(&dp
, marker
, false, c
);
1058 c
->wait_for_complete();
1059 r
= c
->get_return_value();
1062 /* list what's left by fifo2 */
1064 const auto left
= max_entries
- num
;
1065 c
= R::Rados::aio_create_completion();
1066 f2
->list(&dp
, left
, marker
, &result
, &more
, c
);
1067 c
->wait_for_complete();
1068 r
= c
->get_return_value();
1071 ASSERT_EQ(left
, result
.size());
1072 ASSERT_EQ(false, more
);
1074 for (auto i
= num
; i
< max_entries
; ++i
) {
1075 auto& bl
= result
[i
- num
].data
;
1076 ASSERT_EQ(i
, *(int *)bl
.c_str());
1080 TEST_F(AioLegacyFIFO
, TestPushBatch
)
1082 static constexpr auto max_part_size
= 2048ull;
1083 static constexpr auto max_entry_size
= 128ull;
1085 std::unique_ptr
<RCf::FIFO
> f
;
1086 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
, std::nullopt
,
1087 std::nullopt
, false, max_part_size
,
1091 char buf
[max_entry_size
];
1092 memset(buf
, 0, sizeof(buf
));
1093 auto [part_header_size
, part_entry_overhead
] = f
->get_part_layout_info();
1094 auto entries_per_part
= ((max_part_size
- part_header_size
) /
1095 (max_entry_size
+ part_entry_overhead
));
1096 auto max_entries
= entries_per_part
* 4 + 1; /* enough entries to span multiple parts */
1097 std::vector
<cb::list
> bufs
;
1098 for (auto i
= 0u; i
< max_entries
; ++i
) {
1101 bl
.append(buf
, sizeof(buf
));
1104 ASSERT_EQ(max_entries
, bufs
.size());
1106 auto c
= R::Rados::aio_create_completion();
1107 f
->push(&dp
, bufs
, c
);
1108 c
->wait_for_complete();
1109 r
= c
->get_return_value();
1115 std::vector
<RCf::list_entry
> result
;
1117 c
= R::Rados::aio_create_completion();
1118 f
->list(&dp
, max_entries
, std::nullopt
, &result
, &more
, c
);
1119 c
->wait_for_complete();
1120 r
= c
->get_return_value();
1123 ASSERT_EQ(false, more
);
1124 ASSERT_EQ(max_entries
, result
.size());
1125 for (auto i
= 0u; i
< max_entries
; ++i
) {
1126 auto& bl
= result
[i
].data
;
1127 ASSERT_EQ(i
, *(int *)bl
.c_str());
1129 auto& info
= f
->meta();
1130 ASSERT_EQ(info
.head_part_num
, 4);
1133 TEST_F(LegacyFIFO
, TrimAll
)
1135 std::unique_ptr
<RCf::FIFO
> f
;
1136 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
);
1138 static constexpr auto max_entries
= 10u;
1139 for (uint32_t i
= 0; i
< max_entries
; ++i
) {
1142 r
= f
->push(&dp
, bl
, null_yield
);
1146 /* trim one entry */
1147 r
= f
->trim(&dp
, RCf::marker::max().to_string(), false, null_yield
);
1148 ASSERT_EQ(-ENODATA
, r
);
1150 std::vector
<RCf::list_entry
> result
;
1152 r
= f
->list(&dp
, 1, std::nullopt
, &result
, &more
, null_yield
);
1154 ASSERT_TRUE(result
.empty());
1157 TEST_F(LegacyFIFO
, AioTrimAll
)
1159 std::unique_ptr
<RCf::FIFO
> f
;
1160 auto r
= RCf::FIFO::create(&dp
, ioctx
, fifo_id
, &f
, null_yield
);
1162 static constexpr auto max_entries
= 10u;
1163 for (uint32_t i
= 0; i
< max_entries
; ++i
) {
1166 r
= f
->push(&dp
, bl
, null_yield
);
1170 auto c
= R::Rados::aio_create_completion();
1171 f
->trim(&dp
, RCf::marker::max().to_string(), false, c
);
1172 c
->wait_for_complete();
1173 r
= c
->get_return_value();
1175 ASSERT_EQ(-ENODATA
, r
);
1177 std::vector
<RCf::list_entry
> result
;
1179 r
= f
->list(&dp
, 1, std::nullopt
, &result
, &more
, null_yield
);
1181 ASSERT_TRUE(result
.empty());