ceph/src/test/test_denc.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph distributed storage system
   5  *
   6  * Copyright (C) 2016 Red Hat
   7  *
   8  * Author: Sage Weil <sage@redhat.com>
   9  *
  10  *  This library is free software; you can redistribute it and/or
  11  *  modify it under the terms of the GNU Lesser General Public
  12  *  License as published by the Free Software Foundation; either
  13  *  version 2.1 of the License, or (at your option) any later version.
  14  *
  15  */
  16
  17 #include <stdio.h>
  18 #include <numeric>
  19
  20 #include "global/global_init.h"
  21 #include "common/ceph_argparse.h"
  22 #include "global/global_context.h"
  23 #include "gtest/gtest.h"
  24
  25 #include "include/denc.h"
  26
  27 using namespace std;
  28
  29 // test helpers
  30
  31 template<typename T>
  32 void test_encode_decode(T v) {
  33   bufferlist bl;
  34   encode(v, bl);
  35   auto p = bl.cbegin();
  36   T out;
  37   decode(out, p);
  38   ASSERT_EQ(v, out);
  39 }
  40
  41 template<typename T>
  42 void test_denc(T v) {
  43   // estimate
  44   size_t s = 0;
  45   denc(v, s);
  46   ASSERT_NE(s, 0u);
  47
  48   // encode
  49   bufferlist bl;
  50   {
  51     auto a = bl.get_contiguous_appender(s);
  52     denc(v, a);
  53   }
  54   ASSERT_LE(bl.length(), s);
  55
  56   // decode
  57   bl.rebuild();
  58   T out;
  59   auto bpi = bl.front().begin();
  60   denc(out, bpi);
  61   ASSERT_EQ(v, out);
  62   ASSERT_EQ(bpi.get_pos(), bl.c_str() + bl.length());
  63
  64   // test glue
  65   test_encode_decode(v);
  66 }
  67
  68 template<typename T>
  69 void test_encode_decode_featured(T v) {
  70   bufferlist bl;
  71   encode(v, bl, 123);
  72   auto p = bl.cbegin();
  73   T out;
  74   decode(out, p);
  75   ASSERT_EQ(v, out);
  76 }
  77
  78 template<typename T>
  79 void test_denc_featured(T v) {
  80   // estimate
  81   size_t s = 0;
  82   denc(v, s, 0);
  83   ASSERT_GT(s, 0u);
  84
  85   // encode
  86   bufferlist bl;
  87   {
  88     auto a = bl.get_contiguous_appender(s);
  89     denc(v, a, 1);
  90   }
  91   ASSERT_LE(bl.length(), s);
  92
  93   // decode
  94   bl.rebuild();
  95   T out;
  96   auto bpi = bl.front().begin();
  97   denc(out, bpi, 1);
  98   ASSERT_EQ(v, out);
  99   ASSERT_EQ(bpi.get_pos(), bl.c_str() + bl.length());
 100
 101   // test glue
 102   test_encode_decode_featured(v);
 103 }
 104
 105
 106 // hooks to count bound calls
 107
 108 struct counts_t {
 109   int num_bound_encode = 0;
 110   int num_encode = 0;
 111   int num_decode = 0;
 112   void reset() {
 113     num_bound_encode = 0;
 114     num_encode = 0;
 115     num_decode = 0;
 116   }
 117 } counts;
 118
 119 struct denc_counter_t {
 120   void bound_encode(size_t& p) const {
 121     ++counts.num_bound_encode;
 122     ++p;  // denc.h does not like 0-length objects
 123   }
 124   void encode(buffer::list::contiguous_appender& p) const {
 125     p.append("a", 1);
 126     ++counts.num_encode;
 127   }
 128   void decode(buffer::ptr::const_iterator &p) {
 129     p += 1;
 130     ++counts.num_decode;
 131   }
 132 };
 133 WRITE_CLASS_DENC(denc_counter_t)
 134
 135 struct denc_counter_bounded_t {
 136   void bound_encode(size_t& p) const {
 137     ++counts.num_bound_encode;
 138     ++p;  // denc.h does not like 0-length objects
 139   }
 140   void encode(buffer::list::contiguous_appender& p) const {
 141     p.append("a", 1);
 142     ++counts.num_encode;
 143   }
 144   void decode(buffer::ptr::const_iterator &p) {
 145     p += 1;
 146     ++counts.num_decode;
 147   }
 148 };
 149 WRITE_CLASS_DENC_BOUNDED(denc_counter_bounded_t)
 150
 151 TEST(denc, denc_counter)
 152 {
 153   denc_counter_t single, single2;
 154   {
 155     bufferlist bl;
 156     encode(single, bl);
 157     decode(single2, bl);
 158   }
 159   ASSERT_EQ(counts.num_bound_encode, 1);
 160   ASSERT_EQ(counts.num_encode, 1);
 161   ASSERT_EQ(counts.num_decode, 1);
 162   counts.reset();
 163 }
 164
 165 TEST(denc, simple)
 166 {
 167   test_denc((uint8_t)4);
 168   test_denc((int8_t)-5);
 169   test_denc((uint16_t)6);
 170   test_denc((int16_t)-7);
 171   test_denc((uint32_t)8);
 172   test_denc((int32_t)-9);
 173   test_denc((uint64_t)10);
 174   test_denc((int64_t)-11);
 175 }
 176
 177 TEST(denc, string)
 178 {
 179   string a, b("hi"), c("multi\nline\n");
 180   test_denc(a);
 181   test_denc(b);
 182   test_denc(c);
 183 }
 184
 185 struct legacy_t {
 186   int32_t a = 1;
 187   void encode(bufferlist& bl) const {
 188     using ceph::encode;
 189     encode(a, bl);
 190   }
 191   void decode(bufferlist::const_iterator& p) {
 192     using ceph::decode;
 193     decode(a, p);
 194   }
 195   legacy_t() {}
 196   explicit legacy_t(int32_t i) : a(i) {}
 197   friend bool operator<(const legacy_t& l, const legacy_t& r) {
 198     return l.a < r.a;
 199   }
 200   friend bool operator==(const legacy_t& l, const legacy_t& r) {
 201     return l.a == r.a;
 202   }
 203 };
 204 WRITE_CLASS_ENCODER(legacy_t)
 205
 206 template<template<class> class C>
 207 void test_common_veclist(const char* c) {
 208   {
 209     cout << c << "<std::string>" << std::endl;
 210     C<std::string> s;
 211     s.push_back("foo");
 212     s.push_back("bar");
 213     s.push_back("baz");
 214     counts.reset();
 215     test_denc(s);
 216   }
 217   {
 218     cout << c << "<int32_t>" << std::endl;
 219     C<int32_t> s;
 220     s.push_back(1);
 221     s.push_back(2);
 222     s.push_back(3);
 223     test_denc(s);
 224   }
 225   {
 226     cout << c << "<legacy_t>" << std::endl;
 227     C<legacy_t> s;
 228     s.push_back(legacy_t(1));
 229     s.push_back(legacy_t(2));
 230     test_encode_decode(s);
 231   }
 232 }
 233
 234 // We only care about specializing the type, all other template
 235 // parameters should have the default values. (Like first-class
 236 // functions, first-class templates do not bring their defaults.)
 237
 238 template<typename T>
 239 using default_vector = std::vector<T>;
 240
 241 TEST(denc, vector)
 242 {
 243   test_common_veclist<default_vector>("std::vector");
 244   {
 245     counts.reset();
 246     vector<denc_counter_t> v, v2;
 247     v.resize(100);
 248     {
 249       bufferlist bl;
 250       encode(v, bl);
 251       decode(v2, bl);
 252     }
 253     ASSERT_EQ(counts.num_bound_encode, 100);
 254     ASSERT_EQ(counts.num_encode, 100);
 255     ASSERT_EQ(counts.num_decode, 100);
 256   }
 257   {
 258     counts.reset();
 259     vector<denc_counter_bounded_t> v, v2;
 260     v.resize(100);
 261     {
 262       bufferlist bl;
 263       encode(v, bl);
 264       decode(v2, bl);
 265     }
 266     ASSERT_EQ(counts.num_bound_encode, 1);
 267     ASSERT_EQ(counts.num_encode, 100);
 268     ASSERT_EQ(counts.num_decode, 100);
 269   }
 270 }
 271
 272 template<typename T>
 273 using default_list = std::list<T>;
 274
 275 TEST(denc, list)
 276 {
 277   test_common_veclist<default_list>("std::list");
 278   {
 279     counts.reset();
 280     list<denc_counter_bounded_t> l, l2;
 281     for (unsigned i=0; i<100; ++i) {
 282       l.emplace_back(denc_counter_bounded_t());
 283     }
 284     {
 285       bufferlist bl;
 286       encode(l, bl);
 287       decode(l2, bl);
 288     }
 289     ASSERT_EQ(counts.num_bound_encode, 1);
 290     ASSERT_EQ(counts.num_encode, 100);
 291     ASSERT_EQ(counts.num_decode, 100);
 292   }
 293 }
 294
 295 template<template<class> class C>
 296 void test_setlike(const char* c) {
 297   {
 298     cout << c << "<std::string>" << std::endl;
 299     C<std::string> s;
 300     s.insert("foo");
 301     s.insert("bar");
 302     s.insert("baz");
 303     test_denc(s);
 304   }
 305   {
 306     cout << c << "<int32_t>" << std::endl;
 307     C<int32_t> s;
 308     s.insert(1);
 309     s.insert(2);
 310     s.insert(3);
 311     test_denc(s);
 312   }
 313   {
 314     cout << c << "<legacy_t>" << std::endl;
 315     C<legacy_t> s;
 316     s.insert(legacy_t(1));
 317     s.insert(legacy_t(2));
 318     test_encode_decode(s);
 319   }
 320 }
 321
 322 template<typename T>
 323 using default_set = std::set<T>;
 324
 325 TEST(denc, set)
 326 {
 327   test_setlike<default_set>("std::set");
 328 }
 329
 330 template<typename T>
 331 using default_flat_set= boost::container::flat_set<T>;
 332
 333 TEST(denc, flat_set)
 334 {
 335   test_setlike<default_flat_set>("std::set");
 336 }
 337
 338 struct foo_t {
 339   int32_t a = 0;
 340   uint64_t b = 123;
 341
 342   DENC(foo_t, v, p) {
 343     DENC_START(1, 1, p);
 344     ::denc(v.a, p);
 345     ::denc(v.b, p);
 346     DENC_FINISH(p);
 347   }
 348
 349   friend bool operator==(const foo_t& l, const foo_t& r) {
 350     return l.a == r.a && l.b == r.b;
 351   }
 352 };
 353 WRITE_CLASS_DENC_BOUNDED(foo_t)
 354
 355 struct foo2_t {
 356   int32_t c = 0;
 357   uint64_t d = 123;
 358
 359   DENC(foo2_t, v, p) {
 360     DENC_START(1, 1, p);
 361     ::denc(v.c, p);
 362     ::denc(v.d, p);
 363     DENC_FINISH(p);
 364   }
 365
 366   friend bool operator==(const foo2_t& l, const foo2_t& r) {
 367     return l.c == r.c && l.d == r.d;
 368   }
 369 };
 370 WRITE_CLASS_DENC_BOUNDED(foo2_t)
 371
 372
 373 struct bar_t {
 374   int32_t a = 0;
 375   uint64_t b = 123;
 376
 377   DENC_FEATURED(bar_t, v, p, f) {
 378     ::denc(v.a, p, f);
 379     ::denc(v.b, p, f);
 380   }
 381
 382   friend bool operator==(const bar_t& l, const bar_t& r) {
 383     return l.a == r.a && l.b == r.b;
 384   }
 385 };
 386 WRITE_CLASS_DENC_FEATURED_BOUNDED(bar_t)
 387
 388 TEST(denc, foo)
 389 {
 390   foo_t a;
 391   test_denc(a);
 392   bufferlist bl;
 393   encode(a, bl);
 394   bl.hexdump(cout);
 395 }
 396
 397 TEST(denc, bar)
 398 {
 399   bar_t a;
 400   test_denc_featured(a);
 401 }
 402
 403
 404
 405 TEST(denc, pair)
 406 {
 407   pair<int32_t,std::string> p;
 408   bufferlist bl;
 409   {
 410     auto a = bl.get_contiguous_appender(1000);
 411     denc(p, a);
 412     encode(p, bl);
 413   }
 414
 415   pair<int32_t,legacy_t> lp;
 416   encode(lp, bl);
 417 }
 418
 419 template<template<class, class> class C>
 420 void test_common_maplike(const char* c) {
 421   {
 422     cout << c << "<std::string, foo_t>" << std::endl;
 423     C<string, foo_t> s;
 424     s["foo"] = foo_t();
 425     s["bar"] = foo_t();
 426     s["baz"] = foo_t();
 427     test_denc(s);
 428   }
 429   {
 430     cout << c << "<std::string, bar_t>" << std::endl;
 431     C<string, bar_t> s;
 432     s["foo"] = bar_t();
 433     s["bar"] = bar_t();
 434     s["baz"] = bar_t();
 435     test_denc_featured(s);
 436   }
 437   {
 438     cout << c << "<std::string, legacy_t>" << std::endl;
 439     C<std::string, legacy_t> s;
 440     s["foo"] = legacy_t(1);
 441     s["bar"] = legacy_t(2);
 442     test_encode_decode(s);
 443   }
 444 }
 445
 446 template<typename U, typename V>
 447 using default_map = std::map<U, V>;
 448
 449 TEST(denc, map)
 450 {
 451   test_common_maplike<default_map>("std::map");
 452 }
 453
 454 template<typename U, typename V>
 455 using default_flat_map = boost::container::flat_map<U, V>;
 456
 457 TEST(denc, flat_map)
 458 {
 459   test_common_maplike<default_flat_map>("boost::container::flat_map");
 460 }
 461
 462 TEST(denc, bufferptr_shallow_and_deep) {
 463   // shallow encode
 464   int32_t i = 1;
 465   bufferptr p1("foo", 3);
 466   bufferlist bl;
 467   {
 468     auto a = bl.get_contiguous_appender(100);
 469     denc(i, a);
 470     denc(p1, a);
 471     denc(i, a);
 472   }
 473   cout << "bl is " << bl << std::endl;
 474   bl.hexdump(cout);
 475   ASSERT_EQ(3u, bl.get_num_buffers());
 476
 477   bufferlist bl2 = bl;
 478   bl.rebuild();
 479   bl2.rebuild();
 480
 481   // shallow decode
 482   {
 483     cout << "bl is " << bl << std::endl;
 484     bl.hexdump(cout);
 485     auto p = bl.front().begin();
 486     bufferptr op;
 487     int32_t i;
 488     denc(i, p);
 489     denc(op, p);
 490     denc(i, p);
 491     ASSERT_EQ(3u, op.length());
 492     ASSERT_EQ('f', op[0]);
 493     memset(bl.c_str(), 0, bl.length());
 494     ASSERT_EQ(0, op[0]);
 495   }
 496
 497   // deep decode
 498   {
 499     cout << "bl is " << bl2 << std::endl;
 500     bl2.hexdump(cout);
 501     auto p = bl2.front().begin_deep();
 502     bufferptr op;
 503     int32_t i;
 504     denc(i, p);
 505     denc(op, p);
 506     denc(i, p);
 507     ASSERT_EQ('f', op[0]);
 508     memset(bl2.c_str(), 1, bl2.length());
 509     ASSERT_EQ('f', op[0]);
 510   }
 511 }
 512
 513 TEST(denc, array)
 514 {
 515   {
 516     cout << "std::array<std::string, 3>" << std::endl;
 517     std::array<std::string, 3> s = { "foo", "bar", "baz" };
 518     counts.reset();
 519     test_denc(s);
 520   }
 521   {
 522     cout << "std::array<uint32_t, 3>" << std::endl;
 523     std::array<uint32_t, 3> s = { 1UL, 2UL, 3UL };
 524     test_denc(s);
 525   }
 526 }
 527
 528 TEST(denc, tuple)
 529 {
 530   {
 531     cout << "std::tuple<uint64_t, uint32_t>" << std::endl;
 532     std::tuple<uint64_t, uint32_t> s(100ULL, 97UL);
 533     counts.reset();
 534     test_denc(s);
 535   }
 536   {
 537     cout << "std::tuple<std::string, uint3_t>" << std::endl;
 538     std::tuple<std::string, uint32_t> s("foo", 97);
 539     test_denc(s);
 540   }
 541   {
 542     cout << "std::tuple<std::string, std::set<uint32_t>>" << std::endl;
 543     std::tuple<std::string, std::set<uint32_t>> s(
 544       "bar", std::set<uint32_t>{uint32_t(1), uint32_t(2), uint32_t(3)});
 545     test_denc(s);
 546   }
 547 }
 548
 549 TEST(denc, optional)
 550 {
 551   {
 552     cout << "boost::optional<uint64_t>" << std::endl;
 553     boost::optional<uint64_t> s = 97, t = boost::none;
 554     counts.reset();
 555     test_denc(s);
 556     test_denc(t);
 557   }
 558   {
 559     cout << "boost::optional<std::string>" << std::endl;
 560     boost::optional<std::string> s = std::string("Meow"), t = boost::none;
 561     counts.reset();
 562     test_denc(s);
 563     test_denc(t);
 564   }
 565   {
 566     size_t s = 0;
 567     denc(boost::none, s);
 568     ASSERT_NE(s, 0u);
 569
 570     // encode
 571     bufferlist bl;
 572     {
 573       auto a = bl.get_contiguous_appender(s);
 574       denc(boost::none, a);
 575     }
 576     ASSERT_LE(bl.length(), s);
 577
 578     bl.rebuild();
 579     boost::optional<uint32_t> out = 5;
 580     auto bpi = bl.front().begin();
 581     denc(out, bpi);
 582     ASSERT_FALSE(!!out);
 583     ASSERT_EQ(bpi.get_pos(), bl.c_str() + bl.length());
 584   }
 585 }
 586
 587 TEST(denc, stdoptional)
 588 {
 589   {
 590     cout << "std::optional<uint64_t>" << std::endl;
 591     std::optional<uint64_t> s = 97, t = std::nullopt;
 592     counts.reset();
 593     test_denc(s);
 594     test_denc(t);
 595   }
 596   {
 597     cout << "std::optional<std::string>" << std::endl;
 598     std::optional<std::string> s = std::string("Meow"), t = std::nullopt;
 599     counts.reset();
 600     test_denc(s);
 601     test_denc(t);
 602   }
 603   {
 604     size_t s = 0;
 605     denc(std::nullopt, s);
 606     ASSERT_NE(s, 0u);
 607
 608     // encode
 609     bufferlist bl;
 610     {
 611       auto a = bl.get_contiguous_appender(s);
 612       denc(std::nullopt, a);
 613     }
 614     ASSERT_LE(bl.length(), s);
 615
 616     bl.rebuild();
 617     std::optional<uint32_t> out = 5;
 618     auto bpi = bl.front().begin();
 619     denc(out, bpi);
 620     ASSERT_FALSE(!!out);
 621     ASSERT_EQ(bpi.get_pos(), bl.c_str() + bl.length());
 622   }
 623 }
 624
 625 // unlike legacy_t, Legacy supports denc() also.
 626 struct Legacy {
 627   static unsigned n_denc;
 628   static unsigned n_decode;
 629   uint8_t value = 0;
 630   DENC(Legacy, v, p) {
 631     n_denc++;
 632     denc(v.value, p);
 633   }
 634   void decode(buffer::list::const_iterator& p) {
 635     n_decode++;
 636     using ceph::decode;
 637     decode(value, p);
 638   }
 639   static void reset() {
 640     n_denc = n_decode = 0;
 641   }
 642   static bufferlist encode_n(unsigned n, const vector<unsigned>& segments);
 643 };
 644 WRITE_CLASS_DENC(Legacy)
 645 unsigned Legacy::n_denc = 0;
 646 unsigned Legacy::n_decode = 0;
 647
 648 bufferlist Legacy::encode_n(unsigned n, const vector<unsigned>& segments) {
 649   vector<Legacy> v;
 650   for (unsigned i = 0; i < n; i++) {
 651     v.push_back(Legacy());
 652   }
 653   bufferlist bl(n * sizeof(uint8_t));
 654   using ceph::encode;
 655   encode(v, bl);
 656   bufferlist segmented;
 657   auto p = bl.begin();
 658
 659   auto sum = std::accumulate(segments.begin(), segments.end(), 0u);
 660   ceph_assert(sum != 0u);
 661   for (auto i : segments) {
 662     buffer::ptr seg;
 663     p.copy_deep(bl.length() * i / sum, seg);
 664     segmented.push_back(seg);
 665   }
 666   p.copy_all(segmented);
 667   return segmented;
 668 }
 669
 670 TEST(denc, no_copy_if_segmented_and_lengthy)
 671 {
 672   static_assert(_denc::has_legacy_denc<Legacy>::value,
 673                 "Legacy do have legacy denc");
 674   {
 675     // use denc() which shallow_copy() if the buffer is small
 676     constexpr unsigned N_COPIES = 42;
 677     const vector<unsigned> segs{50, 50}; // half-half
 678     bufferlist segmented = Legacy::encode_n(N_COPIES, segs);
 679     ASSERT_GT(segmented.get_num_buffers(), 1u);
 680     ASSERT_LT(segmented.length(), CEPH_PAGE_SIZE);
 681     auto p = segmented.cbegin();
 682     vector<Legacy> v;
 683     // denc() is shared by encode() and decode(), so reset() right before
 684     // decode()
 685     Legacy::reset();
 686     decode(v, p);
 687     ASSERT_EQ(N_COPIES, v.size());
 688     ASSERT_EQ(N_COPIES, Legacy::n_denc);
 689     ASSERT_EQ(0u, Legacy::n_decode);
 690   }
 691   {
 692     // use denc() which shallow_copy() if the buffer is not segmented and large
 693     const unsigned N_COPIES = CEPH_PAGE_SIZE * 2;
 694     const vector<unsigned> segs{100};
 695     bufferlist segmented = Legacy::encode_n(N_COPIES, segs);
 696     ASSERT_EQ(segmented.get_num_buffers(), 1u);
 697     ASSERT_GT(segmented.length(), CEPH_PAGE_SIZE);
 698     auto p = segmented.cbegin();
 699     vector<Legacy> v;
 700     Legacy::reset();
 701     decode(v, p);
 702     ASSERT_EQ(N_COPIES, v.size());
 703     ASSERT_EQ(N_COPIES, Legacy::n_denc);
 704     ASSERT_EQ(0u, Legacy::n_decode);
 705   }
 706   {
 707     // use denc() which shallow_copy() if the buffer is segmented and large,
 708     // but the total size of the chunks to be decoded is smallish.
 709     bufferlist large_bl = Legacy::encode_n(CEPH_PAGE_SIZE * 2, {50, 50});
 710     bufferlist small_bl = Legacy::encode_n(100, {50, 50});
 711     bufferlist segmented;
 712     segmented.append(large_bl);
 713     segmented.append(small_bl);
 714     ASSERT_GT(segmented.get_num_buffers(), 1u);
 715     ASSERT_GT(segmented.length(), CEPH_PAGE_SIZE);
 716     auto p = segmented.cbegin();
 717     p += large_bl.length();
 718     ASSERT_LT(segmented.length() - p.get_off(), CEPH_PAGE_SIZE);
 719     vector<Legacy> v;
 720     Legacy::reset();
 721     decode(v, p);
 722     ASSERT_EQ(Legacy::n_denc, 100u);
 723     ASSERT_EQ(0u, Legacy::n_decode);
 724   }
 725   {
 726     // use decode() which avoids deep copy if the buffer is segmented and large
 727     bufferlist small_bl = Legacy::encode_n(100, {50, 50});
 728     bufferlist large_bl = Legacy::encode_n(CEPH_PAGE_SIZE * 2, {50, 50});
 729     bufferlist segmented;
 730     segmented.append(small_bl);
 731     segmented.append(large_bl);
 732     ASSERT_GT(segmented.get_num_buffers(), 1u);
 733     ASSERT_GT(segmented.length(), CEPH_PAGE_SIZE);
 734     auto p = segmented.cbegin();
 735     p += small_bl.length();
 736     ASSERT_GT(segmented.length() - p.get_off(), CEPH_PAGE_SIZE);
 737     vector<Legacy> v;
 738     Legacy::reset();
 739     decode(v, p);
 740     ASSERT_EQ(0u, Legacy::n_denc);
 741     ASSERT_EQ(CEPH_PAGE_SIZE * 2, Legacy::n_decode);
 742   }
 743 }