ceph/src/rocksdb/utilities/cassandra/cassandra_format_test.cc

   1 // Copyright (c) 2017-present, Facebook, Inc.  All rights reserved.
   2 //  This source code is licensed under both the GPLv2 (found in the
   3 //  COPYING file in the root directory) and Apache 2.0 License
   4 //  (found in the LICENSE.Apache file in the root directory).
   5
   6 #include <cstring>
   7 #include <memory>
   8 #include "util/testharness.h"
   9 #include "utilities/cassandra/format.h"
  10 #include "utilities/cassandra/serialize.h"
  11 #include "utilities/cassandra/test_utils.h"
  12
  13 using namespace rocksdb::cassandra;
  14
  15 namespace rocksdb {
  16 namespace cassandra {
  17
  18 TEST(ColumnTest, Column) {
  19   char data[4] = {'d', 'a', 't', 'a'};
  20   int8_t mask = 0;
  21   int8_t index = 1;
  22   int64_t timestamp = 1494022807044;
  23   Column c = Column(mask, index, timestamp, sizeof(data), data);
  24
  25   EXPECT_EQ(c.Index(), index);
  26   EXPECT_EQ(c.Timestamp(), timestamp);
  27   EXPECT_EQ(c.Size(), 14 + sizeof(data));
  28
  29   // Verify the serialization.
  30   std::string dest;
  31   dest.reserve(c.Size() * 2);
  32   c.Serialize(&dest);
  33
  34   EXPECT_EQ(dest.size(), c.Size());
  35   std::size_t offset = 0;
  36   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask);
  37   offset += sizeof(int8_t);
  38   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index);
  39   offset += sizeof(int8_t);
  40   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), timestamp);
  41   offset += sizeof(int64_t);
  42   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(data));
  43   offset += sizeof(int32_t);
  44   EXPECT_TRUE(std::memcmp(data, dest.c_str() + offset, sizeof(data)) == 0);
  45
  46   // Verify the deserialization.
  47   std::string saved_dest = dest;
  48   std::shared_ptr<Column> c1 = Column::Deserialize(saved_dest.c_str(), 0);
  49   EXPECT_EQ(c1->Index(), index);
  50   EXPECT_EQ(c1->Timestamp(), timestamp);
  51   EXPECT_EQ(c1->Size(), 14 + sizeof(data));
  52
  53   c1->Serialize(&dest);
  54   EXPECT_EQ(dest.size(), 2 * c.Size());
  55   EXPECT_TRUE(
  56     std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0);
  57
  58   // Verify the ColumnBase::Deserialization.
  59   saved_dest = dest;
  60   std::shared_ptr<ColumnBase> c2 =
  61       ColumnBase::Deserialize(saved_dest.c_str(), c.Size());
  62   c2->Serialize(&dest);
  63   EXPECT_EQ(dest.size(), 3 * c.Size());
  64   EXPECT_TRUE(
  65     std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size())
  66       == 0);
  67 }
  68
  69 TEST(ExpiringColumnTest, ExpiringColumn) {
  70   char data[4] = {'d', 'a', 't', 'a'};
  71   int8_t mask = ColumnTypeMask::EXPIRATION_MASK;
  72   int8_t index = 3;
  73   int64_t timestamp = 1494022807044;
  74   int32_t ttl = 3600;
  75   ExpiringColumn c = ExpiringColumn(mask, index, timestamp,
  76                                     sizeof(data), data, ttl);
  77
  78   EXPECT_EQ(c.Index(), index);
  79   EXPECT_EQ(c.Timestamp(), timestamp);
  80   EXPECT_EQ(c.Size(), 18 + sizeof(data));
  81
  82   // Verify the serialization.
  83   std::string dest;
  84   dest.reserve(c.Size() * 2);
  85   c.Serialize(&dest);
  86
  87   EXPECT_EQ(dest.size(), c.Size());
  88   std::size_t offset = 0;
  89   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask);
  90   offset += sizeof(int8_t);
  91   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index);
  92   offset += sizeof(int8_t);
  93   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), timestamp);
  94   offset += sizeof(int64_t);
  95   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(data));
  96   offset += sizeof(int32_t);
  97   EXPECT_TRUE(std::memcmp(data, dest.c_str() + offset, sizeof(data)) == 0);
  98   offset += sizeof(data);
  99   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), ttl);
 100
 101   // Verify the deserialization.
 102   std::string saved_dest = dest;
 103   std::shared_ptr<ExpiringColumn> c1 =
 104       ExpiringColumn::Deserialize(saved_dest.c_str(), 0);
 105   EXPECT_EQ(c1->Index(), index);
 106   EXPECT_EQ(c1->Timestamp(), timestamp);
 107   EXPECT_EQ(c1->Size(), 18 + sizeof(data));
 108
 109   c1->Serialize(&dest);
 110   EXPECT_EQ(dest.size(), 2 * c.Size());
 111   EXPECT_TRUE(
 112     std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0);
 113
 114   // Verify the ColumnBase::Deserialization.
 115   saved_dest = dest;
 116   std::shared_ptr<ColumnBase> c2 =
 117       ColumnBase::Deserialize(saved_dest.c_str(), c.Size());
 118   c2->Serialize(&dest);
 119   EXPECT_EQ(dest.size(), 3 * c.Size());
 120   EXPECT_TRUE(
 121     std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size())
 122       == 0);
 123 }
 124
 125 TEST(TombstoneTest, TombstoneCollectable) {
 126   int32_t now = (int32_t)time(nullptr);
 127   int32_t gc_grace_seconds = 16440;
 128   int32_t time_delta_seconds = 10;
 129   EXPECT_TRUE(Tombstone(ColumnTypeMask::DELETION_MASK, 0,
 130                         now - gc_grace_seconds - time_delta_seconds,
 131                         ToMicroSeconds(now - gc_grace_seconds - time_delta_seconds))
 132                   .Collectable(gc_grace_seconds));
 133   EXPECT_FALSE(Tombstone(ColumnTypeMask::DELETION_MASK, 0,
 134                          now - gc_grace_seconds + time_delta_seconds,
 135                          ToMicroSeconds(now - gc_grace_seconds + time_delta_seconds))
 136                    .Collectable(gc_grace_seconds));
 137 }
 138
 139 TEST(TombstoneTest, Tombstone) {
 140   int8_t mask = ColumnTypeMask::DELETION_MASK;
 141   int8_t index = 2;
 142   int32_t local_deletion_time = 1494022807;
 143   int64_t marked_for_delete_at = 1494022807044;
 144   Tombstone c = Tombstone(mask, index, local_deletion_time,
 145                           marked_for_delete_at);
 146
 147   EXPECT_EQ(c.Index(), index);
 148   EXPECT_EQ(c.Timestamp(), marked_for_delete_at);
 149   EXPECT_EQ(c.Size(), 14);
 150
 151   // Verify the serialization.
 152   std::string dest;
 153   dest.reserve(c.Size() * 2);
 154   c.Serialize(&dest);
 155
 156   EXPECT_EQ(dest.size(), c.Size());
 157   std::size_t offset = 0;
 158   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask);
 159   offset += sizeof(int8_t);
 160   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index);
 161   offset += sizeof(int8_t);
 162   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), local_deletion_time);
 163   offset += sizeof(int32_t);
 164   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), marked_for_delete_at);
 165
 166   // Verify the deserialization.
 167   std::shared_ptr<Tombstone> c1 = Tombstone::Deserialize(dest.c_str(), 0);
 168   EXPECT_EQ(c1->Index(), index);
 169   EXPECT_EQ(c1->Timestamp(), marked_for_delete_at);
 170   EXPECT_EQ(c1->Size(), 14);
 171
 172   c1->Serialize(&dest);
 173   EXPECT_EQ(dest.size(), 2 * c.Size());
 174   EXPECT_TRUE(
 175     std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0);
 176
 177   // Verify the ColumnBase::Deserialization.
 178   std::shared_ptr<ColumnBase> c2 =
 179     ColumnBase::Deserialize(dest.c_str(), c.Size());
 180   c2->Serialize(&dest);
 181   EXPECT_EQ(dest.size(), 3 * c.Size());
 182   EXPECT_TRUE(
 183     std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size())
 184       == 0);
 185 }
 186
 187 TEST(RowValueTest, RowTombstone) {
 188   int32_t local_deletion_time = 1494022807;
 189   int64_t marked_for_delete_at = 1494022807044;
 190   RowValue r = RowValue(local_deletion_time, marked_for_delete_at);
 191
 192   EXPECT_EQ(r.Size(), 12);
 193   EXPECT_EQ(r.IsTombstone(), true);
 194   EXPECT_EQ(r.LastModifiedTime(), marked_for_delete_at);
 195
 196   // Verify the serialization.
 197   std::string dest;
 198   dest.reserve(r.Size() * 2);
 199   r.Serialize(&dest);
 200
 201   EXPECT_EQ(dest.size(), r.Size());
 202   std::size_t offset = 0;
 203   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), local_deletion_time);
 204   offset += sizeof(int32_t);
 205   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), marked_for_delete_at);
 206
 207   // Verify the deserialization.
 208   RowValue r1 = RowValue::Deserialize(dest.c_str(), r.Size());
 209   EXPECT_EQ(r1.Size(), 12);
 210   EXPECT_EQ(r1.IsTombstone(), true);
 211   EXPECT_EQ(r1.LastModifiedTime(), marked_for_delete_at);
 212
 213   r1.Serialize(&dest);
 214   EXPECT_EQ(dest.size(), 2 * r.Size());
 215   EXPECT_TRUE(
 216     std::memcmp(dest.c_str(), dest.c_str() + r.Size(), r.Size()) == 0);
 217 }
 218
 219 TEST(RowValueTest, RowWithColumns) {
 220   std::vector<std::shared_ptr<ColumnBase>> columns;
 221   int64_t last_modified_time = 1494022807048;
 222   std::size_t columns_data_size = 0;
 223
 224   char e_data[5] = {'e', 'd', 'a', 't', 'a'};
 225   int8_t e_index = 0;
 226   int64_t e_timestamp = 1494022807044;
 227   int32_t e_ttl = 3600;
 228   columns.push_back(std::shared_ptr<ExpiringColumn>(
 229     new ExpiringColumn(ColumnTypeMask::EXPIRATION_MASK, e_index,
 230       e_timestamp, sizeof(e_data), e_data, e_ttl)));
 231   columns_data_size += columns[0]->Size();
 232
 233   char c_data[4] = {'d', 'a', 't', 'a'};
 234   int8_t c_index = 1;
 235   int64_t c_timestamp = 1494022807048;
 236   columns.push_back(std::shared_ptr<Column>(
 237     new Column(0, c_index, c_timestamp, sizeof(c_data), c_data)));
 238   columns_data_size += columns[1]->Size();
 239
 240   int8_t t_index = 2;
 241   int32_t t_local_deletion_time = 1494022801;
 242   int64_t t_marked_for_delete_at = 1494022807043;
 243   columns.push_back(std::shared_ptr<Tombstone>(
 244     new Tombstone(ColumnTypeMask::DELETION_MASK,
 245       t_index, t_local_deletion_time, t_marked_for_delete_at)));
 246   columns_data_size += columns[2]->Size();
 247
 248   RowValue r = RowValue(std::move(columns), last_modified_time);
 249
 250   EXPECT_EQ(r.Size(), columns_data_size + 12);
 251   EXPECT_EQ(r.IsTombstone(), false);
 252   EXPECT_EQ(r.LastModifiedTime(), last_modified_time);
 253
 254   // Verify the serialization.
 255   std::string dest;
 256   dest.reserve(r.Size() * 2);
 257   r.Serialize(&dest);
 258
 259   EXPECT_EQ(dest.size(), r.Size());
 260   std::size_t offset = 0;
 261   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset),
 262     std::numeric_limits<int32_t>::max());
 263   offset += sizeof(int32_t);
 264   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset),
 265     std::numeric_limits<int64_t>::min());
 266   offset += sizeof(int64_t);
 267
 268   // Column0: ExpiringColumn
 269   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset),
 270     ColumnTypeMask::EXPIRATION_MASK);
 271   offset += sizeof(int8_t);
 272   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), e_index);
 273   offset += sizeof(int8_t);
 274   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), e_timestamp);
 275   offset += sizeof(int64_t);
 276   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(e_data));
 277   offset += sizeof(int32_t);
 278   EXPECT_TRUE(std::memcmp(e_data, dest.c_str() + offset, sizeof(e_data)) == 0);
 279   offset += sizeof(e_data);
 280   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), e_ttl);
 281   offset += sizeof(int32_t);
 282
 283   // Column1: Column
 284   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), 0);
 285   offset += sizeof(int8_t);
 286   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), c_index);
 287   offset += sizeof(int8_t);
 288   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), c_timestamp);
 289   offset += sizeof(int64_t);
 290   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(c_data));
 291   offset += sizeof(int32_t);
 292   EXPECT_TRUE(std::memcmp(c_data, dest.c_str() + offset, sizeof(c_data)) == 0);
 293   offset += sizeof(c_data);
 294
 295   // Column2: Tombstone
 296   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset),
 297     ColumnTypeMask::DELETION_MASK);
 298   offset += sizeof(int8_t);
 299   EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), t_index);
 300   offset += sizeof(int8_t);
 301   EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), t_local_deletion_time);
 302   offset += sizeof(int32_t);
 303   EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), t_marked_for_delete_at);
 304
 305   // Verify the deserialization.
 306   RowValue r1 = RowValue::Deserialize(dest.c_str(), r.Size());
 307   EXPECT_EQ(r1.Size(), columns_data_size + 12);
 308   EXPECT_EQ(r1.IsTombstone(), false);
 309   EXPECT_EQ(r1.LastModifiedTime(), last_modified_time);
 310
 311   r1.Serialize(&dest);
 312   EXPECT_EQ(dest.size(), 2 * r.Size());
 313   EXPECT_TRUE(
 314     std::memcmp(dest.c_str(), dest.c_str() + r.Size(), r.Size()) == 0);
 315 }
 316
 317 TEST(RowValueTest, PurgeTtlShouldRemvoeAllColumnsExpired) {
 318   int64_t now = time(nullptr);
 319
 320   auto row_value = CreateTestRowValue({
 321     CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now)),
 322     CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
 323     CreateTestColumnSpec(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
 324     CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
 325   });
 326
 327   bool changed = false;
 328   auto purged = row_value.RemoveExpiredColumns(&changed);
 329   EXPECT_TRUE(changed);
 330   EXPECT_EQ(purged.columns_.size(), 3);
 331   VerifyRowValueColumns(purged.columns_, 0, kColumn, 0, ToMicroSeconds(now));
 332   VerifyRowValueColumns(purged.columns_, 1, kExpiringColumn, 2, ToMicroSeconds(now));
 333   VerifyRowValueColumns(purged.columns_, 2, kTombstone, 3, ToMicroSeconds(now));
 334
 335   purged.RemoveExpiredColumns(&changed);
 336   EXPECT_FALSE(changed);
 337 }
 338
 339 TEST(RowValueTest, ExpireTtlShouldConvertExpiredColumnsToTombstones) {
 340   int64_t now = time(nullptr);
 341
 342   auto row_value = CreateTestRowValue({
 343     CreateTestColumnSpec(kColumn, 0, ToMicroSeconds(now)),
 344     CreateTestColumnSpec(kExpiringColumn, 1, ToMicroSeconds(now - kTtl - 10)), //expired
 345     CreateTestColumnSpec(kExpiringColumn, 2, ToMicroSeconds(now)), // not expired
 346     CreateTestColumnSpec(kTombstone, 3, ToMicroSeconds(now))
 347   });
 348
 349   bool changed = false;
 350   auto compacted = row_value.ConvertExpiredColumnsToTombstones(&changed);
 351   EXPECT_TRUE(changed);
 352   EXPECT_EQ(compacted.columns_.size(), 4);
 353   VerifyRowValueColumns(compacted.columns_, 0, kColumn, 0, ToMicroSeconds(now));
 354   VerifyRowValueColumns(compacted.columns_, 1, kTombstone, 1, ToMicroSeconds(now - 10));
 355   VerifyRowValueColumns(compacted.columns_, 2, kExpiringColumn, 2, ToMicroSeconds(now));
 356   VerifyRowValueColumns(compacted.columns_, 3, kTombstone, 3, ToMicroSeconds(now));
 357
 358   compacted.ConvertExpiredColumnsToTombstones(&changed);
 359   EXPECT_FALSE(changed);
 360 }
 361 } // namespace cassandra
 362 } // namespace rocksdb
 363
 364 int main(int argc, char** argv) {
 365   ::testing::InitGoogleTest(&argc, argv);
 366   return RUN_ALL_TESTS();
 367 }