]> git.proxmox.com Git - ceph.git/blame - ceph/src/rocksdb/tools/ldb_test.py
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update ceph source to reef 18.1.2
[ceph.git] / ceph / src / rocksdb / tools / ldb_test.py
CommitLineData
20effc67 1#!/usr/bin/env python3
f67539c2 2# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
20effc67
TL		 3from __future__ import absolute_import, division, print_function, unicode_literals
4
7c673cae 5import glob
1e59de90
TL		 6
7import os
7c673cae 8import os.path
1e59de90 9import re
7c673cae
FG		 10import shutil
11import subprocess
1e59de90 12import tempfile
7c673cae
FG		 13import time
14import unittest
1e59de90 15
7c673cae
FG		 16
17def my_check_output(*popenargs, **kwargs):
18 """
19 If we had python 2.7, we should simply use subprocess.check_output.
20 This is a stop-gap solution for python 2.6
21 """
1e59de90
TL		 22 if "stdout" in kwargs:
23 raise ValueError("stdout argument not allowed, it will be overridden.")
24 process = subprocess.Popen(
25 stderr=subprocess.PIPE, stdout=subprocess.PIPE, *popenargs, **kwargs
26 )
7c673cae
FG		 27 output, unused_err = process.communicate()
28 retcode = process.poll()
29 if retcode:
30 cmd = kwargs.get("args")
31 if cmd is None:
32 cmd = popenargs[0]
1e59de90
TL		 33 raise Exception("Exit code is not 0. It is %d. Command: %s" % (retcode, cmd))
34 return output.decode("utf-8")
35
7c673cae
FG		 36
37def run_err_null(cmd):
38 return os.system(cmd + " 2>/dev/null ")
39
1e59de90 40
7c673cae
FG		 41class LDBTestCase(unittest.TestCase):
42 def setUp(self):
1e59de90 43 self.TMP_DIR = tempfile.mkdtemp(prefix="ldb_test_")
7c673cae
FG		 44 self.DB_NAME = "testdb"
45
46 def tearDown(self):
1e59de90
TL		 47 assert (
48 self.TMP_DIR.strip() != "/"
49 and self.TMP_DIR.strip() != "/tmp"
50 and self.TMP_DIR.strip() != "/tmp/"
51 ) # Just some paranoia
7c673cae
FG		 52
53 shutil.rmtree(self.TMP_DIR)
54
55 def dbParam(self, dbName):
56 return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
57
1e59de90
TL		 58 def assertRunOKFull(
59 self, params, expectedOutput, unexpected=False, isPattern=False
60 ):
7c673cae
FG		 61 """
62 All command-line params must be specified.
63 Allows full flexibility in testing; for example: missing db param.
7c673cae 64 """
1e59de90
TL		 65 output = my_check_output(
66 './ldb %s |grep -v "Created bg thread"' % params, shell=True
67 )
7c673cae
FG		 68 if not unexpected:
69 if isPattern:
1e59de90 70 self.assertNotEqual(expectedOutput.search(output.strip()), None)
7c673cae
FG		 71 else:
72 self.assertEqual(output.strip(), expectedOutput.strip())
73 else:
74 if isPattern:
75 self.assertEqual(expectedOutput.search(output.strip()), None)
76 else:
77 self.assertNotEqual(output.strip(), expectedOutput.strip())
78
79 def assertRunFAILFull(self, params):
80 """
81 All command-line params must be specified.
82 Allows full flexibility in testing; for example: missing db param.
7c673cae
FG		 83 """
84 try:
85
1e59de90
TL		 86 my_check_output(
87 './ldb %s >/dev/null 2>&1 |grep -v "Created bg \
88 thread"'
89 % params,
90 shell=True,
91 )
11fdf7f2 92 except Exception:
7c673cae
FG		 93 return
94 self.fail(
1e59de90
TL		 95 "Exception should have been raised for command with params: %s" % params
96 )
7c673cae
FG		 97
98 def assertRunOK(self, params, expectedOutput, unexpected=False):
99 """
100 Uses the default test db.
7c673cae 101 """
1e59de90
TL		 102 self.assertRunOKFull(
103 "%s %s" % (self.dbParam(self.DB_NAME), params), expectedOutput, unexpected
104 )
7c673cae
FG		 105
106 def assertRunFAIL(self, params):
107 """
108 Uses the default test db.
109 """
110 self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
111
112 def testSimpleStringPutGet(self):
20effc67 113 print("Running testSimpleStringPutGet...")
7c673cae
FG		 114 self.assertRunFAIL("put x1 y1")
115 self.assertRunOK("put --create_if_missing x1 y1", "OK")
116 self.assertRunOK("get x1", "y1")
117 self.assertRunFAIL("get x2")
118
119 self.assertRunOK("put x2 y2", "OK")
120 self.assertRunOK("get x1", "y1")
121 self.assertRunOK("get x2", "y2")
122 self.assertRunFAIL("get x3")
123
124 self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2")
125 self.assertRunOK("put x3 y3", "OK")
126
127 self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3")
128 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
129 self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3")
130
131 self.assertRunOK("scan --to=x2", "x1 : y1")
132 self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1")
1e59de90 133 self.assertRunOK("scan --from=x1 --to=z --max_keys=2", "x1 : y1\nx2 : y2")
7c673cae 134
1e59de90
TL		 135 self.assertRunOK(
136 "scan --from=x1 --to=z --max_keys=3", "x1 : y1\nx2 : y2\nx3 : y3"
137 )
138 self.assertRunOK(
139 "scan --from=x1 --to=z --max_keys=4", "x1 : y1\nx2 : y2\nx3 : y3"
140 )
7c673cae
FG		 141 self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1")
142 self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3")
1e59de90 143 self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL
7c673cae
FG		 144 self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo")
145
146 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
147
148 self.assertRunOK("delete x1", "OK")
149 self.assertRunOK("scan", "x2 : y2\nx3 : y3")
150
151 self.assertRunOK("delete NonExistentKey", "OK")
152 # It is weird that GET and SCAN raise exception for
153 # non-existent key, while delete does not
154
155 self.assertRunOK("checkconsistency", "OK")
156
157 def dumpDb(self, params, dumpFile):
158 return 0 == run_err_null("./ldb dump %s > %s" % (params, dumpFile))
159
160 def loadDb(self, params, dumpFile):
161 return 0 == run_err_null("cat %s | ./ldb load %s" % (dumpFile, params))
162
11fdf7f2 163 def writeExternSst(self, params, inputDumpFile, outputSst):
1e59de90
TL		 164 return 0 == run_err_null(
165 "cat %s | ./ldb write_extern_sst %s %s" % (inputDumpFile, outputSst, params)
166 )
11fdf7f2
TL		 167
168 def ingestExternSst(self, params, inputSst):
1e59de90 169 return 0 == run_err_null("./ldb ingest_extern_sst %s %s" % (inputSst, params))
11fdf7f2 170
7c673cae 171 def testStringBatchPut(self):
20effc67 172 print("Running testStringBatchPut...")
7c673cae
FG		 173 self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
174 self.assertRunOK("scan", "x1 : y1")
1e59de90 175 self.assertRunOK('batchput x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK")
7c673cae
FG		 176 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
177 self.assertRunFAIL("batchput")
178 self.assertRunFAIL("batchput k1")
179 self.assertRunFAIL("batchput k1 v1 k2")
180
1e59de90
TL		 181 def testBlobBatchPut(self):
182 print("Running testBlobBatchPut...")
183
184 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
185 self.assertRunOK("batchput x1 y1 --create_if_missing --enable_blob_files", "OK")
186 self.assertRunOK("scan", "x1 : y1")
187 self.assertRunOK(
188 'batchput --enable_blob_files x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK"
189 )
190 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
191
192 blob_files = self.getBlobFiles(dbPath)
193 self.assertTrue(len(blob_files) >= 1)
194
195 def testBlobPut(self):
196 print("Running testBlobPut...")
197
198 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
199 self.assertRunOK("put --create_if_missing --enable_blob_files x1 y1", "OK")
200 self.assertRunOK("get x1", "y1")
201 self.assertRunOK("put --enable_blob_files x2 y2", "OK")
202 self.assertRunOK("get x1", "y1")
203 self.assertRunOK("get x2", "y2")
204 self.assertRunFAIL("get x3")
205
206 blob_files = self.getBlobFiles(dbPath)
207 self.assertTrue(len(blob_files) >= 1)
208
209 def testBlobStartingLevel(self):
210 print("Running testBlobStartingLevel...")
211
212 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
213 self.assertRunOK(
214 "put --create_if_missing --enable_blob_files --blob_file_starting_level=10 x1 y1",
215 "OK",
216 )
217 self.assertRunOK("get x1", "y1")
218
219 blob_files = self.getBlobFiles(dbPath)
220 self.assertTrue(len(blob_files) == 0)
221
222 self.assertRunOK(
223 "put --enable_blob_files --blob_file_starting_level=0 x2 y2", "OK"
224 )
225 self.assertRunOK("get x1", "y1")
226 self.assertRunOK("get x2", "y2")
227 self.assertRunFAIL("get x3")
228
229 blob_files = self.getBlobFiles(dbPath)
230 self.assertTrue(len(blob_files) >= 1)
231
7c673cae 232 def testCountDelimDump(self):
20effc67 233 print("Running testCountDelimDump...")
7c673cae
FG		 234 self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
235 self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
1e59de90
TL		 236 self.assertRunOK(
237 "dump --count_delim",
238 "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
239 )
240 self.assertRunOK(
241 'dump --count_delim="."',
242 "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
243 )
7c673cae 244 self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
1e59de90
TL		 245 self.assertRunOK(
246 'dump --count_delim=","',
247 "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8",
248 )
7c673cae
FG		 249
250 def testCountDelimIDump(self):
20effc67 251 print("Running testCountDelimIDump...")
7c673cae
FG		 252 self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
253 self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
1e59de90
TL		 254 self.assertRunOK(
255 "idump --count_delim",
256 "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
257 )
258 self.assertRunOK(
259 'idump --count_delim="."',
260 "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8",
261 )
7c673cae 262 self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
1e59de90
TL		 263 self.assertRunOK(
264 'idump --count_delim=","',
265 "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8",
266 )
7c673cae
FG		 267
268 def testInvalidCmdLines(self):
20effc67 269 print("Running testInvalidCmdLines...")
7c673cae
FG		 270 # db not specified
271 self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
272 # No param called he
273 self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
274 # max_keys is not applicable for put
275 self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
276 # hex has invalid boolean value
277
278 def testHexPutGet(self):
20effc67 279 print("Running testHexPutGet...")
7c673cae
FG		 280 self.assertRunOK("put a1 b1 --create_if_missing", "OK")
281 self.assertRunOK("scan", "a1 : b1")
282 self.assertRunOK("scan --hex", "0x6131 : 0x6231")
283 self.assertRunFAIL("put --hex 6132 6232")
284 self.assertRunOK("put --hex 0x6132 0x6232", "OK")
285 self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232")
286 self.assertRunOK("scan", "a1 : b1\na2 : b2")
287 self.assertRunOK("get a1", "b1")
288 self.assertRunOK("get --hex 0x6131", "0x6231")
289 self.assertRunOK("get a2", "b2")
290 self.assertRunOK("get --hex 0x6132", "0x6232")
291 self.assertRunOK("get --key_hex 0x6132", "b2")
292 self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232")
293 self.assertRunOK("get --value_hex a2", "0x6232")
1e59de90
TL		 294 self.assertRunOK(
295 "scan --key_hex --value_hex", "0x6131 : 0x6231\n0x6132 : 0x6232"
296 )
297 self.assertRunOK(
298 "scan --hex --from=0x6131 --to=0x6133", "0x6131 : 0x6231\n0x6132 : 0x6232"
299 )
300 self.assertRunOK("scan --hex --from=0x6131 --to=0x6132", "0x6131 : 0x6231")
7c673cae
FG		 301 self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2")
302 self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232")
303 self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK")
304 self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4")
305 self.assertRunOK("delete --hex 0x6133", "OK")
306 self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
307 self.assertRunOK("checkconsistency", "OK")
308
309 def testTtlPutGet(self):
20effc67 310 print("Running testTtlPutGet...")
7c673cae
FG		 311 self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
312 self.assertRunOK("scan --hex", "0x6131 : 0x6231", True)
313 self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
1e59de90 314 self.assertRunOK("dump --hex --ttl ", "0x6131 ==> 0x6231\nKeys in range: 1")
7c673cae
FG		 315 self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231")
316 self.assertRunOK("get --value_hex a1", "0x6231", True)
317 self.assertRunOK("get --ttl a1", "b1")
318 self.assertRunOK("put a3 b3 --create_if_missing", "OK")
319 # fails because timstamp's length is greater than value's
320 self.assertRunFAIL("get --ttl a3")
321 self.assertRunOK("checkconsistency", "OK")
322
11fdf7f2 323 def testInvalidCmdLines(self): # noqa: F811 T25377293 Grandfathered in
20effc67 324 print("Running testInvalidCmdLines...")
7c673cae
FG		 325 # db not specified
326 self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
327 # No param called he
328 self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
329 # max_keys is not applicable for put
330 self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
331 # hex has invalid boolean value
332 self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
333
334 def testDumpLoad(self):
20effc67 335 print("Running testDumpLoad...")
1e59de90 336 self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK")
7c673cae
FG		 337 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
338 origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
339
340 # Dump and load without any additional params specified
341 dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
342 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump1")
343 self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
1e59de90
TL		 344 self.assertTrue(
345 self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
346 )
347 self.assertRunOKFull(
348 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
349 )
7c673cae
FG		 350
351 # Dump and load in hex
352 dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
353 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump2")
354 self.assertTrue(self.dumpDb("--db=%s --hex" % origDbPath, dumpFilePath))
1e59de90
TL		 355 self.assertTrue(
356 self.loadDb(
357 "--db=%s --hex --create_if_missing" % loadedDbPath, dumpFilePath
358 )
359 )
360 self.assertRunOKFull(
361 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
362 )
7c673cae
FG		 363
364 # Dump only a portion of the key range
365 dumpFilePath = os.path.join(self.TMP_DIR, "dump3")
366 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump3")
1e59de90
TL		 367 self.assertTrue(
368 self.dumpDb("--db=%s --from=x1 --to=x3" % origDbPath, dumpFilePath)
369 )
370 self.assertTrue(
371 self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
372 )
7c673cae
FG		 373 self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2")
374
375 # Dump upto max_keys rows
376 dumpFilePath = os.path.join(self.TMP_DIR, "dump4")
377 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump4")
1e59de90
TL		 378 self.assertTrue(self.dumpDb("--db=%s --max_keys=3" % origDbPath, dumpFilePath))
379 self.assertTrue(
380 self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
381 )
382 self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3")
7c673cae
FG		 383
384 # Load into an existing db, create_if_missing is not specified
385 self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
386 self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath))
1e59de90
TL		 387 self.assertRunOKFull(
388 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
389 )
7c673cae
FG		 390
391 # Dump and load with WAL disabled
392 dumpFilePath = os.path.join(self.TMP_DIR, "dump5")
393 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump5")
394 self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
1e59de90
TL		 395 self.assertTrue(
396 self.loadDb(
397 "--db=%s --disable_wal --create_if_missing" % loadedDbPath, dumpFilePath
398 )
399 )
400 self.assertRunOKFull(
401 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
402 )
7c673cae
FG		 403
404 # Dump and load with lots of extra params specified
1e59de90
TL		 405 extraParams = " ".join(
406 [
407 "--bloom_bits=14",
408 "--block_size=1024",
409 "--auto_compaction=true",
410 "--write_buffer_size=4194304",
411 "--file_size=2097152",
412 ]
413 )
7c673cae
FG		 414 dumpFilePath = os.path.join(self.TMP_DIR, "dump6")
415 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump6")
1e59de90
TL		 416 self.assertTrue(
417 self.dumpDb("--db=%s %s" % (origDbPath, extraParams), dumpFilePath)
418 )
419 self.assertTrue(
420 self.loadDb(
421 "--db=%s %s --create_if_missing" % (loadedDbPath, extraParams),
422 dumpFilePath,
423 )
424 )
425 self.assertRunOKFull(
426 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
427 )
7c673cae
FG		 428
429 # Dump with count_only
430 dumpFilePath = os.path.join(self.TMP_DIR, "dump7")
431 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump7")
1e59de90
TL		 432 self.assertTrue(self.dumpDb("--db=%s --count_only" % origDbPath, dumpFilePath))
433 self.assertTrue(
434 self.loadDb("--db=%s --create_if_missing" % loadedDbPath, dumpFilePath)
435 )
7c673cae
FG		 436 # DB should have atleast one value for scan to work
437 self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK")
438 self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1")
439
440 # Dump command fails because of typo in params
441 dumpFilePath = os.path.join(self.TMP_DIR, "dump8")
1e59de90
TL		 442 self.assertFalse(
443 self.dumpDb("--db=%s --create_if_missing" % origDbPath, dumpFilePath)
444 )
445
446 # Dump and load with BlobDB enabled
447 blobParams = " ".join(
448 ["--enable_blob_files", "--min_blob_size=1", "--blob_file_size=2097152"]
449 )
450 dumpFilePath = os.path.join(self.TMP_DIR, "dump9")
451 loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump9")
452 self.assertTrue(self.dumpDb("--db=%s" % (origDbPath), dumpFilePath))
453 self.assertTrue(
454 self.loadDb(
455 "--db=%s %s --create_if_missing --disable_wal"
456 % (loadedDbPath, blobParams),
457 dumpFilePath,
458 )
459 )
460 self.assertRunOKFull(
461 "scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4"
462 )
463 blob_files = self.getBlobFiles(loadedDbPath)
464 self.assertTrue(len(blob_files) >= 1)
7c673cae 465
11fdf7f2 466 def testIDumpBasics(self):
20effc67 467 print("Running testIDumpBasics...")
11fdf7f2
TL		 468 self.assertRunOK("put a val --create_if_missing", "OK")
469 self.assertRunOK("put b val", "OK")
470 self.assertRunOK(
1e59de90
TL		 471 "idump",
472 "'a' seq:1, type:1 => val\n"
473 "'b' seq:2, type:1 => val\nInternal keys in range: 2",
474 )
11fdf7f2 475 self.assertRunOK(
1e59de90
TL		 476 "idump --input_key_hex --from=%s --to=%s" % (hex(ord("a")), hex(ord("b"))),
477 "'a' seq:1, type:1 => val\nInternal keys in range: 1",
478 )
479
480 def testIDumpDecodeBlobIndex(self):
481 print("Running testIDumpDecodeBlobIndex...")
482 self.assertRunOK("put a val --create_if_missing", "OK")
483 self.assertRunOK("put b val --enable_blob_files", "OK")
484
485 # Pattern to expect from dump with decode_blob_index flag enabled.
486 regex = ".*\[blob ref\].*"
487 expected_pattern = re.compile(regex)
488 cmd = "idump %s --decode_blob_index"
489 self.assertRunOKFull(
490 (cmd) % (self.dbParam(self.DB_NAME)),
491 expected_pattern,
492 unexpected=False,
493 isPattern=True,
494 )
11fdf7f2 495
7c673cae 496 def testMiscAdminTask(self):
20effc67 497 print("Running testMiscAdminTask...")
7c673cae
FG		 498 # These tests need to be improved; for example with asserts about
499 # whether compaction or level reduction actually took place.
1e59de90 500 self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4", "OK")
7c673cae
FG		 501 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
502 origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
503
1e59de90 504 self.assertTrue(0 == run_err_null("./ldb compact --db=%s" % origDbPath))
7c673cae
FG		 505 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
506
1e59de90
TL		 507 self.assertTrue(
508 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=2" % origDbPath)
509 )
7c673cae
FG		 510 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
511
1e59de90
TL		 512 self.assertTrue(
513 0 == run_err_null("./ldb reduce_levels --db=%s --new_levels=3" % origDbPath)
514 )
7c673cae
FG		 515 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
516
1e59de90
TL		 517 self.assertTrue(
518 0 == run_err_null("./ldb compact --db=%s --from=x1 --to=x3" % origDbPath)
519 )
7c673cae
FG		 520 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
521
1e59de90
TL		 522 self.assertTrue(
523 0
524 == run_err_null(
525 "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134" % origDbPath
526 )
527 )
7c673cae
FG		 528 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
529
1e59de90
TL		 530 # TODO(dilip): Not sure what should be passed to WAL.Currently corrupted.
531 self.assertTrue(
532 0
533 == run_err_null(
534 "./ldb dump_wal --db=%s --walfile=%s --header"
535 % (origDbPath, os.path.join(origDbPath, "LOG"))
536 )
537 )
7c673cae
FG		 538 self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
539
540 def testCheckConsistency(self):
20effc67 541 print("Running testCheckConsistency...")
7c673cae
FG		 542
543 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
544 self.assertRunOK("put x1 y1 --create_if_missing", "OK")
545 self.assertRunOK("put x2 y2", "OK")
546 self.assertRunOK("get x1", "y1")
547 self.assertRunOK("checkconsistency", "OK")
548
1e59de90
TL		 549 sstFilePath = my_check_output(
550 "ls %s" % os.path.join(dbPath, "*.sst"), shell=True
551 )
7c673cae
FG		 552
553 # Modify the file
554 my_check_output("echo 'evil' > %s" % sstFilePath, shell=True)
555 self.assertRunFAIL("checkconsistency")
556
557 # Delete the file
558 my_check_output("rm -f %s" % sstFilePath, shell=True)
559 self.assertRunFAIL("checkconsistency")
560
561 def dumpLiveFiles(self, params, dumpFile):
1e59de90 562 return 0 == run_err_null("./ldb dump_live_files %s > %s" % (params, dumpFile))
7c673cae
FG		 563
564 def testDumpLiveFiles(self):
20effc67 565 print("Running testDumpLiveFiles...")
7c673cae
FG		 566
567 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
568 self.assertRunOK("put x1 y1 --create_if_missing", "OK")
1e59de90 569 self.assertRunOK("put x2 y2 --enable_blob_files", "OK")
7c673cae
FG		 570 dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
571 self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
572 self.assertRunOK("delete x1", "OK")
573 self.assertRunOK("put x3 y3", "OK")
574 dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
1e59de90
TL		 575
576 # Test that if the user provides a db path that ends with
577 # a slash '/', there is no double (or more!) slashes in the
578 # SST and manifest file names.
579
580 # Add a '/' at the end of dbPath (which normally shouldnt contain any)
581 if dbPath[-1] != "/":
582 dbPath += "/"
583
584 # Call the dump_live_files function with the edited dbPath name.
585 self.assertTrue(
586 self.dumpLiveFiles(
587 "--db=%s --decode_blob_index --dump_uncompressed_blobs" % dbPath,
588 dumpFilePath,
589 )
590 )
591
592 # Investigate the output
593 with open(dumpFilePath, "r") as tmp:
594 data = tmp.read()
595
596 # Check that all the SST filenames have a correct full path (no multiple '/').
597 sstFileList = re.findall(r"%s.*\d+.sst" % dbPath, data)
598 self.assertTrue(len(sstFileList) >= 1)
599 for sstFilename in sstFileList:
600 filenumber = re.findall(r"\d+.sst", sstFilename)[0]
601 self.assertEqual(sstFilename, dbPath + filenumber)
602
603 # Check that all the Blob filenames have a correct full path (no multiple '/').
604 blobFileList = re.findall(r"%s.*\d+.blob" % dbPath, data)
605 self.assertTrue(len(blobFileList) >= 1)
606 for blobFilename in blobFileList:
607 filenumber = re.findall(r"\d+.blob", blobFilename)[0]
608 self.assertEqual(blobFilename, dbPath + filenumber)
609
610 # Check that all the manifest filenames
611 # have a correct full path (no multiple '/').
612 manifestFileList = re.findall(r"%s.*MANIFEST-\d+" % dbPath, data)
613 self.assertTrue(len(manifestFileList) >= 1)
614 for manifestFilename in manifestFileList:
615 filenumber = re.findall(r"(?<=MANIFEST-)\d+", manifestFilename)[0]
616 self.assertEqual(manifestFilename, dbPath + "MANIFEST-" + filenumber)
617
618 # Check that the blob file index is decoded.
619 decodedBlobIndex = re.findall(r"\[blob ref\]", data)
620 self.assertTrue(len(decodedBlobIndex) >= 1)
621
622 def listLiveFilesMetadata(self, params, dumpFile):
623 return 0 == run_err_null(
624 "./ldb list_live_files_metadata %s > %s" % (params, dumpFile)
625 )
626
627 def testListLiveFilesMetadata(self):
628 print("Running testListLiveFilesMetadata...")
629
630 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
631 self.assertRunOK("put x1 y1 --create_if_missing", "OK")
632 self.assertRunOK("put x2 y2", "OK")
633
634 # Compare the SST filename and the level of list_live_files_metadata
635 # with the data collected from dump_live_files.
636 dumpFilePath1 = os.path.join(self.TMP_DIR, "dump1")
637 self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath1))
638 dumpFilePath2 = os.path.join(self.TMP_DIR, "dump2")
639 self.assertTrue(
640 self.listLiveFilesMetadata(
641 "--sort_by_filename --db=%s" % dbPath, dumpFilePath2
642 )
643 )
644
645 # Collect SST filename and level from dump_live_files
646 with open(dumpFilePath1, "r") as tmp:
647 data = tmp.read()
648 filename1 = re.findall(r".*\d+\.sst", data)[0]
649 level1 = re.findall(r"level:\d+", data)[0].split(":")[1]
650
651 # Collect SST filename and level from list_live_files_metadata
652 with open(dumpFilePath2, "r") as tmp:
653 data = tmp.read()
654 filename2 = re.findall(r".*\d+\.sst", data)[0]
655 level2 = re.findall(r"level \d+", data)[0].split(" ")[1]
656
657 # Assert equality between filenames and levels.
658 self.assertEqual(filename1, filename2)
659 self.assertEqual(level1, level2)
660
661 # Create multiple column families and compare the output
662 # of list_live_files_metadata with dump_live_files once again.
663 # Create new CF, and insert data:
664 self.assertRunOK("create_column_family mycol1", "OK")
665 self.assertRunOK("put --column_family=mycol1 v1 v2", "OK")
666 self.assertRunOK("create_column_family mycol2", "OK")
667 self.assertRunOK("put --column_family=mycol2 h1 h2", "OK")
668 self.assertRunOK("put --column_family=mycol2 h3 h4", "OK")
669
670 # Call dump_live_files and list_live_files_metadata
671 # and pipe the output to compare them later.
672 dumpFilePath3 = os.path.join(self.TMP_DIR, "dump3")
673 self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath3))
674 dumpFilePath4 = os.path.join(self.TMP_DIR, "dump4")
675 self.assertTrue(
676 self.listLiveFilesMetadata(
677 "--sort_by_filename --db=%s" % dbPath, dumpFilePath4
678 )
679 )
680
681 # dump_live_files:
682 # parse the output and create a map:
683 # [key: sstFilename]->[value:[LSM level, Column Family Name]]
684 referenceMap = {}
685 with open(dumpFilePath3, "r") as tmp:
686 data = tmp.read()
687 # Note: the following regex are contingent on what the
688 # dump_live_files outputs.
689 namesAndLevels = re.findall(r"\d+.sst level:\d+", data)
690 cfs = re.findall(r"(?<=column family name=)\w+", data)
691 # re.findall should not reorder the data.
692 # Therefore namesAndLevels[i] matches the data from cfs[i].
693 for count, nameAndLevel in enumerate(namesAndLevels):
694 sstFilename = re.findall(r"\d+.sst", nameAndLevel)[0]
695 sstLevel = re.findall(r"(?<=level:)\d+", nameAndLevel)[0]
696 cf = cfs[count]
697 referenceMap[sstFilename] = [sstLevel, cf]
698
699 # list_live_files_metadata:
700 # parse the output and create a map:
701 # [key: sstFilename]->[value:[LSM level, Column Family Name]]
702 testMap = {}
703 with open(dumpFilePath4, "r") as tmp:
704 data = tmp.read()
705 # Since for each SST file, all the information is contained
706 # on one line, the parsing is easy to perform and relies on
707 # the appearance of an "00xxx.sst" pattern.
708 sstLines = re.findall(r".*\d+.sst.*", data)
709 for line in sstLines:
710 sstFilename = re.findall(r"\d+.sst", line)[0]
711 sstLevel = re.findall(r"(?<=level )\d+", line)[0]
712 cf = re.findall(r"(?<=column family \')\w+(?=\')", line)[0]
713 testMap[sstFilename] = [sstLevel, cf]
714
715 # Compare the map obtained from dump_live_files and the map
716 # obtained from list_live_files_metadata. Everything should match.
717 self.assertEqual(referenceMap, testMap)
7c673cae
FG		 718
719 def getManifests(self, directory):
720 return glob.glob(directory + "/MANIFEST-*")
721
722 def getSSTFiles(self, directory):
723 return glob.glob(directory + "/*.sst")
724
725 def getWALFiles(self, directory):
726 return glob.glob(directory + "/*.log")
727
1e59de90
TL		 728 def getBlobFiles(self, directory):
729 return glob.glob(directory + "/*.blob")
730
7c673cae
FG		 731 def copyManifests(self, src, dest):
732 return 0 == run_err_null("cp " + src + " " + dest)
733
734 def testManifestDump(self):
20effc67 735 print("Running testManifestDump...")
7c673cae
FG		 736 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
737 self.assertRunOK("put 1 1 --create_if_missing", "OK")
738 self.assertRunOK("put 2 2", "OK")
739 self.assertRunOK("put 3 3", "OK")
740 # Pattern to expect from manifest_dump.
741 num = "[0-9]+"
742 st = ".*"
743 subpat = st + " seq:" + num + ", type:" + num
744 regex = num + ":" + num + "\[" + subpat + ".." + subpat + "\]"
745 expected_pattern = re.compile(regex)
746 cmd = "manifest_dump --db=%s"
747 manifest_files = self.getManifests(dbPath)
748 self.assertTrue(len(manifest_files) == 1)
749 # Test with the default manifest file in dbPath.
1e59de90
TL		 750 self.assertRunOKFull(
751 cmd % dbPath, expected_pattern, unexpected=False, isPattern=True
752 )
7c673cae
FG		 753 self.copyManifests(manifest_files[0], manifest_files[0] + "1")
754 manifest_files = self.getManifests(dbPath)
755 self.assertTrue(len(manifest_files) == 2)
756 # Test with multiple manifest files in dbPath.
757 self.assertRunFAILFull(cmd % dbPath)
758 # Running it with the copy we just created should pass.
1e59de90
TL		 759 self.assertRunOKFull(
760 (cmd + " --path=%s") % (dbPath, manifest_files[1]),
761 expected_pattern,
762 unexpected=False,
763 isPattern=True,
764 )
7c673cae
FG		 765 # Make sure that using the dump with --path will result in identical
766 # output as just using manifest_dump.
767 cmd = "dump --path=%s"
1e59de90
TL		 768 self.assertRunOKFull(
769 (cmd) % (manifest_files[1]),
770 expected_pattern,
771 unexpected=False,
772 isPattern=True,
773 )
774
775 # Check if null characters doesn't infer with output format.
776 self.assertRunOK("put a1 b1", "OK")
777 self.assertRunOK("put a2 b2", "OK")
778 self.assertRunOK("put --hex 0x12000DA0 0x80C0000B", "OK")
779 self.assertRunOK("put --hex 0x7200004f 0x80000004", "OK")
780 self.assertRunOK("put --hex 0xa000000a 0xf000000f", "OK")
781 self.assertRunOK("put a3 b3", "OK")
782 self.assertRunOK("put a4 b4", "OK")
783
784 # Verifies that all "levels" are printed out.
785 # There should be 66 mentions of levels.
786 expected_verbose_output = re.compile("matched")
787 # Test manifest_dump verbose and verify that key 0x7200004f
788 # is present. Note that we are forced to use grep here because
789 # an output with a non-terminating null character in it isn't piped
790 # correctly through the Python subprocess object.
791 # Also note that 0x72=r and 0x4f=O, hence the regex \'r.{2}O\'
792 # (we cannot use null character in the subprocess input either,
793 # so we have to use '.{2}')
794 cmd_verbose = (
795 "manifest_dump --verbose --db=%s | grep -aq $''r.{2}O'' && echo 'matched' || echo 'not matched'"
796 % dbPath
797 )
798
799 self.assertRunOKFull(
800 cmd_verbose, expected_verbose_output, unexpected=False, isPattern=True
801 )
802
803 def testGetProperty(self):
804 print("Running testGetProperty...")
805 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
806 self.assertRunOK("put 1 1 --create_if_missing", "OK")
807 self.assertRunOK("put 2 2", "OK")
808 # A "string" property
809 cmd = "--db=%s get_property rocksdb.estimate-num-keys"
810 self.assertRunOKFull(cmd % dbPath, "rocksdb.estimate-num-keys: 2")
811 # A "map" property
812 # FIXME: why doesn't this pick up two entries?
813 cmd = "--db=%s get_property rocksdb.aggregated-table-properties"
814 part = "rocksdb.aggregated-table-properties.num_entries: "
815 expected_pattern = re.compile(part)
816 self.assertRunOKFull(
817 cmd % dbPath, expected_pattern, unexpected=False, isPattern=True
818 )
819 # An invalid property
820 cmd = "--db=%s get_property rocksdb.this-property-does-not-exist"
821 self.assertRunFAILFull(cmd % dbPath)
7c673cae
FG		 822
823 def testSSTDump(self):
20effc67 824 print("Running testSSTDump...")
7c673cae
FG		 825
826 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
827 self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK")
1e59de90 828 self.assertRunOK("put sst2 sst2_val --enable_blob_files", "OK")
7c673cae
FG		 829 self.assertRunOK("get sst1", "sst1_val")
830
831 # Pattern to expect from SST dump.
1e59de90 832 regex = ".*Sst file format:.*\n.*\[blob ref\].*"
7c673cae
FG		 833 expected_pattern = re.compile(regex)
834
835 sst_files = self.getSSTFiles(dbPath)
836 self.assertTrue(len(sst_files) >= 1)
1e59de90
TL		 837 cmd = "dump --path=%s --decode_blob_index"
838 self.assertRunOKFull(
839 (cmd) % (sst_files[0]), expected_pattern, unexpected=False, isPattern=True
840 )
841
842 def testBlobDump(self):
843 print("Running testBlobDump")
844 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
845 self.assertRunOK("batchput x1 y1 --create_if_missing --enable_blob_files", "OK")
846 self.assertRunOK(
847 'batchput --enable_blob_files x2 y2 x3 y3 "x4 abc" "y4 xyz"', "OK"
848 )
849
850 # Pattern to expect from blob file dump.
851 regex = ".*Blob log header[\s\S]*Blob log footer[\s\S]*Read record[\s\S]*Summary" # noqa
852 expected_pattern = re.compile(regex)
853 blob_files = self.getBlobFiles(dbPath)
854 self.assertTrue(len(blob_files) >= 1)
855 cmd = "dump --path=%s --dump_uncompressed_blobs"
856 self.assertRunOKFull(
857 (cmd) % (blob_files[0]), expected_pattern, unexpected=False, isPattern=True
858 )
7c673cae
FG		 859
860 def testWALDump(self):
20effc67 861 print("Running testWALDump...")
7c673cae
FG		 862
863 dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
864 self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK")
865 self.assertRunOK("put wal2 wal2_val", "OK")
866 self.assertRunOK("get wal1", "wal1_val")
867
868 # Pattern to expect from WAL dump.
869 regex = "^Sequence,Count,ByteSize,Physical Offset,Key\(s\).*"
870 expected_pattern = re.compile(regex)
871
872 wal_files = self.getWALFiles(dbPath)
873 self.assertTrue(len(wal_files) >= 1)
874 cmd = "dump --path=%s"
1e59de90
TL		 875 self.assertRunOKFull(
876 (cmd) % (wal_files[0]), expected_pattern, unexpected=False, isPattern=True
877 )
7c673cae
FG		 878
879 def testListColumnFamilies(self):
20effc67 880 print("Running testListColumnFamilies...")
7c673cae 881 self.assertRunOK("put x1 y1 --create_if_missing", "OK")
1e59de90 882 cmd = 'list_column_families | grep -v "Column families"'
7c673cae 883 # Test on valid dbPath.
f67539c2 884 self.assertRunOK(cmd, "{default}")
7c673cae 885 # Test on empty path.
f67539c2 886 self.assertRunFAIL(cmd)
7c673cae
FG		 887
888 def testColumnFamilies(self):
20effc67 889 print("Running testColumnFamilies...")
1e59de90 890 _ = os.path.join(self.TMP_DIR, self.DB_NAME)
7c673cae
FG		 891 self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
892 self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
893 self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
894 # Given non-default column family to single CF DB.
895 self.assertRunFAIL("get cf1_1 --column_family=two")
896 self.assertRunOK("create_column_family two", "OK")
1e59de90
TL		 897 self.assertRunOK("put cf2_1 1 --create_if_missing --column_family=two", "OK")
898 self.assertRunOK("put cf2_2 2 --create_if_missing --column_family=two", "OK")
7c673cae
FG		 899 self.assertRunOK("delete cf1_2", "OK")
900 self.assertRunOK("create_column_family three", "OK")
901 self.assertRunOK("delete cf2_2 --column_family=two", "OK")
1e59de90 902 self.assertRunOK("put cf3_1 3 --create_if_missing --column_family=three", "OK")
7c673cae 903 self.assertRunOK("get cf1_1 --column_family=default", "1")
1e59de90
TL		 904 self.assertRunOK("dump --column_family=two", "cf2_1 ==> 1\nKeys in range: 1")
905 self.assertRunOK(
906 "dump --column_family=two --try_load_options",
907 "cf2_1 ==> 1\nKeys in range: 1",
908 )
909 self.assertRunOK("dump", "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2")
910 self.assertRunOK("get cf2_1 --column_family=two", "1")
911 self.assertRunOK("get cf3_1 --column_family=three", "3")
f67539c2 912 self.assertRunOK("drop_column_family three", "OK")
7c673cae
FG		 913 # non-existing column family.
914 self.assertRunFAIL("get cf3_1 --column_family=four")
f67539c2 915 self.assertRunFAIL("drop_column_family four")
7c673cae 916
11fdf7f2 917 def testIngestExternalSst(self):
20effc67 918 print("Running testIngestExternalSst...")
11fdf7f2
TL		 919
920 # Dump, load, write external sst and ingest it in another db
921 dbPath = os.path.join(self.TMP_DIR, "db1")
922 self.assertRunOK(
1e59de90
TL		 923 "batchput --db=%s --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4" % dbPath,
924 "OK",
925 )
926 self.assertRunOK("scan --db=%s" % dbPath, "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
11fdf7f2 927 dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
1e59de90 928 with open(dumpFilePath, "w") as f:
11fdf7f2
TL		 929 f.write("x1 ==> y10\nx2 ==> y20\nx3 ==> y30\nx4 ==> y40")
930 externSstPath = os.path.join(self.TMP_DIR, "extern_data1.sst")
1e59de90
TL		 931 self.assertTrue(
932 self.writeExternSst(
933 "--create_if_missing --db=%s" % dbPath, dumpFilePath, externSstPath
934 )
935 )
11fdf7f2
TL		 936 # cannot ingest if allow_global_seqno is false
937 self.assertFalse(
938 self.ingestExternSst(
1e59de90
TL		 939 "--create_if_missing --allow_global_seqno=false --db=%s" % dbPath,
940 externSstPath,
941 )
942 )
11fdf7f2
TL		 943 self.assertTrue(
944 self.ingestExternSst(
1e59de90
TL		 945 "--create_if_missing --allow_global_seqno --db=%s" % dbPath,
946 externSstPath,
947 )
948 )
949 self.assertRunOKFull(
950 "scan --db=%s" % dbPath, "x1 : y10\nx2 : y20\nx3 : y30\nx4 : y40"
951 )
952
11fdf7f2 953
7c673cae
FG		 954if __name__ == "__main__":
955 unittest.main()
Ceph