]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/ceph_objectstore_tool.py
2 ceph_objectstore_tool - Simple test of ceph-objectstore-tool utility
4 from cStringIO
import StringIO
8 from teuthology
import misc
as teuthology
12 from teuthology
.orchestra
import run
16 from util
.rados
import (rados
, create_replicated_pool
, create_ec_pool
)
17 # from util.rados import (rados, create_ec_pool,
18 # create_replicated_pool,
21 log
= logging
.getLogger(__name__
)
23 # Should get cluster name "ceph" from somewhere
24 # and normal path from osd_data and osd_journal in conf
25 FSPATH
= "/var/lib/ceph/osd/ceph-{id}"
26 JPATH
= "/var/lib/ceph/osd/ceph-{id}/journal"
29 def cod_setup_local_data(log
, ctx
, NUM_OBJECTS
, DATADIR
,
30 BASE_NAME
, DATALINECOUNT
):
31 objects
= range(1, NUM_OBJECTS
+ 1)
33 NAME
= BASE_NAME
+ "{num}".format(num
=i
)
34 LOCALNAME
= os
.path
.join(DATADIR
, NAME
)
36 dataline
= range(DATALINECOUNT
)
37 fd
= open(LOCALNAME
, "w")
38 data
= "This is the data for " + NAME
+ "\n"
44 def cod_setup_remote_data(log
, ctx
, remote
, NUM_OBJECTS
, DATADIR
,
45 BASE_NAME
, DATALINECOUNT
):
47 objects
= range(1, NUM_OBJECTS
+ 1)
49 NAME
= BASE_NAME
+ "{num}".format(num
=i
)
50 DDNAME
= os
.path
.join(DATADIR
, NAME
)
52 remote
.run(args
=['rm', '-f', DDNAME
])
54 dataline
= range(DATALINECOUNT
)
55 data
= "This is the data for " + NAME
+ "\n"
59 teuthology
.write_file(remote
, DDNAME
, DATA
)
62 def cod_setup(log
, ctx
, remote
, NUM_OBJECTS
, DATADIR
,
63 BASE_NAME
, DATALINECOUNT
, POOL
, db
, ec
):
65 log
.info("Creating {objs} objects in pool".format(objs
=NUM_OBJECTS
))
67 objects
= range(1, NUM_OBJECTS
+ 1)
69 NAME
= BASE_NAME
+ "{num}".format(num
=i
)
70 DDNAME
= os
.path
.join(DATADIR
, NAME
)
72 proc
= rados(ctx
, remote
, ['-p', POOL
, 'put', NAME
, DDNAME
],
74 # proc = remote.run(args=['rados', '-p', POOL, 'put', NAME, DDNAME])
77 log
.critical("Rados put failed with status {ret}".
78 format(ret
=proc
.exitstatus
))
84 db
[NAME
]["xattr"] = {}
88 mykey
= "key{i}-{k}".format(i
=i
, k
=k
)
89 myval
= "val{i}-{k}".format(i
=i
, k
=k
)
90 proc
= remote
.run(args
=['rados', '-p', POOL
, 'setxattr',
94 log
.error("setxattr failed with {ret}".format(ret
=ret
))
96 db
[NAME
]["xattr"][mykey
] = myval
98 # Erasure coded pools don't support omap
102 # Create omap header in all objects but REPobject1
104 myhdr
= "hdr{i}".format(i
=i
)
105 proc
= remote
.run(args
=['rados', '-p', POOL
, 'setomapheader',
109 log
.critical("setomapheader failed with {ret}".format(ret
=ret
))
111 db
[NAME
]["omapheader"] = myhdr
113 db
[NAME
]["omap"] = {}
117 mykey
= "okey{i}-{k}".format(i
=i
, k
=k
)
118 myval
= "oval{i}-{k}".format(i
=i
, k
=k
)
119 proc
= remote
.run(args
=['rados', '-p', POOL
, 'setomapval',
123 log
.critical("setomapval failed with {ret}".format(ret
=ret
))
124 db
[NAME
]["omap"][mykey
] = myval
129 def get_lines(filename
):
130 tmpfd
= open(filename
, "r")
134 line
= tmpfd
.readline().rstrip('\n')
142 @contextlib.contextmanager
143 def task(ctx
, config
):
145 Run ceph_objectstore_tool test
147 The config should be as follows::
149 ceph_objectstore_tool:
150 objects: 20 # <number of objects>
156 assert isinstance(config
, dict), \
157 'ceph_objectstore_tool task only accepts a dict for configuration'
159 log
.info('Beginning ceph_objectstore_tool...')
163 clients
= ctx
.cluster
.only(teuthology
.is_type('client'))
164 assert len(clients
.remotes
) > 0, 'Must specify at least 1 client'
165 (cli_remote
, _
) = clients
.remotes
.popitem()
166 log
.debug(cli_remote
)
168 # clients = dict(teuthology.get_clients(ctx=ctx, roles=config.keys()))
169 # client = clients.popitem()
171 osds
= ctx
.cluster
.only(teuthology
.is_type('osd'))
174 log
.info(osds
.remotes
)
176 manager
= ctx
.managers
['ceph']
177 while (len(manager
.get_osd_status()['up']) !=
178 len(manager
.get_osd_status()['raw'])):
180 while (len(manager
.get_osd_status()['in']) !=
181 len(manager
.get_osd_status()['up'])):
183 manager
.raw_cluster_cmd('osd', 'set', 'noout')
184 manager
.raw_cluster_cmd('osd', 'set', 'nodown')
186 PGNUM
= config
.get('pgnum', 12)
187 log
.info("pgnum: {num}".format(num
=PGNUM
))
191 REP_POOL
= "rep_pool"
192 REP_NAME
= "REPobject"
193 create_replicated_pool(cli_remote
, REP_POOL
, PGNUM
)
194 ERRORS
+= test_objectstore(ctx
, config
, cli_remote
, REP_POOL
, REP_NAME
)
198 create_ec_pool(cli_remote
, EC_POOL
, 'default', PGNUM
)
199 ERRORS
+= test_objectstore(ctx
, config
, cli_remote
,
200 EC_POOL
, EC_NAME
, ec
=True)
203 log
.info("TEST PASSED")
205 log
.error("TEST FAILED WITH {errcount} ERRORS".format(errcount
=ERRORS
))
212 log
.info('Ending ceph_objectstore_tool')
215 def test_objectstore(ctx
, config
, cli_remote
, REP_POOL
, REP_NAME
, ec
=False):
216 manager
= ctx
.managers
['ceph']
218 osds
= ctx
.cluster
.only(teuthology
.is_type('osd'))
220 TEUTHDIR
= teuthology
.get_testdir(ctx
)
221 DATADIR
= os
.path
.join(TEUTHDIR
, "ceph.data")
222 DATALINECOUNT
= 10000
224 NUM_OBJECTS
= config
.get('objects', 10)
225 log
.info("objects: {num}".format(num
=NUM_OBJECTS
))
227 pool_dump
= manager
.get_pool_dump(REP_POOL
)
228 REPID
= pool_dump
['pool']
230 log
.debug("repid={num}".format(num
=REPID
))
234 LOCALDIR
= tempfile
.mkdtemp("cod")
236 cod_setup_local_data(log
, ctx
, NUM_OBJECTS
, LOCALDIR
,
237 REP_NAME
, DATALINECOUNT
)
239 allremote
.append(cli_remote
)
240 allremote
+= osds
.remotes
.keys()
241 allremote
= list(set(allremote
))
242 for remote
in allremote
:
243 cod_setup_remote_data(log
, ctx
, remote
, NUM_OBJECTS
, DATADIR
,
244 REP_NAME
, DATALINECOUNT
)
246 ERRORS
+= cod_setup(log
, ctx
, cli_remote
, NUM_OBJECTS
, DATADIR
,
247 REP_NAME
, DATALINECOUNT
, REP_POOL
, db
, ec
)
250 for stats
in manager
.get_pg_stats():
251 if stats
["pgid"].find(str(REPID
) + ".") != 0:
253 if pool_dump
["type"] == ceph_manager
.CephManager
.REPLICATED_POOL
:
254 for osd
in stats
["acting"]:
255 pgs
.setdefault(osd
, []).append(stats
["pgid"])
256 elif pool_dump
["type"] == ceph_manager
.CephManager
.ERASURE_CODED_POOL
:
258 for osd
in stats
["acting"]:
259 pgs
.setdefault(osd
, []).append("{pgid}s{shard}".
260 format(pgid
=stats
["pgid"],
264 raise Exception("{pool} has an unexpected type {type}".
265 format(pool
=REP_POOL
, type=pool_dump
["type"]))
270 for osd
in manager
.get_osd_status()['up']:
271 manager
.kill_osd(osd
)
274 pgswithobjects
= set()
277 # Test --op list and generate json for all objects
278 log
.info("Test --op list by generating json for all objects")
279 prefix
= ("sudo ceph-objectstore-tool "
280 "--data-path {fpath} "
281 "--journal-path {jpath} ").format(fpath
=FSPATH
, jpath
=JPATH
)
282 for remote
in osds
.remotes
.iterkeys():
284 log
.debug(osds
.remotes
[remote
])
285 for role
in osds
.remotes
[remote
]:
286 if string
.find(role
, "osd.") != 0:
288 osdid
= int(role
.split('.')[1])
289 log
.info("process osd.{id} on {remote}".
290 format(id=osdid
, remote
=remote
))
291 cmd
= (prefix
+ "--op list").format(id=osdid
)
292 proc
= remote
.run(args
=cmd
.split(), check_status
=False,
294 if proc
.exitstatus
!= 0:
295 log
.error("Bad exit status {ret} from --op list request".
296 format(ret
=proc
.exitstatus
))
299 for pgline
in proc
.stdout
.getvalue().splitlines():
302 (pg
, obj
) = json
.loads(pgline
)
305 pgswithobjects
.add(pg
)
306 objsinpg
.setdefault(pg
, []).append(name
)
307 db
[name
].setdefault("pg2json",
308 {})[pg
] = json
.dumps(obj
)
311 log
.info(pgswithobjects
)
314 if pool_dump
["type"] == ceph_manager
.CephManager
.REPLICATED_POOL
:
316 log
.info("Test get-bytes and set-bytes")
317 for basename
in db
.keys():
318 file = os
.path
.join(DATADIR
, basename
)
319 GETNAME
= os
.path
.join(DATADIR
, "get")
320 SETNAME
= os
.path
.join(DATADIR
, "set")
322 for remote
in osds
.remotes
.iterkeys():
323 for role
in osds
.remotes
[remote
]:
324 if string
.find(role
, "osd.") != 0:
326 osdid
= int(role
.split('.')[1])
330 for pg
, JSON
in db
[basename
]["pg2json"].iteritems():
332 cmd
= ((prefix
+ "--pgid {pg}").
333 format(id=osdid
, pg
=pg
).split())
334 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
335 cmd
+= ("get-bytes {fname}".
336 format(fname
=GETNAME
).split())
337 proc
= remote
.run(args
=cmd
, check_status
=False)
338 if proc
.exitstatus
!= 0:
339 remote
.run(args
="rm -f {getfile}".
340 format(getfile
=GETNAME
).split())
341 log
.error("Bad exit status {ret}".
342 format(ret
=proc
.exitstatus
))
345 cmd
= ("diff -q {file} {getfile}".
346 format(file=file, getfile
=GETNAME
))
347 proc
= remote
.run(args
=cmd
.split())
348 if proc
.exitstatus
!= 0:
349 log
.error("Data from get-bytes differ")
351 # cat_file(logging.DEBUG, GETNAME)
352 # log.debug("Expected:")
353 # cat_file(logging.DEBUG, file)
355 remote
.run(args
="rm -f {getfile}".
356 format(getfile
=GETNAME
).split())
358 data
= ("put-bytes going into {file}\n".
360 teuthology
.write_file(remote
, SETNAME
, data
)
361 cmd
= ((prefix
+ "--pgid {pg}").
362 format(id=osdid
, pg
=pg
).split())
363 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
364 cmd
+= ("set-bytes {fname}".
365 format(fname
=SETNAME
).split())
366 proc
= remote
.run(args
=cmd
, check_status
=False)
368 if proc
.exitstatus
!= 0:
369 log
.info("set-bytes failed for object {obj} "
370 "in pg {pg} osd.{id} ret={ret}".
371 format(obj
=basename
, pg
=pg
,
372 id=osdid
, ret
=proc
.exitstatus
))
375 cmd
= ((prefix
+ "--pgid {pg}").
376 format(id=osdid
, pg
=pg
).split())
377 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
378 cmd
+= "get-bytes -".split()
379 proc
= remote
.run(args
=cmd
, check_status
=False,
382 if proc
.exitstatus
!= 0:
383 log
.error("get-bytes after "
384 "set-bytes ret={ret}".
385 format(ret
=proc
.exitstatus
))
388 if data
!= proc
.stdout
.getvalue():
389 log
.error("Data inconsistent after "
391 log
.error(proc
.stdout
.getvalue())
394 cmd
= ((prefix
+ "--pgid {pg}").
395 format(id=osdid
, pg
=pg
).split())
396 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
397 cmd
+= ("set-bytes {fname}".
398 format(fname
=file).split())
399 proc
= remote
.run(args
=cmd
, check_status
=False)
401 if proc
.exitstatus
!= 0:
402 log
.info("set-bytes failed for object {obj} "
403 "in pg {pg} osd.{id} ret={ret}".
404 format(obj
=basename
, pg
=pg
,
405 id=osdid
, ret
=proc
.exitstatus
))
408 log
.info("Test list-attrs get-attr")
409 for basename
in db
.keys():
410 file = os
.path
.join(DATADIR
, basename
)
411 GETNAME
= os
.path
.join(DATADIR
, "get")
412 SETNAME
= os
.path
.join(DATADIR
, "set")
414 for remote
in osds
.remotes
.iterkeys():
415 for role
in osds
.remotes
[remote
]:
416 if string
.find(role
, "osd.") != 0:
418 osdid
= int(role
.split('.')[1])
422 for pg
, JSON
in db
[basename
]["pg2json"].iteritems():
424 cmd
= ((prefix
+ "--pgid {pg}").
425 format(id=osdid
, pg
=pg
).split())
426 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
427 cmd
+= ["list-attrs"]
428 proc
= remote
.run(args
=cmd
, check_status
=False,
429 stdout
=StringIO(), stderr
=StringIO())
431 if proc
.exitstatus
!= 0:
432 log
.error("Bad exit status {ret}".
433 format(ret
=proc
.exitstatus
))
436 keys
= proc
.stdout
.getvalue().split()
437 values
= dict(db
[basename
]["xattr"])
445 if key
not in values
:
446 log
.error("The key {key} should be present".
450 exp
= values
.pop(key
)
451 cmd
= ((prefix
+ "--pgid {pg}").
452 format(id=osdid
, pg
=pg
).split())
453 cmd
.append(run
.Raw("'{json}'".format(json
=JSON
)))
454 cmd
+= ("get-attr {key}".
455 format(key
="_" + key
).split())
456 proc
= remote
.run(args
=cmd
, check_status
=False,
459 if proc
.exitstatus
!= 0:
460 log
.error("get-attr failed with {ret}".
461 format(ret
=proc
.exitstatus
))
464 val
= proc
.stdout
.getvalue()
466 log
.error("For key {key} got value {got} "
467 "instead of {expected}".
468 format(key
=key
, got
=val
,
471 if "hinfo_key" in keys
:
472 cmd_prefix
= prefix
.format(id=osdid
)
474 expected=$({prefix} --pgid {pg} '{json}' get-attr {key} | base64)
475 echo placeholder | {prefix} --pgid {pg} '{json}' set-attr {key} -
476 test $({prefix} --pgid {pg} '{json}' get-attr {key}) = placeholder
477 echo $expected | base64 --decode | \
478 {prefix} --pgid {pg} '{json}' set-attr {key} -
479 test $({prefix} --pgid {pg} '{json}' get-attr {key} | base64) = $expected
480 """.format(prefix
=cmd_prefix
, pg
=pg
, json
=JSON
,
483 proc
= remote
.run(args
=['bash', '-e', '-x',
489 if proc
.exitstatus
!= 0:
490 log
.error("failed with " +
491 str(proc
.exitstatus
))
492 log
.error(proc
.stdout
.getvalue() + " " +
493 proc
.stderr
.getvalue())
497 log
.error("Not all keys found, remaining keys:")
500 log
.info("Test pg info")
501 for remote
in osds
.remotes
.iterkeys():
502 for role
in osds
.remotes
[remote
]:
503 if string
.find(role
, "osd.") != 0:
505 osdid
= int(role
.split('.')[1])
509 for pg
in pgs
[osdid
]:
510 cmd
= ((prefix
+ "--op info --pgid {pg}").
511 format(id=osdid
, pg
=pg
).split())
512 proc
= remote
.run(args
=cmd
, check_status
=False,
515 if proc
.exitstatus
!= 0:
516 log
.error("Failure of --op info command with {ret}".
517 format(proc
.exitstatus
))
520 info
= proc
.stdout
.getvalue()
521 if not str(pg
) in info
:
522 log
.error("Bad data from info: {info}".format(info
=info
))
525 log
.info("Test pg logging")
526 for remote
in osds
.remotes
.iterkeys():
527 for role
in osds
.remotes
[remote
]:
528 if string
.find(role
, "osd.") != 0:
530 osdid
= int(role
.split('.')[1])
534 for pg
in pgs
[osdid
]:
535 cmd
= ((prefix
+ "--op log --pgid {pg}").
536 format(id=osdid
, pg
=pg
).split())
537 proc
= remote
.run(args
=cmd
, check_status
=False,
540 if proc
.exitstatus
!= 0:
541 log
.error("Getting log failed for pg {pg} "
542 "from osd.{id} with {ret}".
543 format(pg
=pg
, id=osdid
, ret
=proc
.exitstatus
))
546 HASOBJ
= pg
in pgswithobjects
547 MODOBJ
= "modify" in proc
.stdout
.getvalue()
549 log
.error("Bad log for pg {pg} from osd.{id}".
550 format(pg
=pg
, id=osdid
))
551 MSG
= (HASOBJ
and [""] or ["NOT "])[0]
552 log
.error("Log should {msg}have a modify entry".
556 log
.info("Test pg export")
558 for remote
in osds
.remotes
.iterkeys():
559 for role
in osds
.remotes
[remote
]:
560 if string
.find(role
, "osd.") != 0:
562 osdid
= int(role
.split('.')[1])
566 for pg
in pgs
[osdid
]:
567 fpath
= os
.path
.join(DATADIR
, "osd{id}.{pg}".
568 format(id=osdid
, pg
=pg
))
570 cmd
= ((prefix
+ "--op export --pgid {pg} --file {file}").
571 format(id=osdid
, pg
=pg
, file=fpath
))
572 proc
= remote
.run(args
=cmd
, check_status
=False,
575 if proc
.exitstatus
!= 0:
576 log
.error("Exporting failed for pg {pg} "
577 "on osd.{id} with {ret}".
578 format(pg
=pg
, id=osdid
, ret
=proc
.exitstatus
))
583 log
.info("Test pg removal")
585 for remote
in osds
.remotes
.iterkeys():
586 for role
in osds
.remotes
[remote
]:
587 if string
.find(role
, "osd.") != 0:
589 osdid
= int(role
.split('.')[1])
593 for pg
in pgs
[osdid
]:
594 cmd
= ((prefix
+ "--force --op remove --pgid {pg}").
595 format(pg
=pg
, id=osdid
))
596 proc
= remote
.run(args
=cmd
, check_status
=False,
599 if proc
.exitstatus
!= 0:
600 log
.error("Removing failed for pg {pg} "
601 "on osd.{id} with {ret}".
602 format(pg
=pg
, id=osdid
, ret
=proc
.exitstatus
))
608 if EXP_ERRORS
== 0 and RM_ERRORS
== 0:
609 log
.info("Test pg import")
611 for remote
in osds
.remotes
.iterkeys():
612 for role
in osds
.remotes
[remote
]:
613 if string
.find(role
, "osd.") != 0:
615 osdid
= int(role
.split('.')[1])
619 for pg
in pgs
[osdid
]:
620 fpath
= os
.path
.join(DATADIR
, "osd{id}.{pg}".
621 format(id=osdid
, pg
=pg
))
623 cmd
= ((prefix
+ "--op import --file {file}").
624 format(id=osdid
, file=fpath
))
625 proc
= remote
.run(args
=cmd
, check_status
=False,
628 if proc
.exitstatus
!= 0:
629 log
.error("Import failed from {file} with {ret}".
630 format(file=fpath
, ret
=proc
.exitstatus
))
633 log
.warning("SKIPPING IMPORT TESTS DUE TO PREVIOUS FAILURES")
637 if EXP_ERRORS
== 0 and RM_ERRORS
== 0 and IMP_ERRORS
== 0:
638 log
.info("Restarting OSDs....")
639 # They are still look to be up because of setting nodown
640 for osd
in manager
.get_osd_status()['up']:
641 manager
.revive_osd(osd
)
644 # Let scrub after test runs verify consistency of all copies
645 log
.info("Verify replicated import data")
646 objects
= range(1, NUM_OBJECTS
+ 1)
648 NAME
= REP_NAME
+ "{num}".format(num
=i
)
649 TESTNAME
= os
.path
.join(DATADIR
, "gettest")
650 REFNAME
= os
.path
.join(DATADIR
, NAME
)
652 proc
= rados(ctx
, cli_remote
,
653 ['-p', REP_POOL
, 'get', NAME
, TESTNAME
], wait
=False)
657 log
.error("After import, rados get failed with {ret}".
658 format(ret
=proc
.exitstatus
))
662 cmd
= "diff -q {gettest} {ref}".format(gettest
=TESTNAME
,
664 proc
= cli_remote
.run(args
=cmd
, check_status
=False)
666 if proc
.exitstatus
!= 0:
667 log
.error("Data comparison failed for {obj}".format(obj
=NAME
))