]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/cephfs/test_forward_scrub.py
3 Test that the forward scrub functionality can traverse metadata and apply
4 requested tags, on well formed metadata.
6 This is *not* the real testing for forward scrub, which will need to test
7 how the functionality responds to damaged metadata.
13 from collections
import namedtuple
14 from io
import BytesIO
15 from textwrap
import dedent
17 from teuthology
.exceptions
import CommandFailedError
18 from tasks
.cephfs
.cephfs_test_case
import CephFSTestCase
22 log
= logging
.getLogger(__name__
)
25 ValidationError
= namedtuple("ValidationError", ["exception", "backtrace"])
28 class TestForwardScrub(CephFSTestCase
):
31 def _read_str_xattr(self
, pool
, obj
, attr
):
33 Read a ceph-encoded string from a rados xattr
35 output
= self
.fs
.mon_manager
.do_rados(["getxattr", obj
, attr
], pool
=pool
,
36 stdout
=BytesIO()).stdout
.getvalue()
37 strlen
= struct
.unpack('i', output
[0:4])[0]
38 return output
[4:(4 + strlen
)].decode(encoding
='ascii')
40 def _get_paths_to_ino(self
):
42 p
= self
.mount_a
.run_shell(["find", "./"])
43 paths
= p
.stdout
.getvalue().strip().split()
45 inos
[path
] = self
.mount_a
.path_to_ino(path
)
49 def test_apply_tag(self
):
50 self
.mount_a
.run_shell(["mkdir", "parentdir"])
51 self
.mount_a
.run_shell(["mkdir", "parentdir/childdir"])
52 self
.mount_a
.run_shell(["touch", "rfile"])
53 self
.mount_a
.run_shell(["touch", "parentdir/pfile"])
54 self
.mount_a
.run_shell(["touch", "parentdir/childdir/cfile"])
56 # Build a structure mapping path to inode, as we will later want
57 # to check object by object and objects are named after ino number
58 inos
= self
._get
_paths
_to
_ino
()
60 # Flush metadata: this is a friendly test of forward scrub so we're skipping
61 # the part where it's meant to cope with dirty metadata
62 self
.mount_a
.umount_wait()
63 self
.fs
.mds_asok(["flush", "journal"])
67 # Execute tagging forward scrub
68 self
.fs
.mds_asok(["tag", "path", "/parentdir", tag
])
72 # FIXME watching clog isn't a nice mechanism for this, once we have a ScrubMap we'll
75 # Check that dirs were tagged
76 for dirpath
in ["./parentdir", "./parentdir/childdir"]:
77 self
.assertTagged(inos
[dirpath
], tag
, self
.fs
.get_metadata_pool_name())
79 # Check that files were tagged
80 for filepath
in ["./parentdir/pfile", "./parentdir/childdir/cfile"]:
81 self
.assertTagged(inos
[filepath
], tag
, self
.fs
.get_data_pool_name())
83 # This guy wasn't in the tag path, shouldn't have been tagged
84 self
.assertUntagged(inos
["./rfile"])
86 def assertUntagged(self
, ino
):
87 file_obj_name
= "{0:x}.00000000".format(ino
)
88 with self
.assertRaises(CommandFailedError
):
90 self
.fs
.get_data_pool_name(),
95 def assertTagged(self
, ino
, tag
, pool
):
96 file_obj_name
= "{0:x}.00000000".format(ino
)
97 wrote
= self
._read
_str
_xattr
(
102 self
.assertEqual(wrote
, tag
)
104 def _validate_linkage(self
, expected
):
105 inos
= self
._get
_paths
_to
_ino
()
107 self
.assertDictEqual(inos
, expected
)
108 except AssertionError:
109 log
.error("Expected: {0}".format(json
.dumps(expected
, indent
=2)))
110 log
.error("Actual: {0}".format(json
.dumps(inos
, indent
=2)))
113 def test_orphan_scan(self
):
114 # Create some files whose metadata we will flush
115 self
.mount_a
.run_python(dedent("""
117 mount_point = "{mount_point}"
118 parent = os.path.join(mount_point, "parent")
120 flushed = os.path.join(parent, "flushed")
122 for f in ["alpha", "bravo", "charlie"]:
123 open(os.path.join(flushed, f), 'w').write(f)
124 """.format(mount_point
=self
.mount_a
.mountpoint
)))
126 inos
= self
._get
_paths
_to
_ino
()
129 # Umount before flush to avoid cap releases putting
130 # things we don't want in the journal later.
131 self
.mount_a
.umount_wait()
132 self
.fs
.mds_asok(["flush", "journal"])
134 # Create a new inode that's just in the log, i.e. would
135 # look orphaned to backward scan if backward scan wisnae
136 # respectin' tha scrub_tag xattr.
137 self
.mount_a
.mount_wait()
138 self
.mount_a
.run_shell(["mkdir", "parent/unflushed"])
139 self
.mount_a
.run_shell(["dd", "if=/dev/urandom",
140 "of=./parent/unflushed/jfile",
142 inos
["./parent/unflushed"] = self
.mount_a
.path_to_ino("./parent/unflushed")
143 inos
["./parent/unflushed/jfile"] = self
.mount_a
.path_to_ino("./parent/unflushed/jfile")
144 self
.mount_a
.umount_wait()
146 # Orphan an inode by deleting its dentry
147 # Our victim will be.... bravo.
148 self
.mount_a
.umount_wait()
150 self
.fs
.set_ceph_conf('mds', 'mds verify scatter', False)
151 self
.fs
.set_ceph_conf('mds', 'mds debug scatterstat', False)
152 frag_obj_id
= "{0:x}.00000000".format(inos
["./parent/flushed"])
153 self
.fs
.radosm(["rmomapkey", frag_obj_id
, "bravo_head"])
155 self
.fs
.set_joinable()
156 self
.fs
.wait_for_daemons()
158 # See that the orphaned file is indeed missing from a client's POV
159 self
.mount_a
.mount_wait()
160 damaged_state
= self
._get
_paths
_to
_ino
()
161 self
.assertNotIn("./parent/flushed/bravo", damaged_state
)
162 self
.mount_a
.umount_wait()
164 # Run a tagging forward scrub
166 self
.fs
.mds_asok(["tag", "path", "/parent", tag
])
168 # See that the orphan wisnae tagged
169 self
.assertUntagged(inos
['./parent/flushed/bravo'])
171 # See that the flushed-metadata-and-still-present files are tagged
172 self
.assertTagged(inos
['./parent/flushed/alpha'], tag
, self
.fs
.get_data_pool_name())
173 self
.assertTagged(inos
['./parent/flushed/charlie'], tag
, self
.fs
.get_data_pool_name())
175 # See that journalled-but-not-flushed file *was* tagged
176 self
.assertTagged(inos
['./parent/unflushed/jfile'], tag
, self
.fs
.get_data_pool_name())
178 # Run cephfs-data-scan targeting only orphans
180 self
.fs
.data_scan(["scan_extents", self
.fs
.get_data_pool_name()])
184 self
.fs
.get_data_pool_name()
187 # After in-place injection stats should be kosher again
188 self
.fs
.set_ceph_conf('mds', 'mds verify scatter', True)
189 self
.fs
.set_ceph_conf('mds', 'mds debug scatterstat', True)
191 # And we should have all the same linkage we started with,
192 # and no lost+found, and no extra inodes!
193 self
.fs
.set_joinable()
194 self
.fs
.wait_for_daemons()
195 self
.mount_a
.mount_wait()
196 self
._validate
_linkage
(inos
)
198 def _stash_inotable(self
):
199 # Get all active ranks
200 ranks
= self
.fs
.get_all_mds_rank()
204 inotable_oid
= "mds{rank:d}_".format(rank
=rank
) + "inotable"
205 print("Trying to fetch inotable object: " + inotable_oid
)
207 #self.fs.get_metadata_object("InoTable", "mds0_inotable")
208 inotable_raw
= self
.fs
.radosmo(['get', inotable_oid
, '-'])
209 inotable_dict
[inotable_oid
] = inotable_raw
212 def test_inotable_sync(self
):
213 self
.mount_a
.write_n_mb("file1_sixmegs", 6)
216 self
.mount_a
.umount_wait()
217 self
.fs
.mds_asok(["flush", "journal"])
219 inotable_copy
= self
._stash
_inotable
()
221 self
.mount_a
.mount_wait()
223 self
.mount_a
.write_n_mb("file2_sixmegs", 6)
224 self
.mount_a
.write_n_mb("file3_sixmegs", 6)
226 inos
= self
._get
_paths
_to
_ino
()
229 self
.mount_a
.umount_wait()
230 self
.fs
.mds_asok(["flush", "journal"])
232 self
.mount_a
.umount_wait()
234 with self
.assert_cluster_log("inode table repaired", invert_match
=True):
235 out_json
= self
.fs
.run_scrub(["start", "/", "repair,recursive"])
236 self
.assertNotEqual(out_json
, None)
237 self
.assertEqual(out_json
["return_code"], 0)
238 self
.assertEqual(self
.fs
.wait_until_scrub_complete(tag
=out_json
["scrub_tag"]), True)
242 # Truncate the journal (to ensure the inotable on disk
243 # is all that will be in the InoTable in memory)
245 self
.fs
.journal_tool(["event", "splice",
246 "--inode={0}".format(inos
["./file2_sixmegs"]), "summary"], 0)
248 self
.fs
.journal_tool(["event", "splice",
249 "--inode={0}".format(inos
["./file3_sixmegs"]), "summary"], 0)
251 # Revert to old inotable.
252 for key
, value
in inotable_copy
.items():
253 self
.fs
.radosm(["put", key
, "-"], stdin
=BytesIO(value
))
255 self
.fs
.set_joinable()
256 self
.fs
.wait_for_daemons()
258 with self
.assert_cluster_log("inode table repaired"):
259 out_json
= self
.fs
.run_scrub(["start", "/", "repair,recursive"])
260 self
.assertNotEqual(out_json
, None)
261 self
.assertEqual(out_json
["return_code"], 0)
262 self
.assertEqual(self
.fs
.wait_until_scrub_complete(tag
=out_json
["scrub_tag"]), True)
265 table_text
= self
.fs
.table_tool(["0", "show", "inode"])
266 table
= json
.loads(table_text
)
268 table
['0']['data']['inotable']['free'][0]['start'],
269 inos
['./file3_sixmegs'])
271 def test_backtrace_repair(self
):
273 That the MDS can repair an inodes backtrace in the data pool
274 if it is found to be damaged.
276 # Create a file for subsequent checks
277 self
.mount_a
.run_shell(["mkdir", "parent_a"])
278 self
.mount_a
.run_shell(["touch", "parent_a/alpha"])
279 file_ino
= self
.mount_a
.path_to_ino("parent_a/alpha")
281 # That backtrace and layout are written after initial flush
282 self
.fs
.mds_asok(["flush", "journal"])
283 backtrace
= self
.fs
.read_backtrace(file_ino
)
284 self
.assertEqual(['alpha', 'parent_a'],
285 [a
['dname'] for a
in backtrace
['ancestors']])
287 # Go corrupt the backtrace
288 self
.fs
._write
_data
_xattr
(file_ino
, "parent",
289 "oh i'm sorry did i overwrite your xattr?")
291 with self
.assert_cluster_log("bad backtrace on inode"):
292 out_json
= self
.fs
.run_scrub(["start", "/", "repair,recursive"])
293 self
.assertNotEqual(out_json
, None)
294 self
.assertEqual(out_json
["return_code"], 0)
295 self
.assertEqual(self
.fs
.wait_until_scrub_complete(tag
=out_json
["scrub_tag"]), True)
297 self
.fs
.mds_asok(["flush", "journal"])
298 backtrace
= self
.fs
.read_backtrace(file_ino
)
299 self
.assertEqual(['alpha', 'parent_a'],
300 [a
['dname'] for a
in backtrace
['ancestors']])