num_strays = mdc_stats['num_strays']
num_strays_purging = pq_stats['pq_executing']
num_purge_ops = pq_stats['pq_executing_ops']
+ files_high_water = pq_stats['pq_executing_high_water']
+ ops_high_water = pq_stats['pq_executing_ops_high_water']
- self.data_log.append([datetime.datetime.now(), num_strays, num_strays_purging, num_purge_ops])
-
- files_high_water = max(files_high_water, num_strays_purging)
- ops_high_water = max(ops_high_water, num_purge_ops)
+ self.data_log.append([datetime.datetime.now(), num_strays, num_strays_purging, num_purge_ops, files_high_water, ops_high_water])
total_strays_created = mdc_stats['strays_created']
total_strays_purged = pq_stats['pq_executed']
raise RuntimeError("Ops in flight high water is unexpectedly low ({0} / {1})".format(
ops_high_water, mds_max_purge_ops
))
+ # The MDS may go over mds_max_purge_ops for some items, like a
+ # heavily fragmented directory. The throttle does not kick in
+ # until *after* we reach or exceed the limit. This is expected
+ # because we don't want to starve the PQ or never purge a
+ # particularly large file/directory.
+ self.assertLessEqual(ops_high_water, mds_max_purge_ops+64)
elif throttle_type == self.FILES_THROTTLE:
if files_high_water < mds_max_purge_files / 2:
raise RuntimeError("Files in flight high water is unexpectedly low ({0} / {1})".format(
- ops_high_water, mds_max_purge_files
+ files_high_water, mds_max_purge_files
))
+ self.assertLessEqual(files_high_water, mds_max_purge_files)
# Sanity check all MDC stray stats
stats = self.fs.mds_asok(['perf', 'dump'])