ceph/src/mds/ScrubStack.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2014 Red Hat
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include <iostream>
  16
  17 #include "ScrubStack.h"
  18 #include "common/Finisher.h"
  19 #include "mds/MDSRank.h"
  20 #include "mds/MDCache.h"
  21 #include "mds/MDSContinuation.h"
  22
  23 #define dout_context g_ceph_context
  24 #define dout_subsys ceph_subsys_mds
  25 #undef dout_prefix
  26 #define dout_prefix _prefix(_dout, scrubstack->mdcache->mds)
  27 static ostream& _prefix(std::ostream *_dout, MDSRank *mds) {
  28   return *_dout << "mds." << mds->get_nodeid() << ".scrubstack ";
  29 }
  30
  31 std::ostream &operator<<(std::ostream &os, const ScrubStack::State &state) {
  32   switch(state) {
  33   case ScrubStack::STATE_RUNNING:
  34     os << "RUNNING";
  35     break;
  36   case ScrubStack::STATE_IDLE:
  37     os << "IDLE";
  38     break;
  39   case ScrubStack::STATE_PAUSING:
  40     os << "PAUSING";
  41     break;
  42   case ScrubStack::STATE_PAUSED:
  43     os << "PAUSED";
  44     break;
  45   default:
  46     ceph_abort();
  47   }
  48
  49   return os;
  50 }
  51
  52 void ScrubStack::push_inode(CInode *in)
  53 {
  54   dout(20) << "pushing " << *in << " on top of ScrubStack" << dendl;
  55   if (!in->item_scrub.is_on_list()) {
  56     in->get(CInode::PIN_SCRUBQUEUE);
  57     stack_size++;
  58   }
  59   inode_stack.push_front(&in->item_scrub);
  60 }
  61
  62 void ScrubStack::push_inode_bottom(CInode *in)
  63 {
  64   dout(20) << "pushing " << *in << " on bottom of ScrubStack" << dendl;
  65   if (!in->item_scrub.is_on_list()) {
  66     in->get(CInode::PIN_SCRUBQUEUE);
  67     stack_size++;
  68   }
  69   inode_stack.push_back(&in->item_scrub);
  70 }
  71
  72 void ScrubStack::pop_inode(CInode *in)
  73 {
  74   dout(20) << "popping " << *in
  75           << " off of ScrubStack" << dendl;
  76   ceph_assert(in->item_scrub.is_on_list());
  77   in->put(CInode::PIN_SCRUBQUEUE);
  78   in->item_scrub.remove_myself();
  79   stack_size--;
  80 }
  81
  82 void ScrubStack::_enqueue_inode(CInode *in, CDentry *parent,
  83                                 ScrubHeaderRef& header,
  84                                 MDSContext *on_finish, bool top)
  85 {
  86   dout(10) << __func__ << " with {" << *in << "}"
  87            << ", on_finish=" << on_finish << ", top=" << top << dendl;
  88   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
  89   in->scrub_initialize(parent, header, on_finish);
  90   if (top)
  91     push_inode(in);
  92   else
  93     push_inode_bottom(in);
  94 }
  95
  96 void ScrubStack::enqueue_inode(CInode *in, ScrubHeaderRef& header,
  97                                MDSContext *on_finish, bool top)
  98 {
  99   // abort in progress
 100   if (clear_inode_stack) {
 101     on_finish->complete(-EAGAIN);
 102     return;
 103   }
 104
 105   _enqueue_inode(in, NULL, header, on_finish, top);
 106   kick_off_scrubs();
 107 }
 108
 109 void ScrubStack::kick_off_scrubs()
 110 {
 111   ceph_assert(ceph_mutex_is_locked(mdcache->mds->mds_lock));
 112   dout(20) << __func__ << ": state=" << state << dendl;
 113
 114   if (clear_inode_stack || state == STATE_PAUSING || state == STATE_PAUSED) {
 115     if (scrubs_in_progress == 0) {
 116       dout(10) << __func__ << ": in progress scrub operations finished, "
 117                << stack_size << " in the stack" << dendl;
 118
 119       State final_state = state;
 120       if (clear_inode_stack) {
 121         abort_pending_scrubs();
 122         final_state = STATE_IDLE;
 123       }
 124       if (state == STATE_PAUSING) {
 125         final_state = STATE_PAUSED;
 126       }
 127
 128       set_state(final_state);
 129       complete_control_contexts(0);
 130     }
 131
 132     return;
 133   }
 134
 135   dout(20) << __func__ << " entering with " << scrubs_in_progress << " in "
 136               "progress and " << stack_size << " in the stack" << dendl;
 137   bool can_continue = true;
 138   elist<CInode*>::iterator i = inode_stack.begin();
 139   while (g_conf()->mds_max_scrub_ops_in_progress > scrubs_in_progress &&
 140          can_continue) {
 141     if (i.end()) {
 142       if (scrubs_in_progress == 0) {
 143         set_state(STATE_IDLE);
 144       }
 145
 146       return;
 147     }
 148
 149     assert(state == STATE_RUNNING || state == STATE_IDLE);
 150     set_state(STATE_RUNNING);
 151
 152     CInode *curi = *i;
 153     ++i; // we have our reference, push iterator forward
 154
 155     dout(20) << __func__ << " examining " << *curi << dendl;
 156
 157     if (!curi->is_dir()) {
 158       // it's a regular file, symlink, or hard link
 159       pop_inode(curi); // we only touch it this once, so remove from stack
 160
 161       if (!curi->scrub_info()->on_finish) {
 162         scrubs_in_progress++;
 163         curi->scrub_set_finisher(&scrub_kick);
 164       }
 165       scrub_file_inode(curi);
 166       can_continue = true;
 167     } else {
 168       bool completed; // it's done, so pop it off the stack
 169       bool terminal; // not done, but we can start ops on other directories
 170       bool progress; // it added new dentries to the top of the stack
 171       scrub_dir_inode(curi, &progress, &terminal, &completed);
 172       if (completed) {
 173         dout(20) << __func__ << " dir completed" << dendl;
 174         pop_inode(curi);
 175       } else if (progress) {
 176         dout(20) << __func__ << " dir progressed" << dendl;
 177         // we added new stuff to top of stack, so reset ourselves there
 178         i = inode_stack.begin();
 179       } else {
 180         dout(20) << __func__ << " dir no-op" << dendl;
 181       }
 182
 183       can_continue = progress || terminal || completed;
 184     }
 185   }
 186 }
 187
 188 void ScrubStack::scrub_dir_inode(CInode *in,
 189                                  bool *added_children,
 190                                  bool *terminal,
 191                                  bool *done)
 192 {
 193   dout(10) << __func__ << " " << *in << dendl;
 194
 195   *added_children = false;
 196   bool all_frags_terminal = true;
 197   bool all_frags_done = true;
 198
 199   ScrubHeaderRef header = in->get_scrub_header();
 200   ceph_assert(header != nullptr);
 201
 202   if (header->get_recursive()) {
 203     frag_vec_t scrubbing_frags;
 204     std::queue<CDir*> scrubbing_cdirs;
 205     in->scrub_dirfrags_scrubbing(&scrubbing_frags);
 206     dout(20) << __func__ << " iterating over " << scrubbing_frags.size()
 207       << " scrubbing frags" << dendl;
 208     for (const auto& fg : scrubbing_frags) {
 209       // turn frags into CDir *
 210       CDir *dir = in->get_dirfrag(fg);
 211       if (dir) {
 212         scrubbing_cdirs.push(dir);
 213         dout(25) << __func__ << " got CDir " << *dir << " presently scrubbing" << dendl;
 214       } else {
 215         in->scrub_dirfrag_finished(fg);
 216         dout(25) << __func__ << " missing dirfrag " << fg << " skip scrubbing" << dendl;
 217       }
 218     }
 219
 220     dout(20) << __func__ << " consuming from " << scrubbing_cdirs.size()
 221              << " scrubbing cdirs" << dendl;
 222
 223     while (g_conf()->mds_max_scrub_ops_in_progress > scrubs_in_progress) {
 224       // select next CDir
 225       CDir *cur_dir = NULL;
 226       if (!scrubbing_cdirs.empty()) {
 227         cur_dir = scrubbing_cdirs.front();
 228         scrubbing_cdirs.pop();
 229         dout(20) << __func__ << " got cur_dir = " << *cur_dir << dendl;
 230       } else {
 231         bool ready = get_next_cdir(in, &cur_dir);
 232         dout(20) << __func__ << " get_next_cdir ready=" << ready << dendl;
 233
 234         if (ready && cur_dir) {
 235           scrubbing_cdirs.push(cur_dir);
 236         } else if (!ready) {
 237           // We are waiting for load of a frag
 238           all_frags_done = false;
 239           all_frags_terminal = false;
 240           break;
 241         } else {
 242           // Finished with all frags
 243           break;
 244         }
 245       }
 246       // scrub that CDir
 247       bool frag_added_children = false;
 248       bool frag_terminal = true;
 249       bool frag_done = false;
 250       scrub_dirfrag(cur_dir, header,
 251                     &frag_added_children, &frag_terminal, &frag_done);
 252       if (frag_done) {
 253         cur_dir->inode->scrub_dirfrag_finished(cur_dir->frag);
 254       }
 255       *added_children |= frag_added_children;
 256       all_frags_terminal = all_frags_terminal && frag_terminal;
 257       all_frags_done = all_frags_done && frag_done;
 258     }
 259
 260     dout(20) << "finished looping; all_frags_terminal=" << all_frags_terminal
 261              << ", all_frags_done=" << all_frags_done << dendl;
 262   } else {
 263     dout(20) << "!scrub_recursive" << dendl;
 264   }
 265
 266   if (all_frags_done) {
 267     assert (!*added_children); // can't do this if children are still pending
 268
 269     // OK, so now I can... fire off a validate on the dir inode, and
 270     // when it completes, come through here again, noticing that we've
 271     // set a flag to indicate the validate happened, and
 272     scrub_dir_inode_final(in);
 273   }
 274
 275   *terminal = all_frags_terminal;
 276   *done = all_frags_done;
 277   dout(10) << __func__ << " is exiting " << *terminal << " " << *done << dendl;
 278   return;
 279 }
 280
 281 bool ScrubStack::get_next_cdir(CInode *in, CDir **new_dir)
 282 {
 283   dout(20) << __func__ << " on " << *in << dendl;
 284   frag_t next_frag;
 285   int r = in->scrub_dirfrag_next(&next_frag);
 286   assert (r >= 0);
 287
 288   if (r == 0) {
 289     // we got a frag to scrub, otherwise it would be ENOENT
 290     dout(25) << "looking up new frag " << next_frag << dendl;
 291     CDir *next_dir = in->get_or_open_dirfrag(mdcache, next_frag);
 292     if (!next_dir->is_complete()) {
 293       scrubs_in_progress++;
 294       next_dir->fetch(&scrub_kick);
 295       dout(25) << "fetching frag from RADOS" << dendl;
 296       return false;
 297     }
 298     *new_dir = next_dir;
 299     dout(25) << "returning dir " << *new_dir << dendl;
 300     return true;
 301   }
 302   ceph_assert(r == ENOENT);
 303   // there are no dirfrags left
 304   *new_dir = NULL;
 305   return true;
 306 }
 307
 308 class C_InodeValidated : public MDSInternalContext
 309 {
 310   public:
 311     ScrubStack *stack;
 312     CInode::validated_data result;
 313     CInode *target;
 314
 315     C_InodeValidated(MDSRank *mds, ScrubStack *stack_, CInode *target_)
 316       : MDSInternalContext(mds), stack(stack_), target(target_)
 317     {}
 318
 319     void finish(int r) override
 320     {
 321       stack->_validate_inode_done(target, r, result);
 322     }
 323 };
 324
 325
 326 void ScrubStack::scrub_dir_inode_final(CInode *in)
 327 {
 328   dout(20) << __func__ << " " << *in << dendl;
 329
 330   // Two passes through this function.  First one triggers inode validation,
 331   // second one sets finally_done
 332   // FIXME: kind of overloading scrub_in_progress here, using it while
 333   // dentry is still on stack to indicate that we have finished
 334   // doing our validate_disk_state on the inode
 335   // FIXME: the magic-constructing scrub_info() is going to leave
 336   // an unneeded scrub_infop lying around here
 337   if (!in->scrub_info()->children_scrubbed) {
 338     if (!in->scrub_info()->on_finish) {
 339       scrubs_in_progress++;
 340       in->scrub_set_finisher(&scrub_kick);
 341     }
 342
 343     in->scrub_children_finished();
 344     C_InodeValidated *fin = new C_InodeValidated(mdcache->mds, this, in);
 345     in->validate_disk_state(&fin->result, fin);
 346   }
 347
 348   return;
 349 }
 350
 351 void ScrubStack::scrub_dirfrag(CDir *dir,
 352                                ScrubHeaderRef& header,
 353                                bool *added_children, bool *is_terminal,
 354                                bool *done)
 355 {
 356   ceph_assert(dir != NULL);
 357
 358   dout(20) << __func__ << " on " << *dir << dendl;
 359   *added_children = false;
 360   *is_terminal = false;
 361   *done = false;
 362
 363
 364   if (!dir->scrub_info()->directory_scrubbing) {
 365     // Get the frag complete before calling
 366     // scrub initialize, so that it can populate its lists
 367     // of dentries.
 368     if (!dir->is_complete()) {
 369       scrubs_in_progress++;
 370       dir->fetch(&scrub_kick);
 371       return;
 372     }
 373
 374     dir->scrub_initialize(header);
 375   }
 376
 377   int r = 0;
 378   while(r == 0) {
 379     CDentry *dn = NULL;
 380     scrubs_in_progress++;
 381     r = dir->scrub_dentry_next(&scrub_kick, &dn);
 382     if (r != EAGAIN) {
 383       scrubs_in_progress--;
 384     }
 385
 386     if (r == EAGAIN) {
 387       // Drop out, CDir fetcher will call back our kicker context
 388       dout(20) << __func__ << " waiting for fetch on " << *dir << dendl;
 389       return;
 390     }
 391
 392     if (r == ENOENT) {
 393       // Nothing left to scrub, are we done?
 394       auto&& scrubbing = dir->scrub_dentries_scrubbing();
 395       if (scrubbing.empty()) {
 396         dout(20) << __func__ << " dirfrag done: " << *dir << dendl;
 397         // FIXME: greg: What's the diff meant to be between done and terminal
 398         dir->scrub_finished();
 399         *done = true;
 400         *is_terminal = true;
 401       } else {
 402         dout(20) << __func__ << " " << scrubbing.size() << " dentries still "
 403                    "scrubbing in " << *dir << dendl;
 404       }
 405       return;
 406     }
 407
 408     // scrub_dentry_next defined to only give EAGAIN, ENOENT, 0 -- we should
 409     // never get random IO errors here.
 410     ceph_assert(r == 0);
 411
 412     _enqueue_inode(dn->get_projected_inode(), dn, header, NULL, true);
 413
 414     *added_children = true;
 415   }
 416 }
 417
 418 void ScrubStack::scrub_file_inode(CInode *in)
 419 {
 420   C_InodeValidated *fin = new C_InodeValidated(mdcache->mds, this, in);
 421   // At this stage the DN is already past scrub_initialize, so
 422   // it's in the cache, it has PIN_SCRUBQUEUE and it is authpinned
 423   in->validate_disk_state(&fin->result, fin);
 424 }
 425
 426 void ScrubStack::_validate_inode_done(CInode *in, int r,
 427                                       const CInode::validated_data &result)
 428 {
 429   LogChannelRef clog = mdcache->mds->clog;
 430   const ScrubHeaderRefConst header = in->scrub_info()->header;
 431
 432   std::string path;
 433   if (!result.passed_validation) {
 434     // Build path string for use in messages
 435     in->make_path_string(path, true);
 436   }
 437
 438   if (result.backtrace.checked && !result.backtrace.passed &&
 439       !result.backtrace.repaired)
 440   {
 441     // Record backtrace fails as remote linkage damage, as
 442     // we may not be able to resolve hard links to this inode
 443     mdcache->mds->damage_table.notify_remote_damaged(in->inode.ino, path);
 444   } else if (result.inode.checked && !result.inode.passed &&
 445              !result.inode.repaired) {
 446     // Record damaged inode structures as damaged dentries as
 447     // that is where they are stored
 448     auto parent = in->get_projected_parent_dn();
 449     if (parent) {
 450       auto dir = parent->get_dir();
 451       mdcache->mds->damage_table.notify_dentry(
 452           dir->inode->ino(), dir->frag, parent->last, parent->get_name(), path);
 453     }
 454   }
 455
 456   // Inform the cluster log if we found an error
 457   if (!result.passed_validation) {
 458     if (result.all_damage_repaired()) {
 459       clog->info() << "Scrub repaired inode " << in->ino()
 460                    << " (" << path << ")";
 461     } else {
 462       clog->warn() << "Scrub error on inode " << in->ino()
 463                    << " (" << path << ") see " << g_conf()->name
 464                    << " log and `damage ls` output for details";
 465     }
 466
 467     // Put the verbose JSON output into the MDS log for later inspection
 468     JSONFormatter f;
 469     result.dump(&f);
 470     std::ostringstream out;
 471     f.flush(out);
 472     derr << __func__ << " scrub error on inode " << *in << ": " << out.str()
 473          << dendl;
 474   } else {
 475     dout(10) << __func__ << " scrub passed on inode " << *in << dendl;
 476   }
 477
 478   MDSContext *c = NULL;
 479   in->scrub_finished(&c);
 480
 481   if (in == header->get_origin()) {
 482     scrub_origins.erase(in);
 483     clog_scrub_summary(in);
 484     if (!header->get_recursive()) {
 485       if (r >= 0) { // we got into the scrubbing dump it
 486         result.dump(&(header->get_formatter()));
 487       } else { // we failed the lookup or something; dump ourselves
 488         header->get_formatter().open_object_section("results");
 489         header->get_formatter().dump_int("return_code", r);
 490         header->get_formatter().close_section(); // results
 491       }
 492     }
 493   }
 494   if (c) {
 495     finisher->queue(new MDSIOContextWrapper(mdcache->mds, c), 0);
 496   }
 497 }
 498
 499 ScrubStack::C_KickOffScrubs::C_KickOffScrubs(MDCache *mdcache, ScrubStack *s)
 500   : MDSInternalContext(mdcache->mds), stack(s) { }
 501
 502 void ScrubStack::complete_control_contexts(int r) {
 503   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 504
 505   for (auto &ctx : control_ctxs) {
 506     ctx->complete(r);
 507   }
 508   control_ctxs.clear();
 509 }
 510
 511 void ScrubStack::set_state(State next_state) {
 512     if (state != next_state) {
 513       dout(20) << __func__ << ", from state=" << state << ", to state="
 514                << next_state << dendl;
 515       state = next_state;
 516       clog_scrub_summary();
 517     }
 518 }
 519
 520 bool ScrubStack::scrub_in_transition_state() {
 521   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 522   dout(20) << __func__ << ": state=" << state << dendl;
 523
 524   // STATE_RUNNING is considered as a transition state so as to
 525   // "delay" the scrub control operation.
 526   if (state == STATE_RUNNING || state == STATE_PAUSING) {
 527     return true;
 528   }
 529
 530   return false;
 531 }
 532
 533 std::string_view ScrubStack::scrub_summary() {
 534   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 535
 536   bool have_more = false;
 537   CachedStackStringStream cs;
 538
 539   if (state == STATE_IDLE) {
 540     return "idle";
 541   }
 542
 543   if (state == STATE_RUNNING) {
 544     if (clear_inode_stack) {
 545       *cs << "aborting";
 546     } else {
 547       *cs << "active";
 548     }
 549   } else {
 550     if (state == STATE_PAUSING) {
 551       have_more = true;
 552       *cs << "pausing";
 553     } else if (state == STATE_PAUSED) {
 554       have_more = true;
 555       *cs << "paused";
 556     }
 557
 558     if (clear_inode_stack) {
 559       if (have_more) {
 560         *cs << "+";
 561       }
 562       *cs << "aborting";
 563     }
 564   }
 565
 566   if (!scrub_origins.empty()) {
 567     *cs << " [paths:";
 568     for (auto inode = scrub_origins.begin(); inode != scrub_origins.end(); ++inode) {
 569       if (inode != scrub_origins.begin()) {
 570         *cs << ",";
 571       }
 572
 573       *cs << scrub_inode_path(*inode);
 574     }
 575     *cs << "]";
 576   }
 577
 578   return cs->strv();
 579 }
 580
 581 void ScrubStack::scrub_status(Formatter *f) {
 582   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 583
 584   f->open_object_section("result");
 585
 586   std::stringstream ss;
 587   bool have_more = false;
 588
 589   if (state == STATE_IDLE) {
 590     ss << "no active scrubs running";
 591   } else if (state == STATE_RUNNING) {
 592     if (clear_inode_stack) {
 593       ss << "ABORTING";
 594     } else {
 595       ss << "scrub active";
 596     }
 597     ss << " (" << stack_size << " inodes in the stack)";
 598   } else {
 599     if (state == STATE_PAUSING || state == STATE_PAUSED) {
 600       have_more = true;
 601       ss << state;
 602     }
 603     if (clear_inode_stack) {
 604       if (have_more) {
 605         ss << "+";
 606       }
 607       ss << "ABORTING";
 608     }
 609
 610     ss << " (" << stack_size << " inodes in the stack)";
 611   }
 612   f->dump_string("status", ss.str());
 613
 614   f->open_object_section("scrubs");
 615   for (auto &inode : scrub_origins) {
 616     have_more = false;
 617     ScrubHeaderRefConst header = inode->get_scrub_header();
 618
 619     std::string tag(header->get_tag());
 620     f->open_object_section(tag.c_str()); // scrub id
 621
 622     f->dump_string("path", scrub_inode_path(inode));
 623
 624     std::stringstream optss;
 625     if (header->get_recursive()) {
 626       optss << "recursive";
 627       have_more = true;
 628     }
 629     if (header->get_repair()) {
 630       if (have_more) {
 631         optss << ",";
 632       }
 633       optss << "repair";
 634       have_more = true;
 635     }
 636     if (header->get_force()) {
 637       if (have_more) {
 638         optss << ",";
 639       }
 640       optss << "force";
 641     }
 642
 643     f->dump_string("options", optss.str());
 644     f->close_section(); // scrub id
 645   }
 646   f->close_section(); // scrubs
 647   f->close_section(); // result
 648 }
 649
 650 void ScrubStack::abort_pending_scrubs() {
 651   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 652   ceph_assert(clear_inode_stack);
 653
 654   for (auto inode = inode_stack.begin(); !inode.end(); ++inode) {
 655     CInode *in = *inode;
 656     if (in == in->scrub_info()->header->get_origin()) {
 657       scrub_origins.erase(in);
 658       clog_scrub_summary(in);
 659     }
 660
 661     MDSContext *ctx = nullptr;
 662     in->scrub_aborted(&ctx);
 663     if (ctx != nullptr) {
 664       ctx->complete(-ECANCELED);
 665     }
 666   }
 667
 668   stack_size = 0;
 669   inode_stack.clear();
 670   clear_inode_stack = false;
 671 }
 672
 673 void ScrubStack::scrub_abort(Context *on_finish) {
 674   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 675   ceph_assert(on_finish != nullptr);
 676
 677   dout(10) << __func__ << ": aborting with " << scrubs_in_progress
 678            << " scrubs in progress and " << stack_size << " in the"
 679            << " stack" << dendl;
 680
 681   clear_inode_stack = true;
 682   if (scrub_in_transition_state()) {
 683     control_ctxs.push_back(on_finish);
 684     return;
 685   }
 686
 687   abort_pending_scrubs();
 688   if (state != STATE_PAUSED) {
 689     set_state(STATE_IDLE);
 690   }
 691   on_finish->complete(0);
 692 }
 693
 694 void ScrubStack::scrub_pause(Context *on_finish) {
 695   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 696   ceph_assert(on_finish != nullptr);
 697
 698   dout(10) << __func__ << ": pausing with " << scrubs_in_progress
 699            << " scrubs in progress and " << stack_size << " in the"
 700            << " stack" << dendl;
 701
 702   // abort is in progress
 703   if (clear_inode_stack) {
 704     on_finish->complete(-EINVAL);
 705     return;
 706   }
 707
 708   bool done = scrub_in_transition_state();
 709   if (done) {
 710     set_state(STATE_PAUSING);
 711     control_ctxs.push_back(on_finish);
 712     return;
 713   }
 714
 715   set_state(STATE_PAUSED);
 716   on_finish->complete(0);
 717 }
 718
 719 bool ScrubStack::scrub_resume() {
 720   ceph_assert(ceph_mutex_is_locked_by_me(mdcache->mds->mds_lock));
 721   dout(20) << __func__ << ": state=" << state << dendl;
 722
 723   int r = 0;
 724
 725   if (clear_inode_stack) {
 726     r = -EINVAL;
 727   } else if (state == STATE_PAUSING) {
 728     set_state(STATE_RUNNING);
 729     complete_control_contexts(-ECANCELED);
 730   } else if (state == STATE_PAUSED) {
 731     set_state(STATE_RUNNING);
 732     kick_off_scrubs();
 733   }
 734
 735   return r;
 736 }
 737
 738 // send current scrub summary to cluster log
 739 void ScrubStack::clog_scrub_summary(CInode *in) {
 740   if (in) {
 741     std::string what;
 742     if (clear_inode_stack) {
 743       what = "aborted";
 744     } else if (scrub_origins.count(in)) {
 745       what = "queued";
 746     } else {
 747       what = "completed";
 748     }
 749     clog->info() << "scrub " << what << " for path: " << scrub_inode_path(in);
 750   }
 751
 752   clog->info() << "scrub summary: " << scrub_summary();
 753 }