ceph/src/mds/SnapServer.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15 #include "SnapServer.h"
  16 #include "MDSRank.h"
  17 #include "osd/OSDMap.h"
  18 #include "osdc/Objecter.h"
  19 #include "mon/MonClient.h"
  20
  21 #include "include/types.h"
  22 #include "messages/MMDSTableRequest.h"
  23 #include "messages/MRemoveSnaps.h"
  24
  25 #include "msg/Messenger.h"
  26
  27 #include "common/config.h"
  28 #include "include/ceph_assert.h"
  29
  30 #define dout_context g_ceph_context
  31 #define dout_subsys ceph_subsys_mds
  32 #undef dout_prefix
  33 #define dout_prefix *_dout << "mds." << rank << ".snap "
  34
  35
  36 void SnapServer::reset_state()
  37 {
  38   last_snap = 1;  /* snapid 1 reserved for initial root snaprealm */
  39   snaps.clear();
  40   need_to_purge.clear();
  41   pending_update.clear();
  42   pending_destroy.clear();
  43   pending_noop.clear();
  44
  45   // find any removed snapshot in data pools
  46   if (mds) {  // only if I'm running in a live MDS
  47     snapid_t first_free = 0;
  48     mds->objecter->with_osdmap([&](const OSDMap& o) {
  49         for (const auto p : mds->mdsmap->get_data_pools()) {
  50           const pg_pool_t *pi = o.get_pg_pool(p);
  51           if (!pi) {
  52             // If pool isn't in OSDMap yet then can't have any snaps
  53             // needing removal, skip.
  54             continue;
  55           }
  56           if (pi->snap_seq > first_free) {
  57             first_free = pi->snap_seq;
  58           }
  59         }
  60       });
  61     if (first_free > last_snap)
  62       last_snap = first_free;
  63   }
  64   last_created = last_snap;
  65   last_destroyed = last_snap;
  66   snaprealm_v2_since = last_snap + 1;
  67
  68   MDSTableServer::reset_state();
  69 }
  70
  71
  72 // SERVER
  73
  74 void SnapServer::_prepare(const bufferlist& bl, uint64_t reqid, mds_rank_t bymds, bufferlist& out)
  75 {
  76   using ceph::decode;
  77   using ceph::encode;
  78   auto p = bl.cbegin();
  79   __u32 op;
  80   decode(op, p);
  81
  82   switch (op) {
  83   case TABLE_OP_CREATE:
  84     {
  85       SnapInfo info;
  86       decode(info.ino, p);
  87       if (!p.end()) {
  88         decode(info.name, p);
  89         decode(info.stamp, p);
  90         info.snapid = ++last_snap;
  91         pending_update[version] = info;
  92         dout(10) << "prepare v" << version << " create " << info << dendl;
  93       } else {
  94         pending_noop.insert(version);
  95         dout(10) << "prepare v" << version << " noop" << dendl;
  96       }
  97
  98       encode(last_snap, out);
  99     }
 100     break;
 101
 102   case TABLE_OP_DESTROY:
 103     {
 104       inodeno_t ino;
 105       snapid_t snapid;
 106       decode(ino, p);    // not used, currently.
 107       decode(snapid, p);
 108
 109       // bump last_snap... we use it as a version value on the snaprealm.
 110       ++last_snap;
 111
 112       pending_destroy[version] = pair<snapid_t,snapid_t>(snapid, last_snap);
 113       dout(10) << "prepare v" << version << " destroy " << snapid << " seq " << last_snap << dendl;
 114
 115       encode(last_snap, out);
 116     }
 117     break;
 118
 119   case TABLE_OP_UPDATE:
 120     {
 121       SnapInfo info;
 122       decode(info.ino, p);
 123       decode(info.snapid, p);
 124       decode(info.name, p);
 125       decode(info.stamp, p);
 126
 127       pending_update[version] = info;
 128       dout(10) << "prepare v" << version << " update " << info << dendl;
 129     }
 130     break;
 131
 132   default:
 133     ceph_abort();
 134   }
 135   //dump();
 136 }
 137
 138 void SnapServer::_get_reply_buffer(version_t tid, bufferlist *pbl) const
 139 {
 140   using ceph::encode;
 141   auto p = pending_update.find(tid);
 142   if (p != pending_update.end()) {
 143     if (pbl && !snaps.count(p->second.snapid)) // create
 144       encode(p->second.snapid, *pbl);
 145     return;
 146   }
 147   auto q = pending_destroy.find(tid);
 148   if (q != pending_destroy.end()) {
 149     if (pbl)
 150       encode(q->second.second, *pbl);
 151     return;
 152   }
 153   auto r = pending_noop.find(tid);
 154   if (r != pending_noop.end()) {
 155     if (pbl)
 156       encode(last_snap, *pbl);
 157     return;
 158   }
 159   assert (0 == "tid not found");
 160 }
 161
 162 void SnapServer::_commit(version_t tid, MMDSTableRequest::const_ref req)
 163 {
 164   if (pending_update.count(tid)) {
 165     SnapInfo &info = pending_update[tid];
 166     string opname;
 167     if (snaps.count(info.snapid)) {
 168       opname = "update";
 169       if (info.stamp == utime_t())
 170         info.stamp = snaps[info.snapid].stamp;
 171     } else {
 172       opname = "create";
 173       if (info.snapid > last_created)
 174         last_created = info.snapid;
 175     }
 176     dout(7) << "commit " << tid << " " << opname << " " << info << dendl;
 177     snaps[info.snapid] = info;
 178     pending_update.erase(tid);
 179   }
 180
 181   else if (pending_destroy.count(tid)) {
 182     snapid_t sn = pending_destroy[tid].first;
 183     snapid_t seq = pending_destroy[tid].second;
 184     dout(7) << "commit " << tid << " destroy " << sn << " seq " << seq << dendl;
 185     snaps.erase(sn);
 186     if (seq > last_destroyed)
 187       last_destroyed = seq;
 188
 189     for (const auto p : mds->mdsmap->get_data_pools()) {
 190       need_to_purge[p].insert(sn);
 191       need_to_purge[p].insert(seq);
 192     }
 193
 194     pending_destroy.erase(tid);
 195   }
 196   else if (pending_noop.count(tid)) {
 197     dout(7) << "commit " << tid << " noop" << dendl;
 198     pending_noop.erase(tid);
 199   }
 200   else
 201     ceph_abort();
 202
 203   //dump();
 204 }
 205
 206 void SnapServer::_rollback(version_t tid)
 207 {
 208   if (pending_update.count(tid)) {
 209     SnapInfo &info = pending_update[tid];
 210     string opname;
 211     if (snaps.count(info.snapid))
 212       opname = "update";
 213     else
 214       opname = "create";
 215     dout(7) << "rollback " << tid << " " << opname << " " << info << dendl;
 216     pending_update.erase(tid);
 217   }
 218
 219   else if (pending_destroy.count(tid)) {
 220     dout(7) << "rollback " << tid << " destroy " << pending_destroy[tid] << dendl;
 221     pending_destroy.erase(tid);
 222   }
 223
 224   else if (pending_noop.count(tid)) {
 225     dout(7) << "rollback " << tid << " noop" << dendl;
 226     pending_noop.erase(tid);
 227   }
 228
 229   else
 230     ceph_abort();
 231
 232   //dump();
 233 }
 234
 235 void SnapServer::_server_update(bufferlist& bl)
 236 {
 237   using ceph::decode;
 238   auto p = bl.cbegin();
 239   map<int, vector<snapid_t> > purge;
 240   decode(purge, p);
 241
 242   dout(7) << "_server_update purged " << purge << dendl;
 243   for (map<int, vector<snapid_t> >::iterator p = purge.begin();
 244        p != purge.end();
 245        ++p) {
 246     for (vector<snapid_t>::iterator q = p->second.begin();
 247          q != p->second.end();
 248          ++q)
 249       need_to_purge[p->first].erase(*q);
 250     if (need_to_purge[p->first].empty())
 251       need_to_purge.erase(p->first);
 252   }
 253 }
 254
 255 bool SnapServer::_notify_prep(version_t tid)
 256 {
 257   using ceph::encode;
 258   bufferlist bl;
 259   char type = 'F';
 260   encode(type, bl);
 261   encode(snaps, bl);
 262   encode(pending_update, bl);
 263   encode(pending_destroy, bl);
 264   encode(last_created, bl);
 265   encode(last_destroyed, bl);
 266   ceph_assert(version == tid);
 267
 268   for (auto &p : active_clients) {
 269     auto m = MMDSTableRequest::create(table, TABLESERVER_OP_NOTIFY_PREP, 0, version);
 270     m->bl = bl;
 271     mds->send_message_mds(m, p);
 272   }
 273   return true;
 274 }
 275
 276 void SnapServer::handle_query(const MMDSTableRequest::const_ref &req)
 277 {
 278   using ceph::encode;
 279   using ceph::decode;
 280   char op;
 281   auto p = req->bl.cbegin();
 282   decode(op, p);
 283
 284   auto reply = MMDSTableRequest::create(table, TABLESERVER_OP_QUERY_REPLY, req->reqid, version);
 285
 286   switch (op) {
 287     case 'F': // full
 288       version_t have_version;
 289       decode(have_version, p);
 290       ceph_assert(have_version <= version);
 291       if (have_version == version) {
 292         char type = 'U';
 293         encode(type, reply->bl);
 294       } else {
 295         char type = 'F';
 296         encode(type, reply->bl);
 297         encode(snaps, reply->bl);
 298         encode(pending_update, reply->bl);
 299         encode(pending_destroy, reply->bl);
 300         encode(last_created, reply->bl);
 301         encode(last_destroyed, reply->bl);
 302       }
 303       // FIXME: implement incremental change
 304       break;
 305     default:
 306       ceph_abort();
 307   };
 308
 309   mds->send_message(reply, req->get_connection());
 310 }
 311
 312 void SnapServer::check_osd_map(bool force)
 313 {
 314   if (!force && version == last_checked_osdmap) {
 315     dout(10) << "check_osd_map - version unchanged" << dendl;
 316     return;
 317   }
 318   dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl;
 319
 320   map<int, vector<snapid_t> > all_purge;
 321   map<int, vector<snapid_t> > all_purged;
 322
 323   mds->objecter->with_osdmap(
 324     [this, &all_purged, &all_purge](const OSDMap& osdmap) {
 325       for (const auto& p : need_to_purge) {
 326         int id = p.first;
 327         const pg_pool_t *pi = osdmap.get_pg_pool(id);
 328         if (pi == NULL) {
 329           // The pool is gone.  So are the snapshots.
 330           all_purged[id] = std::vector<snapid_t>(p.second.begin(),
 331                                                  p.second.end());
 332           continue;
 333         }
 334
 335         for (const auto& q : p.second) {
 336           if (pi->is_removed_snap(q)) {
 337             dout(10) << " osdmap marks " << q << " as removed" << dendl;
 338             all_purged[id].push_back(q);
 339           } else {
 340             all_purge[id].push_back(q);
 341           }
 342         }
 343       }
 344   });
 345
 346   if (!all_purged.empty()) {
 347     // prepare to remove from need_to_purge list
 348     bufferlist bl;
 349     using ceph::encode;
 350     encode(all_purged, bl);
 351     do_server_update(bl);
 352   }
 353
 354   if (!all_purge.empty()) {
 355     dout(10) << "requesting removal of " << all_purge << dendl;
 356     auto m = MRemoveSnaps::create(all_purge);
 357     mon_client->send_mon_message(m.detach());
 358   }
 359
 360   last_checked_osdmap = version;
 361 }
 362
 363
 364 void SnapServer::dump(Formatter *f) const
 365 {
 366   f->open_object_section("snapserver");
 367
 368   f->dump_int("last_snap", last_snap);
 369   f->dump_int("last_created", last_created);
 370   f->dump_int("last_destroyed", last_destroyed);
 371
 372   f->open_array_section("pending_noop");
 373   for(set<version_t>::const_iterator i = pending_noop.begin(); i != pending_noop.end(); ++i) {
 374     f->dump_unsigned("version", *i);
 375   }
 376   f->close_section();
 377
 378   f->open_array_section("snaps");
 379   for (map<snapid_t, SnapInfo>::const_iterator i = snaps.begin(); i != snaps.end(); ++i) {
 380     f->open_object_section("snap");
 381     i->second.dump(f);
 382     f->close_section();
 383   }
 384   f->close_section();
 385
 386   f->open_object_section("need_to_purge");
 387   for (map<int, set<snapid_t> >::const_iterator i = need_to_purge.begin(); i != need_to_purge.end(); ++i) {
 388     stringstream pool_id;
 389     pool_id << i->first;
 390     f->open_array_section(pool_id.str().c_str());
 391     for (set<snapid_t>::const_iterator s = i->second.begin(); s != i->second.end(); ++s) {
 392       f->dump_unsigned("snapid", s->val);
 393     }
 394     f->close_section();
 395   }
 396   f->close_section();
 397
 398   f->open_array_section("pending_update");
 399   for(map<version_t, SnapInfo>::const_iterator i = pending_update.begin(); i != pending_update.end(); ++i) {
 400     f->open_object_section("snap");
 401     f->dump_unsigned("version", i->first);
 402     f->open_object_section("snapinfo");
 403     i->second.dump(f);
 404     f->close_section();
 405     f->close_section();
 406   }
 407   f->close_section();
 408
 409   f->open_array_section("pending_destroy");
 410   for(map<version_t, pair<snapid_t, snapid_t> >::const_iterator i = pending_destroy.begin(); i != pending_destroy.end(); ++i) {
 411     f->open_object_section("snap");
 412     f->dump_unsigned("version", i->first);
 413     f->dump_unsigned("removed_snap", i->second.first);
 414     f->dump_unsigned("seq", i->second.second);
 415     f->close_section();
 416   }
 417   f->close_section();
 418
 419   f->close_section();
 420 }
 421
 422 void SnapServer::generate_test_instances(list<SnapServer*>& ls)
 423 {
 424   list<SnapInfo*> snapinfo_instances;
 425   SnapInfo::generate_test_instances(snapinfo_instances);
 426   SnapInfo populated_snapinfo = *(snapinfo_instances.back());
 427   for (list<SnapInfo*>::iterator i = snapinfo_instances.begin(); i != snapinfo_instances.end(); ++i) {
 428     delete *i;
 429   }
 430
 431   SnapServer *blank = new SnapServer();
 432   ls.push_back(blank);
 433   SnapServer *populated = new SnapServer();
 434   populated->last_snap = 123;
 435   populated->snaps[456] = populated_snapinfo;
 436   populated->need_to_purge[2].insert(012);
 437   populated->pending_update[234] = populated_snapinfo;
 438   populated->pending_destroy[345].first = 567;
 439   populated->pending_destroy[345].second = 768;
 440   populated->pending_noop.insert(890);
 441
 442   ls.push_back(populated);
 443 }
 444
 445 bool SnapServer::force_update(snapid_t last, snapid_t v2_since,
 446                               map<snapid_t, SnapInfo>& _snaps)
 447 {
 448   bool modified = false;
 449   if (last > last_snap) {
 450     derr << " updating last_snap " << last_snap << " -> " << last << dendl;
 451     last_snap = last;
 452     last_created = last;
 453     last_destroyed = last;
 454     modified = true;
 455   }
 456   if (v2_since > snaprealm_v2_since) {
 457     derr << " updating snaprealm_v2_since " << snaprealm_v2_since
 458          << " -> " << v2_since << dendl;
 459     snaprealm_v2_since = v2_since;
 460     modified = true;
 461   }
 462   if (snaps != _snaps) {
 463     derr << " updating snaps {" << snaps << "} -> {" << _snaps << "}" << dendl;
 464     snaps = _snaps;
 465     modified = true;
 466   }
 467
 468   if (modified) {
 469     need_to_purge.clear();
 470     pending_update.clear();
 471     pending_destroy.clear();
 472     pending_noop.clear();
 473     MDSTableServer::reset_state();
 474   }
 475   return modified;
 476 }