ceph/src/crimson/osd/osd.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #pragma once
   5
   6 #include <seastar/core/future.hh>
   7 #include <seastar/core/shared_future.hh>
   8 #include <seastar/core/gate.hh>
   9 #include <seastar/core/shared_ptr.hh>
  10 #include <seastar/core/shared_future.hh>
  11 #include <seastar/core/timer.hh>
  12
  13 #include "crimson/common/type_helpers.h"
  14 #include "crimson/common/auth_handler.h"
  15 #include "crimson/common/gated.h"
  16 #include "crimson/admin/admin_socket.h"
  17 #include "crimson/common/simple_lru.h"
  18 #include "crimson/common/shared_lru.h"
  19 #include "crimson/mgr/client.h"
  20 #include "crimson/net/Dispatcher.h"
  21 #include "crimson/osd/osdmap_service.h"
  22 #include "crimson/osd/state.h"
  23 #include "crimson/osd/shard_services.h"
  24 #include "crimson/osd/osdmap_gate.h"
  25 #include "crimson/osd/pg_map.h"
  26 #include "crimson/osd/osd_operations/peering_event.h"
  27
  28 #include "messages/MOSDOp.h"
  29 #include "osd/PeeringState.h"
  30 #include "osd/osd_types.h"
  31 #include "osd/osd_perf_counters.h"
  32 #include "osd/PGPeeringEvent.h"
  33
  34 class MCommand;
  35 class MOSDMap;
  36 class MOSDRepOpReply;
  37 class MOSDRepOp;
  38 class MOSDScrub2;
  39 class OSDMap;
  40 class OSDMeta;
  41 class Heartbeat;
  42
  43 namespace ceph::os {
  44   class Transaction;
  45 }
  46
  47 namespace crimson::mon {
  48   class Client;
  49 }
  50
  51 namespace crimson::net {
  52   class Messenger;
  53 }
  54
  55 namespace crimson::os {
  56   class FuturizedStore;
  57 }
  58
  59 namespace crimson::osd {
  60 class PG;
  61
  62 class OSD final : public crimson::net::Dispatcher,
  63                   private OSDMapService,
  64                   private crimson::common::AuthHandler,
  65                   private crimson::mgr::WithStats {
  66   const int whoami;
  67   const uint32_t nonce;
  68   seastar::timer<seastar::lowres_clock> beacon_timer;
  69   // talk with osd
  70   crimson::net::MessengerRef cluster_msgr;
  71   // talk with client/mon/mgr
  72   crimson::net::MessengerRef public_msgr;
  73   std::unique_ptr<crimson::mon::Client> monc;
  74   std::unique_ptr<crimson::mgr::Client> mgrc;
  75
  76   SharedLRU<epoch_t, OSDMap> osdmaps;
  77   SimpleLRU<epoch_t, bufferlist, false> map_bl_cache;
  78   cached_map_t osdmap;
  79   // TODO: use a wrapper for ObjectStore
  80   std::unique_ptr<crimson::os::FuturizedStore> store;
  81   std::unique_ptr<OSDMeta> meta_coll;
  82
  83   OSDState state;
  84
  85   /// _first_ epoch we were marked up (after this process started)
  86   epoch_t boot_epoch = 0;
  87   /// _most_recent_ epoch we were marked up
  88   epoch_t up_epoch = 0;
  89   //< epoch we last did a bind to new ip:ports
  90   epoch_t bind_epoch = 0;
  91   //< since when there is no more pending pg creates from mon
  92   epoch_t last_pg_create_epoch = 0;
  93
  94   ceph::mono_time startup_time;
  95
  96   OSDSuperblock superblock;
  97
  98   // Dispatcher methods
  99   std::optional<seastar::future<>> ms_dispatch(crimson::net::ConnectionRef, MessageRef) final;
 100   void ms_handle_reset(crimson::net::ConnectionRef conn, bool is_replace) final;
 101   void ms_handle_remote_reset(crimson::net::ConnectionRef conn) final;
 102
 103   // mgr::WithStats methods
 104   // pg statistics including osd ones
 105   osd_stat_t osd_stat;
 106   uint32_t osd_stat_seq = 0;
 107   void update_stats();
 108   MessageRef get_stats() const final;
 109
 110   // AuthHandler methods
 111   void handle_authentication(const EntityName& name,
 112                              const AuthCapsInfo& caps) final;
 113
 114   crimson::osd::ShardServices shard_services;
 115
 116   std::unique_ptr<Heartbeat> heartbeat;
 117   seastar::timer<seastar::lowres_clock> tick_timer;
 118
 119   // admin-socket
 120   seastar::lw_shared_ptr<crimson::admin::AdminSocket> asok;
 121
 122 public:
 123   OSD(int id, uint32_t nonce,
 124       crimson::net::MessengerRef cluster_msgr,
 125       crimson::net::MessengerRef client_msgr,
 126       crimson::net::MessengerRef hb_front_msgr,
 127       crimson::net::MessengerRef hb_back_msgr);
 128   ~OSD() final;
 129
 130   seastar::future<> mkfs(uuid_d osd_uuid, uuid_d cluster_fsid);
 131
 132   seastar::future<> start();
 133   seastar::future<> stop();
 134
 135   void dump_status(Formatter*) const;
 136   void dump_pg_state_history(Formatter*) const;
 137   void print(std::ostream&) const;
 138
 139   seastar::future<> send_incremental_map(crimson::net::ConnectionRef conn,
 140                                          epoch_t first);
 141
 142   /// @return the seq id of the pg stats being sent
 143   uint64_t send_pg_stats();
 144
 145 private:
 146   seastar::future<> start_boot();
 147   seastar::future<> _preboot(version_t oldest_osdmap, version_t newest_osdmap);
 148   seastar::future<> _send_boot();
 149   seastar::future<> _add_me_to_crush();
 150
 151   seastar::future<Ref<PG>> make_pg(cached_map_t create_map,
 152                                    spg_t pgid,
 153                                    bool do_create);
 154   seastar::future<Ref<PG>> load_pg(spg_t pgid);
 155   seastar::future<> load_pgs();
 156
 157   // OSDMapService methods
 158   epoch_t get_up_epoch() const final {
 159     return up_epoch;
 160   }
 161   seastar::future<cached_map_t> get_map(epoch_t e) final;
 162   cached_map_t get_map() const final;
 163   seastar::future<std::unique_ptr<OSDMap>> load_map(epoch_t e);
 164   seastar::future<bufferlist> load_map_bl(epoch_t e);
 165   seastar::future<std::map<epoch_t, bufferlist>>
 166   load_map_bls(epoch_t first, epoch_t last);
 167   void store_map_bl(ceph::os::Transaction& t,
 168                     epoch_t e, bufferlist&& bl);
 169   seastar::future<> store_maps(ceph::os::Transaction& t,
 170                                epoch_t start, Ref<MOSDMap> m);
 171   seastar::future<> osdmap_subscribe(version_t epoch, bool force_request);
 172
 173   void write_superblock(ceph::os::Transaction& t);
 174   seastar::future<> read_superblock();
 175
 176   bool require_mon_peer(crimson::net::Connection *conn, Ref<Message> m);
 177
 178   seastar::future<Ref<PG>> handle_pg_create_info(
 179     std::unique_ptr<PGCreateInfo> info);
 180
 181   seastar::future<> handle_osd_map(crimson::net::ConnectionRef conn,
 182                                    Ref<MOSDMap> m);
 183   seastar::future<> handle_osd_op(crimson::net::ConnectionRef conn,
 184                                   Ref<MOSDOp> m);
 185   seastar::future<> handle_rep_op(crimson::net::ConnectionRef conn,
 186                                   Ref<MOSDRepOp> m);
 187   seastar::future<> handle_rep_op_reply(crimson::net::ConnectionRef conn,
 188                                         Ref<MOSDRepOpReply> m);
 189   seastar::future<> handle_peering_op(crimson::net::ConnectionRef conn,
 190                                       Ref<MOSDPeeringOp> m);
 191   seastar::future<> handle_recovery_subreq(crimson::net::ConnectionRef conn,
 192                                            Ref<MOSDFastDispatchOp> m);
 193   seastar::future<> handle_scrub(crimson::net::ConnectionRef conn,
 194                                  Ref<MOSDScrub2> m);
 195   seastar::future<> handle_mark_me_down(crimson::net::ConnectionRef conn,
 196                                         Ref<MOSDMarkMeDown> m);
 197
 198   seastar::future<> committed_osd_maps(version_t first,
 199                                        version_t last,
 200                                        Ref<MOSDMap> m);
 201
 202   void check_osdmap_features();
 203
 204   seastar::future<> handle_command(crimson::net::ConnectionRef conn,
 205                                    Ref<MCommand> m);
 206   seastar::future<> start_asok_admin();
 207
 208 public:
 209   OSDMapGate osdmap_gate;
 210
 211   ShardServices &get_shard_services() {
 212     return shard_services;
 213   }
 214
 215   seastar::future<> consume_map(epoch_t epoch);
 216
 217 private:
 218   PGMap pg_map;
 219   crimson::common::Gated gate;
 220
 221   seastar::promise<> stop_acked;
 222   void got_stop_ack() {
 223     stop_acked.set_value();
 224   }
 225   seastar::future<> prepare_to_stop();
 226 public:
 227   blocking_future<Ref<PG>> get_or_create_pg(
 228     spg_t pgid,
 229     epoch_t epoch,
 230     std::unique_ptr<PGCreateInfo> info);
 231   blocking_future<Ref<PG>> wait_for_pg(
 232     spg_t pgid);
 233   Ref<PG> get_pg(spg_t pgid);
 234
 235   bool should_restart() const;
 236   seastar::future<> restart();
 237   seastar::future<> shutdown();
 238
 239   seastar::future<> send_beacon();
 240   void update_heartbeat_peers();
 241
 242   friend class PGAdvanceMap;
 243 };
 244
 245 inline std::ostream& operator<<(std::ostream& out, const OSD& osd) {
 246   osd.print(out);
 247   return out;
 248 }
 249
 250 }