ceph/src/tools/neorados.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2019 Red Hat <contact@redhat.com>
   7  * Author: Adam C. Emerson <aemerson@redhat.com>
   8  *
   9  * This is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License version 2.1, as published by the Free Software
  12  * Foundation.  See file COPYING.
  13  *
  14  */
  15
  16 #define BOOST_COROUTINES_NO_DEPRECATION_WARNING
  17
  18 #include <algorithm>
  19 #include <cassert>
  20 #include <iostream>
  21 #include <string>
  22 #include <string_view>
  23 #include <tuple>
  24 #include <vector>
  25
  26 #include <boost/asio.hpp>
  27 #include <boost/io/ios_state.hpp>
  28 #include <boost/program_options.hpp>
  29 #include <boost/system/system_error.hpp>
  30
  31 #include <fmt/format.h>
  32 #include <fmt/ostream.h>
  33
  34 #include <spawn/spawn.hpp>
  35
  36 #include "include/buffer.h" // :(
  37
  38 #include "include/neorados/RADOS.hpp"
  39
  40 using namespace std::literals;
  41
  42 namespace ba = boost::asio;
  43 namespace bs = boost::system;
  44 namespace R = neorados;
  45 namespace s = spawn;
  46
  47 std::string verstr(const std::tuple<uint32_t, uint32_t, uint32_t>& v)
  48 {
  49   const auto [maj, min, p] = v;
  50   return fmt::format("v{}.{}.{}", maj, min, p);
  51 }
  52
  53 template<typename V>
  54 void printseq(const V& v, std::ostream& m)
  55 {
  56   std::for_each(v.cbegin(), v.cend(),
  57                 [&m](const auto& e) {
  58                   fmt::print(m, "{}\n", e);
  59                 });
  60 }
  61
  62 template<typename V, typename F>
  63 void printseq(const V& v, std::ostream& m, F&& f)
  64 {
  65   std::for_each(v.cbegin(), v.cend(),
  66                 [&m, &f](const auto& e) {
  67                   fmt::print(m, "{}\n", f(e));
  68                 });
  69 }
  70
  71 std::int64_t lookup_pool(R::RADOS& r, const std::string& pname,
  72                          s::yield_context y)
  73 {
  74   bs::error_code ec;
  75   auto p = r.lookup_pool(pname, y[ec]);
  76   if (ec)
  77     throw bs::system_error(
  78       ec, fmt::format("when looking up '{}'", pname));
  79   return p;
  80 }
  81
  82
  83 void lspools(R::RADOS& r, const std::vector<std::string>&,
  84              s::yield_context y)
  85 {
  86   const auto l = r.list_pools(y);
  87   printseq(l, std::cout, [](const auto& p) -> const std::string& {
  88                            return p.second;
  89                          });
  90 }
  91
  92
  93 void ls(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
  94 {
  95   const auto& pname = p[0];
  96   const auto pool = lookup_pool(r, pname, y);
  97
  98   std::vector<R::Entry> ls;
  99   R::Cursor next = R::Cursor::begin();
 100   bs::error_code ec;
 101   do {
 102     std::tie(ls, next) = r.enumerate_objects(pool, next, R::Cursor::end(),
 103                                              1000, {}, y[ec], R::all_nspaces);
 104     if (ec)
 105       throw bs::system_error(ec, fmt::format("when listing {}", pname));
 106     printseq(ls, std::cout);
 107     ls.clear();
 108   } while (next != R::Cursor::end());
 109 }
 110
 111 void mkpool(R::RADOS& r, const std::vector<std::string>& p,
 112             s::yield_context y)
 113 {
 114   const auto& pname = p[0];
 115   bs::error_code ec;
 116   r.create_pool(pname, std::nullopt, y[ec]);
 117   if (ec)
 118     throw bs::system_error(ec, fmt::format("when creating pool '{}'", pname));
 119 }
 120
 121 void rmpool(R::RADOS& r, const std::vector<std::string>& p,
 122             s::yield_context y)
 123 {
 124   const auto& pname = p[0];
 125   bs::error_code ec;
 126   r.delete_pool(pname, y[ec]);
 127   if (ec)
 128     throw bs::system_error(ec, fmt::format("when removing pool '{}'", pname));
 129 }
 130
 131 void create(R::RADOS& r, const std::vector<std::string>& p,
 132             s::yield_context y)
 133 {
 134   const auto& pname = p[0];
 135   const R::Object obj = p[1];
 136   const auto pool = lookup_pool(r, pname, y);
 137
 138   bs::error_code ec;
 139   R::WriteOp op;
 140   op.create(true);
 141   r.execute(obj, pool, std::move(op), y[ec]);
 142   if (ec)
 143     throw bs::system_error(ec,
 144                            fmt::format(
 145                              "when creating object '{}' in pool '{}'",
 146                              obj, pname));
 147 }
 148
 149 inline constexpr std::size_t io_size = 4 << 20;
 150
 151 void write(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 152 {
 153   const auto& pname = p[0];
 154   const R::Object obj(p[1]);
 155   const auto pool = lookup_pool(r, pname, y);
 156
 157   bs::error_code ec;
 158   std::unique_ptr<char[]> buf = std::make_unique<char[]>(io_size);
 159   std::size_t off = 0;
 160   boost::io::ios_exception_saver ies(std::cin);
 161
 162   std::cin.exceptions(std::istream::badbit);
 163   std::cin.clear();
 164
 165   while (!std::cin.eof()) {
 166     auto curoff = off;
 167     std::cin.read(buf.get(), io_size);
 168     auto len = std::cin.gcount();
 169     off += len;
 170     if (len == 0)
 171       break; // Nothin' to do.
 172
 173     ceph::buffer::list bl;
 174     bl.append(buffer::create_static(len, buf.get()));
 175     R::WriteOp op;
 176     op.write(curoff, std::move(bl));
 177     r.execute(obj, pool, std::move(op), y[ec]);
 178
 179     if (ec)
 180       throw bs::system_error(ec, fmt::format(
 181                                "when writing object '{}' in pool '{}'",
 182                                obj, pname));
 183   }
 184 }
 185
 186 void read(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 187 {
 188   const auto& pname = p[0];
 189   const R::Object obj(p[1]);
 190   const auto pool = lookup_pool(r, pname, y);
 191
 192   bs::error_code ec;
 193   std::uint64_t len;
 194   {
 195     R::ReadOp op;
 196     op.stat(&len, nullptr);
 197     r.execute(obj, pool, std::move(op),
 198               nullptr, y[ec]);
 199     if (ec)
 200       throw bs::system_error(
 201         ec,
 202         fmt::format("when getting length of object '{}' in pool '{}'",
 203                     obj, pname));
 204   }
 205
 206   std::size_t off = 0;
 207   ceph::buffer::list bl;
 208   while (auto toread = std::max(len - off, io_size)) {
 209     R::ReadOp op;
 210     op.read(off, toread, &bl);
 211     r.execute(obj, pool, std::move(op), nullptr, y[ec]);
 212     if (ec)
 213       throw bs::system_error(
 214         ec,
 215         fmt::format("when reading from object '{}' in pool '{}'",
 216                     obj, pool));
 217
 218     off += bl.length();
 219     bl.write_stream(std::cout);
 220     bl.clear();
 221   }
 222 }
 223
 224 void rm(R::RADOS& r, const std::vector<std::string>& p, s::yield_context y)
 225 {
 226   const auto& pname = p[0];
 227   const R::Object obj = p[1];
 228   const auto pool = lookup_pool(r, pname, y);
 229
 230   bs::error_code ec;
 231   R::WriteOp op;
 232   op.remove();
 233   r.execute(obj, pool, std::move(op), y[ec]);
 234   if (ec)
 235     throw bs::system_error(ec, fmt::format(
 236                              "when removing object '{}' in pool '{}'",
 237                              obj, pname));
 238 }
 239
 240 static constexpr auto version = std::make_tuple(0ul, 0ul, 1ul);
 241
 242 using cmdfunc = void (*)(R::RADOS& r, const std::vector<std::string>& p,
 243                          s::yield_context);
 244
 245 struct cmdesc {
 246   std::string_view name;
 247   std::size_t arity;
 248   cmdfunc f;
 249   std::string_view usage;
 250   std::string_view desc;
 251 };
 252
 253 const std::array commands = {
 254   // Pools operations ;)
 255
 256   cmdesc{ "lspools"sv,
 257           0, &lspools,
 258           ""sv,
 259           "List all pools"sv },
 260
 261   // Pool operations
 262
 263   cmdesc{ "ls"sv,
 264           1, &ls,
 265           "POOL"sv,
 266           "list all objects in POOL"sv },
 267   cmdesc{ "mkpool"sv,
 268           1, &mkpool,
 269           "POOL"sv,
 270           "create POOL"sv },
 271   cmdesc{ "rmpool"sv,
 272           1, &rmpool,
 273           "POOL"sv,
 274           "remove POOL"sv },
 275
 276   // Object operations
 277
 278   cmdesc{ "create"sv,
 279           2, &create,
 280           "POOL OBJECT"sv,
 281           "exclusively create OBJECT in POOL"sv },
 282   cmdesc{ "write"sv,
 283           2, &write,
 284           "POOL OBJECT"sv,
 285           "write to OBJECT in POOL from standard input"sv },
 286   cmdesc{ "read"sv,
 287           2, &read,
 288           "POOL OBJECT"sv,
 289           "read contents of OBJECT in POOL to standard out"sv },
 290   cmdesc{ "rm"sv,
 291           2, &rm,
 292           "POOL OBJECT"sv,
 293           "remove OBJECT in POOL"sv }
 294 };
 295
 296 #if FMT_VERSION >= 90000
 297 template <> struct fmt::formatter<boost::program_options::options_description> : ostream_formatter {};
 298 #endif // FMT_VERSION
 299
 300 int main(int argc, char* argv[])
 301 {
 302   const std::string_view prog(argv[0]);
 303   std::string command;
 304   namespace po = boost::program_options;
 305   try {
 306     std::vector<std::string> parameters;
 307
 308     po::options_description desc(fmt::format("{} options", prog));
 309     desc.add_options()
 310       ("help", "show help")
 311       ("version", "show version")
 312       ("command", po::value<std::string>(&command), "the operation to perform")
 313       ("parameters", po::value<std::vector<std::string>>(&parameters),
 314        "parameters to the command");
 315
 316     po::positional_options_description p;
 317     p.add("command", 1);
 318     p.add("parameters", -1);
 319
 320     po::variables_map vm;
 321
 322     po::store(po::command_line_parser(argc, argv).
 323               options(desc).positional(p).run(), vm);
 324
 325     po::notify(vm);
 326
 327     if (vm.count("help")) {
 328       fmt::print("{}", desc);
 329       fmt::print("Commands:\n");
 330       for (const auto& cmd : commands) {
 331         fmt::print("    {} {}{}{}\n",
 332                    cmd.name, cmd.usage,
 333                    cmd.name.length() + cmd.usage.length() < 13 ?
 334                    "\t\t"sv : "\t"sv,
 335                    cmd.desc);
 336       }
 337       return 0;
 338     }
 339
 340     if (vm.count("version")) {
 341       fmt::print(
 342         "{}: RADOS command exerciser, {},\n"
 343         "RADOS library version {}\n"
 344         "Copyright (C) 2019 Red Hat <contact@redhat.com>\n"
 345         "This is free software; you can redistribute it and/or\n"
 346         "modify it under the terms of the GNU Lesser General Public\n"
 347         "License version 2.1, as published by the Free Software\n"
 348         "Foundation.  See file COPYING.\n", prog,
 349         verstr(version), verstr(R::RADOS::version()));
 350       return 0;
 351     }
 352
 353     if (vm.find("command") == vm.end()) {
 354       fmt::print(std::cerr, "{}: a command is required\n", prog);
 355       return 1;
 356     }
 357
 358     ba::io_context c;
 359
 360     if (auto ci = std::find_if(commands.begin(), commands.end(),
 361                                [&command](const cmdesc& c) {
 362                                  return c.name == command;
 363                                }); ci != commands.end()) {
 364       if (parameters.size() < ci->arity) {
 365         fmt::print(std::cerr, "{}: {}: too few arguments\n\t{} {}\n",
 366                    prog, command, ci->name, ci->usage);
 367         return 1;
 368       }
 369       if (parameters.size() > ci->arity) {
 370         fmt::print(std::cerr, "{}: {}: too many arguments\n\t{} {}\n",
 371                    prog, command, ci->name, ci->usage);
 372         return 1;
 373       }
 374       s::spawn(c, [&](s::yield_context y) {
 375                     auto r = R::RADOS::Builder{}.build(c, y);
 376                     ci->f(r, parameters, y);
 377                   });
 378     } else {
 379       fmt::print(std::cerr, "{}: {}: unknown command\n", prog, command);
 380       return 1;
 381     }
 382     c.run();
 383   } catch (const std::exception& e) {
 384     fmt::print(std::cerr, "{}: {}: {}\n", prog, command, e.what());
 385     return 1;
 386   }
 387
 388   return 0;
 389 }