ceph/src/common/ConfUtils.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2011 New Dream Network
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14 // #define BOOST_SPIRIT_DEBUG
  15
  16 #include <algorithm>
  17 #include <cctype>
  18 #include <experimental/iterator>
  19 #include <filesystem>
  20 #include <fstream>
  21 #include <iostream>
  22 #include <iterator>
  23 #include <map>
  24 #include <sstream>
  25
  26 #include <boost/algorithm/string.hpp>
  27 #include <boost/algorithm/string/trim_all.hpp>
  28 #include <boost/spirit/include/qi.hpp>
  29 #include <boost/phoenix.hpp>
  30 #include <boost/spirit/include/support_line_pos_iterator.hpp>
  31
  32 #include "include/buffer.h"
  33 #include "common/errno.h"
  34 #include "common/utf8.h"
  35 #include "common/ConfUtils.h"
  36
  37 namespace fs = std::filesystem;
  38
  39 using std::ostringstream;
  40 using std::string;
  41
  42 #define MAX_CONFIG_FILE_SZ 0x40000000
  43
  44 conf_line_t::conf_line_t(const std::string& key, const std::string& val)
  45   : key{ConfFile::normalize_key_name(key)},
  46     val{boost::algorithm::trim_copy_if(
  47           val,
  48           [](unsigned char c) {
  49             return std::isspace(c);
  50           })}
  51 {}
  52
  53 bool conf_line_t::operator<(const conf_line_t &rhs) const
  54 {
  55   // We only compare keys.
  56   // If you have more than one line with the same key in a given section, the
  57   // last one wins.
  58   return key < rhs.key;
  59 }
  60
  61 std::ostream &operator<<(std::ostream& oss, const conf_line_t &l)
  62 {
  63   oss << "conf_line_t(key = '" << l.key << "', val='" << l.val << "')";
  64   return oss;
  65 }
  66
  67 conf_section_t::conf_section_t(const std::string& heading,
  68                                const std::vector<conf_line_t>& lines)
  69   : heading{heading}
  70 {
  71   for (auto& line : lines) {
  72     auto [where, inserted] = insert(line);
  73     if (!inserted) {
  74       erase(where);
  75       insert(line);
  76     }
  77   }
  78 }
  79
  80 ///////////////////////// ConfFile //////////////////////////
  81
  82 ConfFile::ConfFile(const std::vector<conf_section_t>& sections)
  83 {
  84   for (auto& section : sections) {
  85     auto [old_sec, sec_inserted] = emplace(section.heading, section);
  86     if (!sec_inserted) {
  87       // merge lines in section into old_sec
  88       for (auto& line : section) {
  89         auto [old_line, line_inserted] = old_sec->second.emplace(line);
  90         // and replace the existing ones if any
  91         if (!line_inserted) {
  92           old_sec->second.erase(old_line);
  93           old_sec->second.insert(line);
  94         }
  95       }
  96     }
  97   }
  98 }
  99
 100 /* We load the whole file into memory and then parse it.  Although this is not
 101  * the optimal approach, it does mean that most of this code can be shared with
 102  * the bufferlist loading function. Since bufferlists are always in-memory, the
 103  * load_from_buffer interface works well for them.
 104  * In general, configuration files should be a few kilobytes at maximum, so
 105  * loading the whole configuration into memory shouldn't be a problem.
 106  */
 107 int ConfFile::parse_file(const std::string &fname,
 108                          std::ostream *warnings)
 109 {
 110   clear();
 111   try {
 112     if (auto file_size = fs::file_size(fname); file_size > MAX_CONFIG_FILE_SZ) {
 113       *warnings << __func__ << ": config file '" << fname
 114                 << "' is " << file_size << " bytes, "
 115                 << "but the maximum is " << MAX_CONFIG_FILE_SZ;
 116       return -EINVAL;
 117     }
 118   } catch (const fs::filesystem_error& e) {
 119     std::error_code ec;
 120     auto is_other = fs::is_other(fname, ec);
 121     if (!ec && is_other) {
 122       // /dev/null?
 123       return 0;
 124     } else {
 125       *warnings << __func__ << ": " << e.what();
 126       return -e.code().value();
 127     }
 128   }
 129   std::ifstream ifs{fname};
 130   std::string buffer{std::istreambuf_iterator<char>(ifs),
 131                                        std::istreambuf_iterator<char>()};
 132   if (parse_buffer(buffer, warnings)) {
 133     return 0;
 134   } else {
 135     return -EINVAL;
 136   }
 137 }
 138
 139 namespace {
 140
 141 namespace qi = boost::spirit::qi;
 142 namespace phoenix = boost::phoenix;
 143
 144 template<typename Iterator, typename Skipper>
 145 struct IniGrammer : qi::grammar<Iterator, ConfFile(), Skipper>
 146 {
 147   struct error_handler_t {
 148     std::ostream& os;
 149     template<typename Iter>
 150     auto operator()(Iter first, Iter last, Iter where,
 151                     const boost::spirit::info& what) const {
 152       auto line_start = boost::spirit::get_line_start(first, where);
 153       os << "parse error: expected '" << what
 154          << "' in line " << boost::spirit::get_line(where)
 155          << " at position " << boost::spirit::get_column(line_start, where) << "\n";
 156       return qi::fail;
 157     }
 158   };
 159   IniGrammer(Iterator begin, std::ostream& err)
 160     : IniGrammer::base_type{conf_file},
 161       report_error{error_handler_t{err}}
 162   {
 163     using qi::_1;
 164     using qi::_2;
 165     using qi::_val;
 166     using qi::char_;
 167     using qi::eoi;
 168     using qi::eol;
 169     using qi::blank;
 170     using qi::lexeme;
 171     using qi::lit;
 172     using qi::raw;
 173
 174     blanks = *blank;
 175     comment_start = lit('#') | lit(';');
 176     continue_marker = lit('\\') >> eol;
 177
 178     text_char %=
 179       (lit('\\') >> (char_ - eol)) |
 180       (char_ - (comment_start | eol));
 181
 182     key %= raw[+(text_char - char_("=[ ")) % +blank];
 183     quoted_value %=
 184       lexeme[lit('"') >> *(text_char - '"') > '"'] |
 185       lexeme[lit('\'') >> *(text_char - '\'') > '\''];
 186     unquoted_value %= *text_char;
 187     comment = *blank >> comment_start > *(char_ - eol);
 188     empty_line = -(blanks|comment) >> eol;
 189     value %= quoted_value | unquoted_value;
 190     key_val =
 191       (blanks >> key >> blanks >> '=' > blanks > value > +empty_line)
 192       [_val = phoenix::construct<conf_line_t>(_1, _2)];
 193
 194     heading %= lit('[') > +(text_char - ']') > ']' > +empty_line;
 195     section =
 196       (heading >> *(key_val - heading) >> *eol)
 197       [_val = phoenix::construct<conf_section_t>(_1, _2)];
 198     conf_file =
 199       (key_val [_val = phoenix::construct<ConfFile>(_1)]
 200        |
 201        (*eol >> (*section)[_val = phoenix::construct<ConfFile>(_1)])
 202       ) > eoi;
 203
 204     empty_line.name("empty_line");
 205     key.name("key");
 206     quoted_value.name("quoted value");
 207     unquoted_value.name("unquoted value");
 208     key_val.name("key=val");
 209     heading.name("section name");
 210     section.name("section");
 211
 212     qi::on_error<qi::fail>(
 213       conf_file,
 214       report_error(qi::_1, qi::_2, qi::_3, qi::_4));
 215
 216     BOOST_SPIRIT_DEBUG_NODE(heading);
 217     BOOST_SPIRIT_DEBUG_NODE(section);
 218     BOOST_SPIRIT_DEBUG_NODE(key);
 219     BOOST_SPIRIT_DEBUG_NODE(quoted_value);
 220     BOOST_SPIRIT_DEBUG_NODE(unquoted_value);
 221     BOOST_SPIRIT_DEBUG_NODE(key_val);
 222     BOOST_SPIRIT_DEBUG_NODE(conf_file);
 223   }
 224
 225   qi::rule<Iterator> blanks;
 226   qi::rule<Iterator> empty_line;
 227   qi::rule<Iterator> comment_start;
 228   qi::rule<Iterator> continue_marker;
 229   qi::rule<Iterator, char()> text_char;
 230   qi::rule<Iterator, std::string(), Skipper> key;
 231   qi::rule<Iterator, std::string(), Skipper> quoted_value;
 232   qi::rule<Iterator, std::string(), Skipper> unquoted_value;
 233   qi::rule<Iterator> comment;
 234   qi::rule<Iterator, std::string(), Skipper> value;
 235   qi::rule<Iterator, conf_line_t(), Skipper> key_val;
 236   qi::rule<Iterator, std::string(), Skipper> heading;
 237   qi::rule<Iterator, conf_section_t(), Skipper> section;
 238   qi::rule<Iterator, ConfFile(), Skipper> conf_file;
 239   boost::phoenix::function<error_handler_t> report_error;
 240 };
 241 }
 242
 243 bool ConfFile::parse_buffer(std::string_view buf, std::ostream* err)
 244 {
 245   assert(err);
 246 #ifdef _WIN32
 247   // We'll need to ensure that there's a new line at the end of the buffer,
 248   // otherwise the config parsing will fail.
 249   std::string _buf = std::string(buf) + "\n";
 250 #else
 251   std::string_view _buf = buf;
 252 #endif
 253   if (int err_pos = check_utf8(_buf.data(), _buf.size()); err_pos > 0) {
 254     *err << "parse error: invalid UTF-8 found at line "
 255          << std::count(_buf.begin(), std::next(_buf.begin(), err_pos), '\n') + 1;
 256     return false;
 257   }
 258   using iter_t = boost::spirit::line_pos_iterator<decltype(_buf.begin())>;
 259   iter_t first{_buf.begin()};
 260   using skipper_t = qi::rule<iter_t>;
 261   IniGrammer<iter_t, skipper_t> grammar{first, *err};
 262   skipper_t skipper = grammar.continue_marker | grammar.comment;
 263   return qi::phrase_parse(first, iter_t{_buf.end()},
 264                           grammar, skipper, *this);
 265 }
 266
 267 int ConfFile::parse_bufferlist(ceph::bufferlist *bl,
 268                                std::ostream *warnings)
 269 {
 270   clear();
 271   ostringstream oss;
 272   if (!warnings) {
 273     warnings = &oss;
 274   }
 275   return parse_buffer({bl->c_str(), bl->length()}, warnings) ? 0 : -EINVAL;
 276 }
 277
 278 int ConfFile::read(std::string_view section_name,
 279                    std::string_view key,
 280                    std::string &val) const
 281 {
 282   string k(normalize_key_name(key));
 283
 284   if (auto s = base_type::find(section_name); s != end()) {
 285     conf_line_t exemplar{k, {}};
 286     if (auto line = s->second.find(exemplar); line != s->second.end()) {
 287       val = line->val;
 288       return 0;
 289     }
 290   }
 291   return -ENOENT;
 292 }
 293
 294 /* Normalize a key name.
 295  *
 296  * Normalized key names have no leading or trailing whitespace, and all
 297  * whitespace is stored as underscores.  The main reason for selecting this
 298  * normal form is so that in common/config.cc, we can use a macro to stringify
 299  * the field names of md_config_t and get a key in normal form.
 300  */
 301 std::string ConfFile::normalize_key_name(std::string_view key)
 302 {
 303   std::string k{key};
 304   boost::algorithm::trim_fill_if(k, "_", isspace);
 305   return k;
 306 }
 307
 308 void ConfFile::check_old_style_section_names(const std::vector<std::string>& prefixes,
 309                                              std::ostream& os)
 310 {
 311   // Warn about section names that look like old-style section names
 312   std::vector<std::string> old_style_section_names;
 313   for (auto& [name, section] : *this) {
 314     for (auto& prefix : prefixes) {
 315       if (name.find(prefix) == 0 && name.size() > 3 && name[3] != '.') {
 316         old_style_section_names.push_back(name);
 317       }
 318     }
 319   }
 320   if (!old_style_section_names.empty()) {
 321     os << "ERROR! old-style section name(s) found: ";
 322     std::copy(std::begin(old_style_section_names),
 323               std::end(old_style_section_names),
 324               std::experimental::make_ostream_joiner(os, ", "));
 325     os << ". Please use the new style section names that include a period.";
 326   }
 327 }
 328
 329 std::ostream &operator<<(std::ostream &oss, const ConfFile &cf)
 330 {
 331   for (auto& [name, section] : cf) {
 332     oss << "[" << name << "]\n";
 333     for (auto& [key, val] : section) {
 334       if (!key.empty()) {
 335         oss << "\t" << key << " = \"" << val << "\"\n";
 336       }
 337     }
 338   }
 339   return oss;
 340 }