ceph/src/seastar/tests/unit/file_utils_test.cc

   1 /*
   2  * This file is open source software, licensed to you under the terms
   3  * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
   4  * distributed with this work for additional information regarding copyright
   5  * ownership.  You may not use this file except in compliance with the License.
   6  *
   7  * You may obtain a copy of the License at
   8  *
   9  *   http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing,
  12  * software distributed under the License is distributed on an
  13  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14  * KIND, either express or implied.  See the License for the
  15  * specific language governing permissions and limitations
  16  * under the License.
  17  */
  18
  19 /*
  20  * Copyright (C) 2020 ScyllaDB
  21  */
  22
  23 #include <stdlib.h>
  24
  25 #include <seastar/testing/test_case.hh>
  26 #include <seastar/testing/thread_test_case.hh>
  27 #include <seastar/testing/test_runner.hh>
  28
  29 #include <seastar/core/file.hh>
  30 #include <seastar/core/seastar.hh>
  31 #include <seastar/core/print.hh>
  32 #include <seastar/core/loop.hh>
  33 #include <seastar/util/tmp_file.hh>
  34 #include <seastar/util/file.hh>
  35
  36 using namespace seastar;
  37 namespace fs = std::filesystem;
  38
  39 class expected_exception : std::runtime_error {
  40 public:
  41     expected_exception() : runtime_error("expected") {}
  42 };
  43
  44 SEASTAR_TEST_CASE(test_make_tmp_file) {
  45     return make_tmp_file().then([] (tmp_file tf) {
  46         return async([tf = std::move(tf)] () mutable {
  47             const sstring tmp_path = tf.get_path().native();
  48             BOOST_REQUIRE(file_exists(tmp_path).get0());
  49             tf.close().get();
  50             tf.remove().get();
  51             BOOST_REQUIRE(!file_exists(tmp_path).get0());
  52         });
  53     });
  54 }
  55
  56 static temporary_buffer<char> get_init_buffer(file& f) {
  57     auto buf = temporary_buffer<char>::aligned(f.memory_dma_alignment(), f.memory_dma_alignment());
  58     memset(buf.get_write(), 0, buf.size());
  59     return buf;
  60 }
  61
  62 SEASTAR_THREAD_TEST_CASE(test_tmp_file) {
  63     size_t expected = ~0;
  64     size_t actual = 0;
  65
  66     tmp_file::do_with([&] (tmp_file& tf) mutable {
  67         auto& f = tf.get_file();
  68         auto buf = get_init_buffer(f);
  69         return do_with(std::move(buf), [&] (auto& buf) mutable {
  70             expected = buf.size();
  71             return f.dma_write(0, buf.get(), buf.size()).then([&] (size_t written) {
  72                 actual = written;
  73                 return make_ready_future<>();
  74             });
  75         });
  76     }).get();
  77     BOOST_REQUIRE_EQUAL(expected , actual);
  78 }
  79
  80 SEASTAR_THREAD_TEST_CASE(test_non_existing_TMPDIR) {
  81     auto old_tmpdir = getenv("TMPDIR");
  82     setenv("TMPDIR", "/tmp/non-existing-TMPDIR", true);
  83     BOOST_REQUIRE_EXCEPTION(tmp_file::do_with("/tmp/non-existing-TMPDIR", [] (tmp_file& tf) {}).get(),
  84             std::system_error, testing::exception_predicate::message_contains("No such file or directory"));
  85     if (old_tmpdir) {
  86         setenv("TMPDIR", old_tmpdir, true);
  87     } else {
  88         unsetenv("TMPDIR");
  89     }
  90 }
  91
  92 static future<> touch_file(const sstring& filename, open_flags oflags = open_flags::rw | open_flags::create) noexcept {
  93     return open_file_dma(filename, oflags).then([] (file f) {
  94         return f.close().finally([f] {});
  95     });
  96 }
  97
  98 SEASTAR_THREAD_TEST_CASE(test_recursive_remove_directory) {
  99     struct test_dir {
 100         test_dir *parent;
 101         sstring name;
 102         std::list<sstring> sub_files = {};
 103         std::list<test_dir> sub_dirs = {};
 104
 105         test_dir(test_dir* parent, sstring name)
 106             : parent(parent)
 107             , name(std::move(name))
 108         { }
 109
 110         fs::path path() const {
 111             if (!parent) {
 112                 return fs::path(name.c_str());
 113             }
 114             return parent->path() / name.c_str();
 115         }
 116
 117         void fill_random_file(std::uniform_int_distribution<unsigned>& dist, std::default_random_engine& eng) {
 118             sub_files.emplace_back(format("file-{}", dist(eng)));
 119         }
 120
 121         test_dir& fill_random_dir(std::uniform_int_distribution<unsigned>& dist, std::default_random_engine& eng) {
 122             sub_dirs.emplace_back(this, format("dir-{}", dist(eng)));
 123             return sub_dirs.back();
 124         }
 125
 126         void random_fill(int level, int levels, std::uniform_int_distribution<unsigned>& dist, std::default_random_engine& eng) {
 127             int num_files = dist(eng) % 10;
 128             int num_dirs = (level < levels - 1) ? (1 + dist(eng) % 3) : 0;
 129
 130             for (int i = 0; i < num_files; i++) {
 131                 fill_random_file(dist, eng);
 132             }
 133
 134             if (num_dirs) {
 135                 level++;
 136                 for (int i = 0; i < num_dirs; i++) {
 137                     fill_random_dir(dist, eng).random_fill(level, levels, dist, eng);
 138                 }
 139             }
 140         }
 141
 142         future<> populate() {
 143             return touch_directory(path().native()).then([this] {
 144                 return parallel_for_each(sub_files, [this] (auto& name) {
 145                     return touch_file((path() / name.c_str()).native());
 146                 }).then([this] {
 147                     return parallel_for_each(sub_dirs, [] (auto& sub_dir) {
 148                         return sub_dir.populate();
 149                     });
 150                 });
 151             });
 152         }
 153     };
 154
 155     auto& eng = testing::local_random_engine;
 156     auto dist = std::uniform_int_distribution<unsigned>();
 157     int levels = 1 + dist(eng) % 3;
 158     test_dir root = { nullptr, default_tmpdir().native() };
 159     test_dir base = { &root, format("base-{}", dist(eng)) };
 160     base.random_fill(0, levels, dist, eng);
 161     base.populate().get();
 162     recursive_remove_directory(base.path()).get();
 163     BOOST_REQUIRE(!file_exists(base.path().native()).get0());
 164 }
 165
 166 SEASTAR_TEST_CASE(test_make_tmp_dir) {
 167     return make_tmp_dir().then([] (tmp_dir td) {
 168         return async([td = std::move(td)] () mutable {
 169             const sstring tmp_path = td.get_path().native();
 170             BOOST_REQUIRE(file_exists(tmp_path).get0());
 171             td.remove().get();
 172             BOOST_REQUIRE(!file_exists(tmp_path).get0());
 173         });
 174     });
 175 }
 176
 177 SEASTAR_THREAD_TEST_CASE(test_tmp_dir) {
 178     size_t expected;
 179     size_t actual;
 180     tmp_dir::do_with([&] (tmp_dir& td) {
 181         return tmp_file::do_with(td.get_path(), [&] (tmp_file& tf) {
 182             auto& f = tf.get_file();
 183             auto buf = get_init_buffer(f);
 184             return do_with(std::move(buf), [&] (auto& buf) mutable {
 185                 expected = buf.size();
 186                 return f.dma_write(0, buf.get(), buf.size()).then([&] (size_t written) {
 187                     actual = written;
 188                     return make_ready_future<>();
 189                 });
 190             });
 191         });
 192     }).get();
 193     BOOST_REQUIRE_EQUAL(expected , actual);
 194 }
 195
 196 SEASTAR_THREAD_TEST_CASE(test_tmp_dir_with_path) {
 197     size_t expected;
 198     size_t actual;
 199     tmp_dir::do_with(".", [&] (tmp_dir& td) {
 200         return tmp_file::do_with(td.get_path(), [&] (tmp_file& tf) {
 201             auto& f = tf.get_file();
 202             auto buf = get_init_buffer(f);
 203             return do_with(std::move(buf), [&] (auto& buf) mutable {
 204                 expected = buf.size();
 205                 return tf.get_file().dma_write(0, buf.get(), buf.size()).then([&] (size_t written) {
 206                     actual = written;
 207                     return make_ready_future<>();
 208                 });
 209             });
 210         });
 211     }).get();
 212     BOOST_REQUIRE_EQUAL(expected , actual);
 213 }
 214
 215 SEASTAR_THREAD_TEST_CASE(test_tmp_dir_with_non_existing_path) {
 216     BOOST_REQUIRE_EXCEPTION(tmp_dir::do_with("/tmp/this_name_should_not_exist", [] (tmp_dir&) {}).get(),
 217             std::system_error, testing::exception_predicate::message_contains("No such file or directory"));
 218 }
 219
 220 SEASTAR_TEST_CASE(tmp_dir_with_thread_test) {
 221     return tmp_dir::do_with_thread([] (tmp_dir& td) {
 222         tmp_file tf = make_tmp_file(td.get_path()).get0();
 223         auto& f = tf.get_file();
 224         auto buf = get_init_buffer(f);
 225         auto expected = buf.size();
 226         auto actual = f.dma_write(0, buf.get(), buf.size()).get0();
 227         BOOST_REQUIRE_EQUAL(expected, actual);
 228         tf.close().get();
 229         tf.remove().get();
 230     });
 231 }
 232
 233 SEASTAR_TEST_CASE(tmp_dir_with_leftovers_test) {
 234     return tmp_dir::do_with_thread([] (tmp_dir& td) {
 235         fs::path path = td.get_path() / "testfile.tmp";
 236         touch_file(path.native()).get();
 237         BOOST_REQUIRE(file_exists(path.native()).get0());
 238     });
 239 }
 240
 241 SEASTAR_TEST_CASE(tmp_dir_do_with_fail_func_test) {
 242     return tmp_dir::do_with_thread([] (tmp_dir& outer) {
 243         BOOST_REQUIRE_THROW(tmp_dir::do_with([] (tmp_dir& inner) mutable {
 244             return make_exception_future<>(expected_exception());
 245         }).get(), expected_exception);
 246     });
 247 }
 248
 249 SEASTAR_TEST_CASE(tmp_dir_do_with_fail_remove_test) {
 250     return tmp_dir::do_with_thread([] (tmp_dir& outer) {
 251         auto saved_default_tmpdir = default_tmpdir();
 252         sstring outer_path = outer.get_path().native();
 253         sstring inner_path;
 254         sstring inner_path_renamed;
 255         set_default_tmpdir(outer_path.c_str());
 256         BOOST_REQUIRE_THROW(tmp_dir::do_with([&] (tmp_dir& inner) mutable {
 257             inner_path = inner.get_path().native();
 258             inner_path_renamed = inner_path + ".renamed";
 259             return rename_file(inner_path, inner_path_renamed);
 260         }).get(), std::system_error);
 261         BOOST_REQUIRE(!file_exists(inner_path).get0());
 262         BOOST_REQUIRE(file_exists(inner_path_renamed).get0());
 263         set_default_tmpdir(saved_default_tmpdir.c_str());
 264     });
 265 }
 266
 267 SEASTAR_TEST_CASE(tmp_dir_do_with_thread_fail_func_test) {
 268     return tmp_dir::do_with_thread([] (tmp_dir& outer) {
 269         BOOST_REQUIRE_THROW(tmp_dir::do_with_thread([] (tmp_dir& inner) mutable {
 270             throw expected_exception();
 271         }).get(), expected_exception);
 272     });
 273 }
 274
 275 SEASTAR_TEST_CASE(tmp_dir_do_with_thread_fail_remove_test) {
 276     return tmp_dir::do_with_thread([] (tmp_dir& outer) {
 277         auto saved_default_tmpdir = default_tmpdir();
 278         sstring outer_path = outer.get_path().native();
 279         sstring inner_path;
 280         sstring inner_path_renamed;
 281         set_default_tmpdir(outer_path.c_str());
 282         BOOST_REQUIRE_THROW(tmp_dir::do_with_thread([&] (tmp_dir& inner) mutable {
 283             inner_path = inner.get_path().native();
 284             inner_path_renamed = inner_path + ".renamed";
 285             return rename_file(inner_path, inner_path_renamed);
 286         }).get(), std::system_error);
 287         BOOST_REQUIRE(!file_exists(inner_path).get0());
 288         BOOST_REQUIRE(file_exists(inner_path_renamed).get0());
 289         set_default_tmpdir(saved_default_tmpdir.c_str());
 290     });
 291 }
 292
 293 SEASTAR_TEST_CASE(test_read_entire_file_contiguous) {
 294     return tmp_file::do_with([] (tmp_file& tf) {
 295         return async([&tf] {
 296             file& f = tf.get_file();
 297             auto& eng = testing::local_random_engine;
 298             auto dist = std::uniform_int_distribution<unsigned>();
 299             size_t size = f.memory_dma_alignment() * (1 + dist(eng) % 1000);
 300             auto wbuf = temporary_buffer<char>::aligned(f.memory_dma_alignment(), size);
 301             for (size_t i = 0; i < size; i++) {
 302                 static char chars[] = "abcdefghijklmnopqrstuvwxyz0123456789";
 303                 wbuf.get_write()[i] = chars[dist(eng) % sizeof(chars)];
 304             }
 305
 306             BOOST_REQUIRE_EQUAL(f.dma_write(0, wbuf.begin(), wbuf.size()).get0(), wbuf.size());
 307             f.flush().get();
 308
 309             sstring res = util::read_entire_file_contiguous(tf.get_path()).get0();
 310             BOOST_REQUIRE_EQUAL(res, std::string_view(wbuf.begin(), wbuf.size()));
 311         });
 312     });
 313 }