diff --git a/benchmarks/README.md b/benchmarks/README.md index bd4093037..7a5bab08b 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -38,6 +38,16 @@ to only run some of the benchmarks: ./build/benchmarks/all/reflect-cpp-all-format-benchmarks --benchmark_filter=canada_read ``` +The `BM_load_file` cases measure the shared binary and text file loaders with +1 KiB, 1 MiB, and 32 MiB inputs. Fixture creation and content checks happen before +timing; timed work includes opening, reading, closing, and destroying the result. +The files are freshly written and read once for validation, so these measure +loading with a warm filesystem cache, not cold-storage throughput. + +``` +./build/benchmarks/all/reflect-cpp-all-format-benchmarks --benchmark_filter=BM_load_file +``` + ## Results summary ### How reflect-cpp compares to other JSON libraries diff --git a/benchmarks/all/file_io.cpp b/benchmarks/all/file_io.cpp new file mode 100644 index 000000000..56ddc32ff --- /dev/null +++ b/benchmarks/all/file_io.cpp @@ -0,0 +1,85 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace file_io { + +class InputFile { + public: + explicit InputFile(std::size_t size) { + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + for (unsigned i = 0; i < 100; ++i) { + dir_ = std::filesystem::temp_directory_path() / + ("reflectcpp-io-bench-" + std::to_string(stamp) + "-" + + std::to_string(i)); + if (std::filesystem::create_directory(dir_)) { + name_ = (dir_ / "input").string(); + std::ofstream out; + out.exceptions(std::ios::badbit | std::ios::failbit); + out.open(name_, std::ios::binary); + const std::string data(size, 'x'); + out.write(data.data(), static_cast(data.size())); + out.close(); + return; + } + } + throw std::runtime_error("Cannot create benchmark directory"); + } + ~InputFile() { + std::error_code ec; + std::filesystem::remove_all(dir_, ec); + } + const std::string& name() const { return name_; } + + private: + std::filesystem::path dir_; + std::string name_; +}; + +template +void BM_load_file(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + const InputFile file(size); + const auto load = [&] { + if constexpr (Binary) + return rfl::io::load_bytes(file.name()); + else + return rfl::io::load_string(file.name()); + }; + { + const auto result = load(); + if (!result || result->size() != size) { + state.SkipWithError("File loading failed"); + return; + } + for (const char c : *result) { + if (c != 'x') { + state.SkipWithError("File contents changed"); + return; + } + } + } + for (auto _ : state) { + auto result = load(); + benchmark::DoNotOptimize(result); + } + state.SetBytesProcessed(state.iterations() * state.range(0)); +} + +BENCHMARK_TEMPLATE(BM_load_file, true) + ->Arg(1024) + ->Arg(1024 * 1024) + ->Arg(32 * 1024 * 1024); +BENCHMARK_TEMPLATE(BM_load_file, false) + ->Arg(1024) + ->Arg(1024 * 1024) + ->Arg(32 * 1024 * 1024); + +} // namespace file_io diff --git a/include/rfl/io/load_bytes.hpp b/include/rfl/io/load_bytes.hpp index 84bd21fc4..d61673e11 100644 --- a/include/rfl/io/load_bytes.hpp +++ b/include/rfl/io/load_bytes.hpp @@ -1,26 +1,17 @@ #ifndef RFL_IO_LOAD_BYTES_HPP_ #define RFL_IO_LOAD_BYTES_HPP_ -#include -#include #include #include #include "../Result.hpp" +#include "load_file.hpp" namespace rfl { namespace io { inline Result> load_bytes(const std::string& _fname) { - std::ifstream input(_fname, std::ios::binary); - if (input.is_open()) { - std::istreambuf_iterator begin(input), end; - const auto bytes = std::vector(begin, end); - input.close(); - return bytes; - } else { - return error("File '" + _fname + "' not found!"); - } + return internal::load_file>(_fname, std::ios::binary); } } // namespace io diff --git a/include/rfl/io/load_file.hpp b/include/rfl/io/load_file.hpp new file mode 100644 index 000000000..c39214ded --- /dev/null +++ b/include/rfl/io/load_file.hpp @@ -0,0 +1,66 @@ +#ifndef RFL_IO_LOAD_FILE_HPP_ +#define RFL_IO_LOAD_FILE_HPP_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../Result.hpp" + +namespace rfl::io::internal { + +template +Result load_file(const std::string& _fname, + const std::ios::openmode _mode) { + try { + std::ifstream input(_fname, _mode); + if (!input.is_open()) { + return error("Unable to open file '" + _fname + "' for reading."); + } + Container bytes; + std::error_code ec; + const auto size = std::filesystem::file_size(_fname, ec); + if (!ec) { + if (size > bytes.max_size() || + size > static_cast( + (std::numeric_limits::max)())) { + return error("File '" + _fname + "' is too large to read."); + } + bytes.resize(static_cast(size)); + if (!bytes.empty()) { + input.read(bytes.data(), static_cast(bytes.size())); + bytes.resize(static_cast(input.gcount())); + if (input.bad() || (input.fail() && !input.eof())) { + return error("Unable to read file '" + _fname + "'."); + } + if (input.eof()) { + return bytes; + } + } + } else if (std::filesystem::is_directory(_fname, ec)) { + return error("Cannot read directory '" + _fname + "' as a file."); + } + // The size is only a hint: text conversion, growing files, and special + // files can produce a different number of bytes. Never require seeking. + std::array buffer; + do { + input.read(buffer.data(), static_cast(buffer.size())); + if (input.bad() || (input.fail() && !input.eof())) { + return error("Unable to read file '" + _fname + "'."); + } + bytes.insert(bytes.end(), buffer.data(), buffer.data() + input.gcount()); + } while (!input.eof()); + return bytes; + } catch (const std::exception& e) { + return error("Unable to read file '" + _fname + "': " + e.what()); + } +} + +} // namespace rfl::io::internal + +#endif diff --git a/include/rfl/io/load_string.hpp b/include/rfl/io/load_string.hpp index 8718632bc..0292cc8fc 100644 --- a/include/rfl/io/load_string.hpp +++ b/include/rfl/io/load_string.hpp @@ -1,25 +1,16 @@ #ifndef RFL_IO_LOAD_STRING_HPP_ #define RFL_IO_LOAD_STRING_HPP_ -#include #include #include "../Result.hpp" +#include "load_file.hpp" namespace rfl { namespace io { inline Result load_string(const std::string& _fname) { - std::ifstream infile(_fname); - if (infile.is_open()) { - auto r = std::string(std::istreambuf_iterator(infile), - std::istreambuf_iterator()); - infile.close(); - return r; - } else { - return error("Unable to open file '" + _fname + - "' or file could not be found."); - } + return internal::load_file(_fname, std::ios::in); } } // namespace io diff --git a/include/rfl/io/save_bytes.hpp b/include/rfl/io/save_bytes.hpp index 868a0cda1..a6c29b0e6 100644 --- a/include/rfl/io/save_bytes.hpp +++ b/include/rfl/io/save_bytes.hpp @@ -2,7 +2,6 @@ #define RFL_IO_SAVE_BYTES_HPP_ #include -#include #include #include "../Result.hpp" @@ -20,7 +19,10 @@ Result save_bytes(const std::string& _fname, const T& _obj, } _write(_obj, output); output.close(); - } catch (std::exception& e) { + if (!output) { + return error("Unable to write or close file '" + _fname + "'."); + } + } catch (const std::exception& e) { return error(e.what()); } return Nothing{}; diff --git a/include/rfl/io/save_string.hpp b/include/rfl/io/save_string.hpp index 17ee1433d..8d2abdb98 100644 --- a/include/rfl/io/save_string.hpp +++ b/include/rfl/io/save_string.hpp @@ -20,7 +20,10 @@ Result save_string(const std::string& _fname, const T& _obj, } _write(_obj, outfile); outfile.close(); - } catch (std::exception& e) { + if (!outfile) { + return error("Unable to write or close file '" + _fname + "'."); + } + } catch (const std::exception& e) { return error(e.what()); } return Nothing{}; diff --git a/tests/generic/test_file_io.cpp b/tests/generic/test_file_io.cpp new file mode 100644 index 000000000..f7937b580 --- /dev/null +++ b/tests/generic/test_file_io.cpp @@ -0,0 +1,164 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__unix__) || defined(__APPLE__) +#include + +#include +#endif + +namespace test_file_io { + +class FileIO : public testing::Test { + protected: + void SetUp() override { + const auto stamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + for (unsigned i = 0; i < 100; ++i) { + dir = + std::filesystem::temp_directory_path() / + ("reflectcpp-io-" + std::to_string(stamp) + "-" + std::to_string(i)); + if (std::filesystem::create_directory(dir)) return; + } + throw std::runtime_error("Cannot create test directory"); + } + void TearDown() override { + std::error_code ec; + std::filesystem::remove_all(dir, ec); + } + std::string path() const { return (dir / "data").string(); } + static void write(const std::string& data, std::ostream& out) { + out.write(data.data(), static_cast(data.size())); + } + std::filesystem::path dir; +}; + +TEST_F(FileIO, BinaryRoundTripAndTruncate) { + std::string data(2 * 1024 * 1024 + 17, '\0'); + for (std::size_t i = 0; i < data.size(); ++i) data[i] = char(i % 256); + ASSERT_TRUE(rfl::io::save_bytes(path(), data, write)); + const auto result = rfl::io::load_bytes(path()); + ASSERT_TRUE(result); + EXPECT_EQ(std::string(result->begin(), result->end()), data); + ASSERT_TRUE(rfl::io::save_bytes(path(), std::string("short"), write)); + EXPECT_EQ(std::filesystem::file_size(path()), 5); +} + +TEST_F(FileIO, EmptyFiles) { + ASSERT_TRUE(rfl::io::save_bytes(path(), std::string{}, write)); + const auto bytes = rfl::io::load_bytes(path()); + ASSERT_TRUE(bytes); + EXPECT_TRUE(bytes->empty()); + ASSERT_TRUE(rfl::io::save_string(path(), std::string{}, write)); + const auto text = rfl::io::load_string(path()); + ASSERT_TRUE(text); + EXPECT_TRUE(text->empty()); +} + +TEST_F(FileIO, TextModeMatchesStreamIterators) { + std::string data; + for (int i = 0; i < 20000; ++i) data += "line\r\nnext\n"; + data.append( + "\0tail\x1a" + "end", + 9); + ASSERT_TRUE(rfl::io::save_bytes(path(), data, write)); + std::ifstream reference(path()); + const std::string expected{std::istreambuf_iterator(reference), {}}; + const auto result = rfl::io::load_string(path()); + ASSERT_TRUE(result); + EXPECT_EQ(*result, expected); +} + +TEST_F(FileIO, TextSaveRoundTrip) { + const std::string data = "first\nsecond\n"; + ASSERT_TRUE(rfl::io::save_string(path(), data, write)); + const auto result = rfl::io::load_string(path()); + ASSERT_TRUE(result); + EXPECT_EQ(*result, data); +} + +TEST_F(FileIO, MissingFilesAndInvalidOutputPaths) { + EXPECT_FALSE(rfl::io::load_bytes(path())); + EXPECT_FALSE(rfl::io::load_string(path())); + EXPECT_FALSE(rfl::io::save_bytes(dir.string(), std::string("x"), write)); + EXPECT_FALSE(rfl::io::save_string(dir.string(), std::string("x"), write)); +} + +TEST_F(FileIO, ReadFailuresAreErrors) { + EXPECT_FALSE(rfl::io::load_bytes(dir.string())); + EXPECT_FALSE(rfl::io::load_string(dir.string())); +} + +TEST_F(FileIO, WriteStateFailuresAreErrors) { + for (const auto state : {std::ios::failbit, std::ios::badbit}) { + const auto fail = [state](const auto&, std::ostream& out) { + out.setstate(state); + }; + EXPECT_FALSE(rfl::io::save_bytes(path(), 0, fail)); + EXPECT_FALSE(rfl::io::save_string(path(), 0, fail)); + } +} + +TEST_F(FileIO, CloseFailuresAreErrors) { + const auto close_early = [](const auto&, std::ofstream& out) { out.close(); }; + EXPECT_FALSE(rfl::io::save_bytes(path(), 0, close_early)); + EXPECT_FALSE(rfl::io::save_string(path(), 0, close_early)); +} + +TEST_F(FileIO, WriterExceptionsAreErrors) { + const auto fail = [](const auto&, std::ostream&) { + throw std::runtime_error("writer failure"); + }; + const auto bytes = rfl::io::save_bytes(path(), 0, fail); + const auto text = rfl::io::save_string(path(), 0, fail); + ASSERT_FALSE(bytes); + ASSERT_FALSE(text); + EXPECT_EQ(bytes.error().what(), "writer failure"); + EXPECT_EQ(text.error().what(), "writer failure"); +} + +#if defined(__unix__) || defined(__APPLE__) +TEST_F(FileIO, NonSeekableFiles) { + ASSERT_EQ(::mkfifo(path().c_str(), 0600), 0); + const std::string data(192 * 1024 + 7, 'x'); + for (const bool binary : {true, false}) { + auto writer = std::async(std::launch::async, [&] { + std::ofstream out(path(), std::ios::binary); + write(data, out); + out.close(); + return bool(out); + }); + if (binary) { + const auto result = rfl::io::load_bytes(path()); + ASSERT_TRUE(result); + EXPECT_EQ(std::string(result->begin(), result->end()), data); + } else { + const auto result = rfl::io::load_string(path()); + ASSERT_TRUE(result); + EXPECT_EQ(*result, data); + } + EXPECT_TRUE(writer.get()); + } +} +#endif + +#if defined(__linux__) +TEST_F(FileIO, BufferedOutputFailureAtClose) { + // A small write can remain buffered until close; /dev/full rejects the flush. + EXPECT_FALSE(rfl::io::save_bytes("/dev/full", std::string("x"), write)); + EXPECT_FALSE(rfl::io::save_string("/dev/full", std::string("x"), write)); +} +#endif + +} // namespace test_file_io