diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af437c51..ba27c934 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./TestResults/*.cobertura.xml + flags: csharp fail_ci_if_error: false isa-fallback: diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 527b73a1..7cdea891 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -105,7 +105,11 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - components: clippy + components: clippy, llvm-tools-preview + + - name: Install cargo-llvm-cov (Linux) + if: runner.os == 'Linux' + uses: taiki-e/install-action@cargo-llvm-cov - name: Cache cargo uses: Swatinem/rust-cache@v2 @@ -143,11 +147,29 @@ jobs: run: cargo clippy --workspace --all-targets --all-features -- -D warnings - name: cargo test + if: runner.os != 'Linux' working-directory: rust env: EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }} run: cargo test --workspace --all-features --verbose + # The derive crate runs inside the compiler, where coverage is not recorded, so it is left out. + - name: cargo test with coverage (Linux) + if: runner.os == 'Linux' + working-directory: rust + env: + EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }} + run: cargo llvm-cov --workspace --all-features --ignore-filename-regex excelreader-derive --lcov --output-path lcov.info + + - name: Upload Rust coverage to Codecov + if: runner.os == 'Linux' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./rust/lcov.info + flags: rust + fail_ci_if_error: false + # The next two compile the crate again (default features, then the bench profile); that # catches nothing platform-specific the test build above missed, so one OS is enough. - name: cargo build (default features) @@ -197,10 +219,21 @@ jobs: run: chmod +x python/src/excelreader/_lib/* - name: Install the Python package - run: pip install -e "python[dev]" + run: pip install -e "python[dev]" ${{ matrix.os == 'ubuntu-latest' && 'pytest-cov' || '' }} - name: Run the Python tests - run: pytest python/tests -v + run: >- + pytest python/tests -v + ${{ matrix.os == 'ubuntu-latest' && '--cov=excelreader --cov-report=xml:coverage-python.xml' || '' }} + + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage-python.xml + flags: python + fail_ci_if_error: false - name: Build the wheel and check it installs here run: | diff --git a/README.md b/README.md index 5f745b20..cb3bd6e6 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages XLSX, XLSB, XLS and CSV without a .NET runtime installed. - C ABI header: [`src/ExcelReader.Native/include/excelreader.h`](src/ExcelReader.Native/include/excelreader.h) -- Python package: [`python/`](python/README.md) +- Python package: `pip install excelreader-native` — see [`python/README.md`](python/README.md). - C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet`/`xl::write_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md). - Rust crate: safe `Workbook`/`parse_sheet`/`write_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md). diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..203f0652 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,14 @@ +# Each language uploads under its own flag from a path-filtered job, so a flag a commit did not run keeps its last report. +flag_management: + default_rules: + carryforward: true + individual_flags: + - name: csharp + paths: + - src/ + - name: rust + paths: + - rust/ + - name: python + paths: + - python/ diff --git a/cpp/README.md b/cpp/README.md index ff58c24f..a9c7ecc6 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -350,6 +350,16 @@ toggles. Its `format` defaults to `XL_FORMAT_AUTO`, which takes the format from extension (.xlsx, .xlsb, .xls or .csv, in any case); an unrecognized extension, or AUTO with a vector destination, fails the write rather than silently choosing a format. +For XLSX and XLSB, `degree_of_parallelism` formats and compresses batches of `batch_rows` rows on +several threads: `0` uses every processor, `1` (the default) writes serially, and `batch_rows = 0` +keeps the library default of 8,192. The columns must not change while the write runs. With +`use_shared_strings` on, two parallel runs give the same content in files that differ byte for byte. +XLS, CSV and `xl::WriterHandle` ignore both fields. + +```cpp +xl::write_columns("big.xlsx", columns, {.degree_of_parallelism = 0}); +``` + `write_sheet` walks the range once and appends each field to its own column buffer, with the per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing. @@ -394,6 +404,40 @@ auto counted = xl::aggregate_csv("big.csv", RowCount{}, {.csv_delimiter = ';'}, The dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from `xl::OpenOptions`; the header row and the degree of parallelism (0 = every processor) follow it. +## Callback reads + +`Sheet::for_each_row` calls a function once per row on the calling thread, with cells borrowed from +the library's buffers for that call instead of copied into a cursor: + +```cpp +double total = 0; +auto walked = workbook->sheet(0).for_each_row([&](xl::RowView row) +{ + if (row.size() > 1) + { + total += row[1].as_number().value_or(0); + } + return XL_OK; +}); +``` + +`Sheet::aggregate` folds any sheet, in any format, across worker threads; the accumulator has the same +shape as for `aggregate_csv`, so `RowCount` above works unchanged. New code should use it rather than +`aggregate_csv`: + +```cpp +auto counted = workbook->sheet(0).aggregate(RowCount{}, /*header_row*/ 1); +``` + +A cell in these callbacks carries text, a number, or both. `value` holds only the stored text, so it +is empty for a cell that carries only a number; `text()` formats that number itself (shortest +round-trip form) and `as_number()` returns it directly, or parses the text when there is no number. +A return other than `XL_OK` stops the run and comes back as the error's code (use positive codes), +and an exception is reported as an error. Calling other functions on the workbook from inside the +callback is allowed. `Sheet::aggregate` seeds a copy per chunk of sheet data, not per thread, and may +destroy a chunk's copy without combining it; XLS, table views and CSV opened from a `Source` or +`InputStream` read sequentially. + ## Bounds and ABI `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index d89c0e34..ace20067 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -58,6 +58,8 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE) GIT_TAG v1.5.0 ) FetchContent_MakeAvailable(xlnt) + target_compile_options(xlnt PUBLIC + "$<$,$>>:SHELL:-include cstdint;SHELL:-include limits>") # xlsxio (https://github.com/brechtsanders/xlsxio) - its own CMakeLists.txt requires an # already-installed expat + Minizip (FIND_PACKAGE(... REQUIRED), no FetchContent-friendly diff --git a/cpp/benchmarks/benchmark_csv_compare.cpp b/cpp/benchmarks/benchmark_csv_compare.cpp index 8890e48c..008cec41 100644 --- a/cpp/benchmarks/benchmark_csv_compare.cpp +++ b/cpp/benchmarks/benchmark_csv_compare.cpp @@ -290,6 +290,73 @@ static void BM_ExcelReader_Csv_Cells(benchmark::State &state) } BENCHMARK(BM_ExcelReader_Csv_Cells); +static void BM_ExcelReader_Csv_ForEach(benchmark::State &state) +{ + const std::vector buffer = read_fixture(); + for (auto _ : state) + { + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); + if (!workbook.has_value()) + { + state.SkipWithError(workbook.error().message); + return; + } + int64_t acc = 0; + auto opened = workbook->sheet(0).for_each_row([&acc](const xl::RowView &row) { + for (const xl::CellView cell : row) + { + acc += static_cast(cell.value.size()); + } + return XL_OK; + }); + benchmark::DoNotOptimize(acc); + state.counters["checksum"] = static_cast(acc); + } +} +BENCHMARK(BM_ExcelReader_Csv_ForEach); + +static void BM_ExcelReader_Csv_Aggregate(benchmark::State &state) +{ + struct AggregateRun + { + std::int64_t cell_bytes = 0; + std::int32_t accumulate(xl::RowView row) + { + for (const xl::CellView cell : row) + { + cell_bytes += static_cast(cell.value.size()); + } + return XL_OK; + } + std::int32_t combine(AggregateRun &next) + { + cell_bytes += next.cell_bytes; + return XL_OK; + } + }; + const std::vector buffer = read_fixture(); + for (auto _ : state) + { + auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV}); + if (!workbook.has_value()) + { + state.SkipWithError(workbook.error().message); + return; + } + int64_t acc = 0; + auto data = workbook->sheet(0).aggregate(); + if (!data.has_value()) + { + state.SkipWithError(data.error().message); + return; + } + auto &agg = data.value(); + benchmark::DoNotOptimize(agg); + state.counters["checksum"] = static_cast(agg.cell_bytes); + } +} +BENCHMARK(BM_ExcelReader_Csv_Aggregate); + static void BM_Zsv_Csv_Cells(benchmark::State &state, unsigned char engine) { const std::vector buffer = read_fixture(); diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index a91732ae..fccf9fbd 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -33,6 +33,7 @@ target_link_libraries(excelreader_cpp_rows PRIVATE xl::excelreader) set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../RealExcel.xlsx" CACHE FILEPATH "Path to the RealExcel.xlsx fixture the C++ row tests read") target_compile_definitions(excelreader_cpp_rows PRIVATE + EXCELREADER_FIXTURE_PATH="${EXCELREADER_FIXTURE_PATH}" EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") add_test(NAME excelreader_cpp_rows COMMAND excelreader_cpp_rows) @@ -68,4 +69,17 @@ add_executable(excelreader_cpp_aggregate aggregate.cpp) target_link_libraries(excelreader_cpp_aggregate PRIVATE xl::excelreader) add_test(NAME excelreader_cpp_aggregate COMMAND excelreader_cpp_aggregate) -excelreader_copy_native_library(excelreader_cpp_aggregate) \ No newline at end of file +excelreader_copy_native_library(excelreader_cpp_aggregate) + +add_executable(excelreader_cpp_numbers numbers.cpp) +target_link_libraries(excelreader_cpp_numbers PRIVATE xl::excelreader) + +add_test(NAME excelreader_cpp_numbers COMMAND excelreader_cpp_numbers) +excelreader_copy_native_library(excelreader_cpp_numbers) + +add_executable(excelreader_cpp_numbers_fallback numbers.cpp) +target_link_libraries(excelreader_cpp_numbers_fallback PRIVATE xl::excelreader) +target_compile_definitions(excelreader_cpp_numbers_fallback PRIVATE EXCELREADER_TEST_NO_TO_CHARS) + +add_test(NAME excelreader_cpp_numbers_fallback COMMAND excelreader_cpp_numbers_fallback) +excelreader_copy_native_library(excelreader_cpp_numbers_fallback) diff --git a/cpp/tests/aggregate.cpp b/cpp/tests/aggregate.cpp index b4c72147..366da363 100644 --- a/cpp/tests/aggregate.cpp +++ b/cpp/tests/aggregate.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -313,8 +314,200 @@ static int test_aggregate_takes_the_dialect_from_open_options() return 0; } +namespace +{ + std::vector generate(int32_t format, int64_t rows) + { + std::vector bytes; + xl::WriteOptions options{}; + options.format = format; + auto writer = xl::WriterHandle::open(bytes, options); + if (!writer.has_value() || !writer->start_sheet("S1").has_value()) + { + return {}; + } + (void)writer->start_row(); + (void)writer->write(std::string_view("id")); + (void)writer->write(std::string_view("label")); + (void)writer->end_row(); + for (int64_t i = 1; i <= rows; ++i) + { + (void)writer->start_row(); + (void)writer->write(i); + (void)writer->write("filler-value-" + std::to_string(i)); + (void)writer->end_row(); + } + (void)writer->end_sheet(); + if (!writer->finish().has_value()) + { + return {}; + } + return bytes; + } + + std::expected open_generated(const std::vector &bytes, int32_t format) + { + return xl::Workbook::open(std::span(bytes), xl::OpenOptions{.format = format}); + } + + struct Totals + { + int64_t sum = 0; + int64_t rows = 0; + int64_t combines = 0; + + int32_t accumulate(xl::RowView row) + { + const auto number = row.empty() ? std::nullopt : row[0].as_number(); + if (!number.has_value()) + { + return 1; + } + sum += static_cast(*number); + ++rows; + return XL_OK; + } + + int32_t combine(Totals &next) + { + sum += next.sum; + rows += next.rows; + combines += next.combines + 1; + return XL_OK; + } + }; + + static_assert(xl::Accumulator, "Totals must satisfy xl::Accumulator"); + + std::atomic g_live{0}; + + struct Live + { + Live() { g_live.fetch_add(1); } + Live(const Live &) { g_live.fetch_add(1); } + Live(Live &&) noexcept { g_live.fetch_add(1); } + ~Live() { g_live.fetch_sub(1); } + + int32_t accumulate(xl::RowView) noexcept { return XL_OK; } + int32_t combine(Live &) noexcept { return XL_OK; } + }; + + struct Aborting + { + std::atomic *seen = nullptr; + + int32_t accumulate(xl::RowView) noexcept { return seen->fetch_add(1) >= 20'000 ? 9 : XL_OK; } + int32_t combine(Aborting &) noexcept { return XL_OK; } + }; + + struct Throwing + { + int64_t rows = 0; + + int32_t accumulate(xl::RowView) + { + if (++rows == 100) + { + throw std::runtime_error("boom"); + } + return XL_OK; + } + int32_t combine(Throwing &next) + { + rows += next.rows; + return XL_OK; + } + }; +} + +static int test_sheet_aggregate_sums_every_data_row_once() +{ + for (const int32_t format : {XL_FORMAT_XLSX, XL_FORMAT_XLSB, XL_FORMAT_XLS, XL_FORMAT_CSV}) + { + const auto bytes = generate(format, 60'000); + auto workbook = open_generated(bytes, format); + CHECK(workbook.has_value(), "open the generated workbook"); + auto totals = workbook->sheet(0).aggregate(Totals{}, 1, 4); + CHECK(totals.has_value(), "Sheet::aggregate must succeed"); + CHECK(totals->rows == 60'000, "every data row is accumulated once"); + CHECK(totals->sum == 60'000LL * 60'001 / 2, "every data row is summed once"); + if (format == XL_FORMAT_XLSX || format == XL_FORMAT_XLSB) + { + CHECK(totals->combines >= 1, "XLSX and XLSB must partition"); + } + } + return 0; +} + +static int test_sheet_aggregate_destroys_every_copy() +{ + const auto bytes = generate(XL_FORMAT_XLSX, 60'000); + auto workbook = open_generated(bytes, XL_FORMAT_XLSX); + CHECK(workbook.has_value(), "open the generated workbook"); + { + auto result = workbook->sheet(0).aggregate(Live{}, 1, 4); + CHECK(result.has_value(), "the run must succeed"); + } + CHECK(g_live.load() == 0, "every copy of the accumulator, including the one returned, is destroyed"); + return 0; +} + +static int test_sheet_aggregate_returns_a_callback_code() +{ + const auto bytes = generate(XL_FORMAT_XLSX, 60'000); + auto workbook = open_generated(bytes, XL_FORMAT_XLSX); + CHECK(workbook.has_value(), "open the generated workbook"); + std::atomic seen{0}; + auto result = workbook->sheet(0).aggregate(Aborting{&seen}, 1, 4); + CHECK(!result.has_value() && result.error().code == 9, "a callback code comes back as the error code"); + return 0; +} + +static int test_sheet_aggregate_reports_a_throwing_accumulate() +{ + const auto bytes = generate(XL_FORMAT_XLSB, 60'000); + auto workbook = open_generated(bytes, XL_FORMAT_XLSB); + CHECK(workbook.has_value(), "open the generated workbook"); + auto result = workbook->sheet(0).aggregate(Throwing{}, 1, 4); + CHECK(!result.has_value(), "an exception in accumulate is reported as an error"); + return 0; +} + +static int test_existing_csv_accumulator_works_unchanged() +{ + const auto bytes = generate(XL_FORMAT_CSV, 10'000); + auto workbook = open_generated(bytes, XL_FORMAT_CSV); + CHECK(workbook.has_value(), "open the generated CSV"); + auto through_sheet = workbook->sheet(0).aggregate(RowCounter{}, 1, 4); + auto through_csv = xl::aggregate_csv(std::span(bytes), RowCounter{}, {}, 1, 4); + CHECK(through_sheet.has_value() && through_csv.has_value(), "both runs succeed"); + CHECK(through_sheet->rows == through_csv->rows && through_sheet->rows == 10'000, + "a CsvAccumulator gives the same result through Sheet::aggregate"); + return 0; +} + int main() { + if (int failed = test_sheet_aggregate_sums_every_data_row_once()) + { + return failed; + } + if (int failed = test_sheet_aggregate_destroys_every_copy()) + { + return failed; + } + if (int failed = test_sheet_aggregate_returns_a_callback_code()) + { + return failed; + } + if (int failed = test_sheet_aggregate_reports_a_throwing_accumulate()) + { + return failed; + } + if (int failed = test_existing_csv_accumulator_works_unchanged()) + { + return failed; + } if (int failed = test_aggregate_csv_memory()) { return failed; diff --git a/cpp/tests/numbers.cpp b/cpp/tests/numbers.cpp new file mode 100644 index 00000000..152e2123 --- /dev/null +++ b/cpp/tests/numbers.cpp @@ -0,0 +1,77 @@ +// Built twice: once as is, and once with EXCELREADER_TEST_NO_TO_CHARS, which hides the standard +// library's floating-point to exercise the fallback a libc++ without it compiles. +#include +#ifdef EXCELREADER_TEST_NO_TO_CHARS +#undef __cpp_lib_to_chars +#endif +#include + +#include +#include +#include + +namespace +{ + int failures = 0; + + void check(bool condition, const char *what) + { + if (!condition) + { + std::fprintf(stderr, "FAILED: %s\n", what); + ++failures; + } + } + + xl::CellView text_cell(std::string_view text) + { + return xl::CellView{0, xl::CellType::Number, text, std::nullopt}; + } + + void round_trips(double value) + { + const std::string text = xl::CellView{0, xl::CellType::Number, {}, value}.text(); + check(text_cell(text).as_number() == value, "text() of a number parses back to the same double"); + } +} + +int main() +{ + check(text_cell("12.5").as_number() == 12.5, "a decimal parses"); + check(text_cell("1e3").as_number() == 1000.0, "an exponent parses"); + check(text_cell("-7").as_number() == -7.0, "a negative parses"); + check(!text_cell("abc").as_number().has_value(), "text is not a number"); + check(!text_cell("").as_number().has_value(), "empty is not a number"); + check(!text_cell(" 5").as_number().has_value(), "leading space is not a number"); + check(!text_cell("5x").as_number().has_value(), "trailing garbage is not a number"); + check(!text_cell("1,5").as_number().has_value(), "a comma decimal is not a number"); + + check(xl::CellView{0, xl::CellType::Number, {}, 42.5}.text() == "42.5", "a short number formats short"); + check(xl::CellView{0, xl::CellType::Number, {}, 3.0}.text() == "3", "an integral number has no fraction"); + check(xl::CellView{0, xl::CellType::Number, "kept", 1.0}.text() == "kept", "stored text wins over the number"); + check(xl::CellView{0, xl::CellType::Number, {}, 1000.0}.text() == "1000", "fixed wins when it is shorter"); + check(xl::CellView{0, xl::CellType::Number, {}, 0.001}.text() == "0.001", "fixed wins a tie"); + check(xl::CellView{0, xl::CellType::Number, {}, 1e20}.text() == "1e+20", "scientific wins when it is shorter"); + check(xl::CellView{0, xl::CellType::Number, {}, 1e-7}.text() == "1e-07", "a small number goes scientific"); + check(xl::CellView{0, xl::CellType::Number, {}, 123456789012345680000.0}.text() == "123456789012345683968", + "a large integer prints its exact value"); + check(xl::CellView{0, xl::CellType::Number, {}, 0.0001}.text() == "1e-04", "scientific wins at 1e-04"); + check(xl::CellView{0, xl::CellType::Number, {}, 123456.0}.text() == "123456", "an integer stays fixed"); + check(xl::CellView{0, xl::CellType::Number, {}, 1e15}.text() == "1e+15", "a round power of ten goes scientific"); + check(xl::CellView{0, xl::CellType::Number, {}, 0.0125}.text() == "0.0125", "a fraction places its point"); + check(xl::CellView{0, xl::CellType::Number, {}, -2.5}.text() == "-2.5", "a negative keeps its sign"); + check(xl::CellView{0, xl::CellType::Number, {}, 0.0}.text() == "0", "zero is 0"); + round_trips(0.1); + round_trips(1.0 / 3.0); + round_trips(1e21); + round_trips(-2.5e-300); + round_trips(45000.25); + + if (failures != 0) + { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + std::puts("numbers: OK"); + return EXIT_SUCCESS; +} diff --git a/cpp/tests/rows.cpp b/cpp/tests/rows.cpp index 655c457b..934d3960 100644 --- a/cpp/tests/rows.cpp +++ b/cpp/tests/rows.cpp @@ -1,7 +1,11 @@ #include +#include #include #include +#include +#include +#include #include #include #include @@ -60,10 +64,242 @@ namespace } return all; } + + using Cell = std::tuple, std::string>; + + std::vector collect(xl::RowView row) + { + std::vector cells; + for (auto cell : row) + { + cells.emplace_back(cell.column, cell.type, cell.as_number(), std::string(cell.value)); + } + return cells; + } + + std::vector large_numbers(int32_t format) + { + std::vector bytes; + xl::WriteOptions options{}; + options.format = format; + auto writer = xl::WriterHandle::open(bytes, options); + if (!writer.has_value() || !writer->start_sheet("S1").has_value()) + { + return {}; + } + for (int64_t i = 1; i <= 20; ++i) + { + (void)writer->start_row(); + (void)writer->write(static_cast(i) * 1e20); + (void)writer->end_row(); + } + (void)writer->end_sheet(); + (void)writer->finish(); + return bytes; + } + + std::string shortest(double value) + { + char buffer[64]; + const auto result = std::to_chars(buffer, buffer + sizeof(buffer), value); + return std::string(buffer, result.ptr); + } + + void for_each_row_matches_the_cursor(const std::expected &workbook) + { + check(workbook.has_value(), "open the workbook for for_each_row"); + if (!workbook.has_value()) + { + return; + } + const xl::Sheet sheet = workbook->sheet(0); + + std::vector> pushed; + auto walked = sheet.for_each_row([&](xl::RowView row) + { + pushed.push_back(collect(row)); + return XL_OK; }); + check(walked.has_value(), "for_each_row succeeds"); + + std::vector> pulled; + auto cursor = sheet.rows(); + check(cursor.has_value(), "open a cursor to compare against"); + if (!cursor.has_value()) + { + return; + } + for (auto row = cursor->next_row(); row.has_value(); row = cursor->next_row()) + { + pulled.push_back(collect(*row)); + } + + check(pushed.size() == pulled.size(), "for_each_row sees as many rows as the cursor"); + for (size_t i = 0; i < pushed.size() && i < pulled.size(); ++i) + { + check(pushed[i].size() == pulled[i].size(), "same cell count per row"); + for (size_t j = 0; j < pushed[i].size() && j < pulled[i].size(); ++j) + { + const auto &[column, type, number, text] = pushed[i][j]; + const auto &[cursor_column, cursor_type, cursor_number, cursor_text] = pulled[i][j]; + check(column == cursor_column && type == cursor_type, "same column and type"); + if (!text.empty() && !cursor_text.empty()) + { + check(text == cursor_text, "same text when both carry text"); + } + else + { + check(number == cursor_number, "same number otherwise"); + } + } + } + } + + void number_only_cell_formats_its_text() + { + auto workbook = xl::Workbook::open(EXCELREADER_FIXTURE_PATH); + check(workbook.has_value(), "open the xlsb fixture"); + if (!workbook.has_value()) + { + return; + } + int seen = 0; + auto walked = workbook->sheet(0).for_each_row([&](xl::RowView row) + { + for (auto cell : row) + { + if (cell.value.empty() && cell.number.has_value()) + { + check(cell.text() == shortest(*cell.number), "a number-only cell formats its number"); + ++seen; + } + } + return XL_OK; }); + check(walked.has_value(), "for_each_row over the xlsb fixture succeeds"); + check(seen > 0, "the xlsb fixture has a cell that carries only a number"); + } + + void positive_code_stops_the_walk() + { + auto workbook = xl::Workbook::open(fixture()); + if (!workbook.has_value()) + { + check(false, "open the fixture"); + return; + } + int calls = 0; + auto walked = workbook->sheet(0).for_each_row([&](xl::RowView) + { return ++calls == 3 ? 7 : XL_OK; }); + check(!walked.has_value() && walked.error().code == 7, "a positive code comes back as the error code"); + check(calls == 3, "a positive code stops the walk"); + } + + void non_positive_code_is_an_error() + { + auto workbook = xl::Workbook::open(fixture()); + if (!workbook.has_value()) + { + check(false, "open the fixture"); + return; + } + auto walked = workbook->sheet(0).for_each_row([](xl::RowView) + { return -1; }); + check(!walked.has_value(), "a code <= 0 is still an error"); + } + + void nested_call_inside_the_callback_works() + { + auto workbook = xl::Workbook::open(fixture()); + if (!workbook.has_value()) + { + check(false, "open the fixture"); + return; + } + const xl::Sheet sheet = workbook->sheet(0); + const auto name = sheet.name(); + check(name.has_value(), "read the sheet name"); + auto walked = sheet.for_each_row([&](xl::RowView) + { + const auto nested = sheet.name(); + check(nested.has_value() && name.has_value() && *nested == *name, "a nested call returns the right name"); + return XL_OK; }); + check(walked.has_value(), "for_each_row with a nested call succeeds"); + } + + void exception_is_reported_and_the_workbook_stays_usable() + { + auto workbook = xl::Workbook::open(fixture()); + if (!workbook.has_value()) + { + check(false, "open the fixture"); + return; + } + const xl::Sheet sheet = workbook->sheet(0); + auto thrown = sheet.for_each_row([](xl::RowView) -> int32_t + { throw std::runtime_error("boom"); }); + check(!thrown.has_value(), "an exception in the callback is reported as an error"); + auto again = sheet.for_each_row([](xl::RowView) + { return XL_OK; }); + check(again.has_value(), "the workbook stays usable after an exception"); + } + + void empty_row_is_empty() + { + std::vector bytes; + { + xl::WriteOptions options{}; + options.format = XL_FORMAT_XLSX; + auto writer = xl::WriterHandle::open(bytes, options); + check(writer.has_value(), "open a writer"); + if (!writer.has_value()) + { + return; + } + check(writer->start_sheet("S1").has_value(), "start sheet"); + check(writer->start_row().has_value() && writer->end_row().has_value(), "write an empty row"); + check(writer->start_row().has_value(), "start a row"); + check(writer->write(std::string_view("x")).has_value(), "write a cell"); + check(writer->end_row().has_value() && writer->end_sheet().has_value(), "end the sheet"); + check(writer->finish().has_value(), "finish"); + } + auto workbook = xl::Workbook::open(std::span(bytes), xl::OpenOptions{.format = XL_FORMAT_XLSX}); + check(workbook.has_value(), "open the written workbook"); + if (!workbook.has_value()) + { + return; + } + std::vector sizes; + auto walked = workbook->sheet(0).for_each_row([&](xl::RowView row) + { + size_t iterated = 0; + for (auto cell : row) + { + (void)cell; + ++iterated; + } + check(iterated == row.size(), "iterating a row visits size() cells"); + sizes.push_back(row.size()); + return XL_OK; }); + check(walked.has_value(), "for_each_row over the written workbook succeeds"); + check(!sizes.empty() && sizes.front() == 0, "the first row seen is empty"); + } } int main() { + for_each_row_matches_the_cursor(xl::Workbook::open(fixture())); + for_each_row_matches_the_cursor(xl::Workbook::open(EXCELREADER_FIXTURE_PATH)); + { + const auto bytes = large_numbers(XL_FORMAT_XLSB); + for_each_row_matches_the_cursor( + xl::Workbook::open(std::span(bytes), xl::OpenOptions{.format = XL_FORMAT_XLSB})); + } + number_only_cell_formats_its_text(); + positive_code_stops_the_walk(); + non_positive_code_is_an_error(); + nested_call_inside_the_callback_works(); + exception_is_reported_and_the_workbook_stays_usable(); + empty_row_is_empty(); + const auto rows = read_with_cursor(fixture()); check(rows.size() > 1, "the fixture has a header plus data rows"); diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp index 9381e135..b1d8d0b9 100644 --- a/cpp/tests/write.cpp +++ b/cpp/tests/write.cpp @@ -228,6 +228,45 @@ static int test_write_columns_rejects_bad_input() return 0; } +static int test_write_columns_in_parallel() +{ + CHECK(xl::WriteOptions{}.degree_of_parallelism == 1, "the wrapper must default to a serial write"); + + const std::vector offsets{0, 1, 2, 3, 4, 5, 6, 7}; + const std::vector blob{'a', 'b', 'c', 'd', 'e', 'f', 'g'}; + const std::vector values{0, 10, 20, 30, 40, 50, 60}; + const std::array columns{xl::string_column("texto", offsets, blob), xl::i64_column("inteiro", values)}; + + for (const int32_t format : {XL_FORMAT_XLSX, XL_FORMAT_XLSB}) + { + std::vector bytes; + CHECK(xl::write_columns(bytes, columns, {.format = format, .degree_of_parallelism = 4, .batch_rows = 2}).has_value(), + "a parallel write must succeed"); + + auto workbook = xl::Workbook::open(bytes, {.format = format}); + CHECK(workbook.has_value(), "the parallel output must open"); + auto cursor = workbook->sheet(0).rows(); + CHECK(cursor.has_value(), "the parallel output must have a row cursor"); + int64_t index = 0; + for (auto row = cursor->next_row(); row.has_value(); row = cursor->next_row(), ++index) + { + std::vector cells; + for (auto cell : *row) + { + cells.emplace_back(cell.value); + } + CHECK(cells.size() == 2, "every row must have both cells"); + if (index > 0) + { + CHECK(cells[0] == std::string(1, static_cast('a' + index - 1)), "the string column must round-trip in order"); + CHECK(cells[1] == std::to_string((index - 1) * 10), "the int64 column must round-trip in order"); + } + } + CHECK(index == 8, "the header and seven data rows must be written"); + } + return 0; +} + static int test_write_options() { const std::vector values{1, 2}; @@ -678,6 +717,10 @@ int main() { return 1; } + if (test_write_columns_in_parallel() != 0) + { + return 1; + } if (test_format_from_extension() != 0) { return 1; diff --git a/docs/guide/versioning.md b/docs/guide/versioning.md index a4a15214..ea345f43 100644 --- a/docs/guide/versioning.md +++ b/docs/guide/versioning.md @@ -79,6 +79,12 @@ The C ABI is versioned on its own by `XL_ABI_VERSION`, which `xl_abi_version()` only when a signature or a struct layout does: 7.0 kept version 6. Every options struct starts with `struct_size`, so a field added in a later version does not break a caller compiled against an earlier header. The Rust, Python and C++ bindings check the ABI version when they load the library. +`xl_write_options` grew from 32 to 40 bytes after 7.0 (`degree_of_parallelism` and `batch_rows`, +appended); the 7.0 size is still accepted and writes serially, so the ABI version stayed 6. + +7.x also added `xl_rows_foreach` and `xl_aggregate`. They are new exports only, so the ABI version +stayed 6; a binding built against them fails with a missing-symbol error when it loads an older +ABI-6 library that lacks them. ## How it is enforced diff --git a/docs/guide/writing.md b/docs/guide/writing.md index dd82c515..26e6e90e 100644 --- a/docs/guide/writing.md +++ b/docs/guide/writing.md @@ -339,3 +339,26 @@ Writing 1,000,000 rows of four columns (text, integer, date, number) to a `Memor The file comes out 0.18% (XLSX) or 0.04% (XLSB) larger, because each batch is compressed on its own. As with `PrefetchWrite`, this is for single-file batch work: a server already writing many files at once has no idle cores to give it. + +### From the native library and the bindings + +`xl_write_typed` and `xl_write_typed_to_memory` take the same two knobs in `xl_write_options`: +`degree_of_parallelism` and `batch_rows`, with the same meaning (`0` is every processor and the +8,192-row default). A zero-initialized struct therefore writes XLSX and XLSB in parallel; a struct +with the 7.0 size (32 bytes) or a NULL options pointer writes serially, as before. The C++, Rust and +Python wrappers default to a serial write and turn it on only when asked: + +```cpp +xl::write_columns("big.xlsx", columns, {.degree_of_parallelism = 0}); +``` + +```rust +write_columns("big.xlsx", &columns, &WriteOptions::new().degree_of_parallelism(0))?; +``` + +```python +write_workbook("big.xlsx", table, types, options=WriteOptions(degree_of_parallelism=0)) +``` + +The table's buffers are read from several threads, so they must not change while the call runs. +XLS, CSV and the `xl_writer_handle` API ignore both fields. diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md index cf2dfeac..d58161b9 100644 --- a/docs/performance/benchmarks.md +++ b/docs/performance/benchmarks.md @@ -7,7 +7,7 @@ Throwaway numbers go stale; these are regenerated from the benchmark suite in Benchmarks were run with BenchmarkDotNet v0.15.8 on Windows 10 (22H2), AMD Ryzen 7 5700X, .NET 10.0.12 (SDK 10.0.401). Generated-data benchmarks use 50,000 rows, except the string-heavy reads, which use 65,536. Raw results: [GitHub Pages benchmark dashboard](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/) — `tests/ExcelReader.Benchmarks/BenchmarkDotNet.Artifacts/` is `.gitignore`-excluded and never actually reaches GitHub, so a link to it 404s for every reader. -Every BenchmarkDotNet table is a `--job Short` run (1 launch, 3 warmup and 3 measured iterations) on an otherwise idle machine. Rows whose 99.9% confidence interval came out wider than 25% of the mean were run again with `--job Medium` (2 launches, 15 iterations) and carry those figures instead: the parallel CSV tables, the CSV cell-by-cell read, the Sylvan XLS read, the Sylvan string-heavy reads, the string-heavy XLSX matched-work read, the generated-workbook matched-work read, `DataTable.Load` and the large encrypted read. The cold-start rows set their own launch count, and the file-size and working-set figures under the parallel record writes, the Python/C++/Rust figures and the native-library comparison come from other harnesses that were not re-run. The large-sheet and parallel-write tables are `--job Medium` runs. +Every BenchmarkDotNet table is a `--job Short` run (1 launch, 3 warmup and 3 measured iterations) on an otherwise idle machine. Rows whose 99.9% confidence interval came out wider than 25% of the mean were run again with `--job Medium` (2 launches, 15 iterations) and carry those figures instead: the parallel CSV tables, the CSV cell-by-cell read, the Sylvan XLS read, the Sylvan string-heavy reads, the string-heavy XLSX matched-work read, the generated-workbook matched-work read, `DataTable.Load`, the large encrypted read and the `InternStrings` typed parse. The cold-start rows set their own launch count, and the file-size and working-set figures under the parallel record writes, the Python/C++/Rust figures and the native-library comparison come from other harnesses that were not re-run. The large-sheet and parallel-write tables are `--job Medium` runs. **Benchmark methodology.** In the Excel-format tables below, the **"Cell-by-cell read"** rows (including the per-format rows in "Real data reads" / "String-heavy reads") read ExcelReader's `cell.Value` — a zero-copy `ReadOnlySpan`, no decode or allocation — against each competitor's own idiomatic read call. For Sylvan.Data.Excel, that's the ADO.NET-style `GetString(i)`, which is forced to materialize a UTF-16 `string`; its API has no zero-copy accessor, so it cannot avoid that cost. These rows are therefore not matched work: part of the reported gap is "we parse faster" and part is "we skipped an allocation you were never offered a way to skip." Each affected benchmark class also has a `*_Materialized` sibling (calling `cell.GetString()`, the same UTF-16 materialization Sylvan pays) so the matched-work number is measurable — see `tests/ExcelReader.Benchmarks/Shared/BenchmarkAccumulators.cs`. **Those results are published under [Matched-work reads](#matched-work-reads-_materialized), and reading them is the honest way to judge the comparison:** treat the Excel cell-by-cell ratios in the tables below as an upper bound, not a like-for-like number. The CSV cell-by-cell rows are matched work: Sep and Sylvan.Data.Csv both expose span accessors (`row[i].Span`, `GetFieldSpan(i)`), and the benchmarks use them. The **"Typed row parsing"** / **"Typed record writing"** rows are unaffected — both sides already materialize real objects/strings there, so those comparisons are matched work as published. @@ -22,10 +22,11 @@ Compares ExcelReader against established XLSX libraries on the same generated wo | Typed row parsing | 10.979 ms, 3.87 MB | 56.677 ms, 10.48 MB | 11.802 ms, 2.36 MB | - | | Typed row parsing async | 10.958 ms, 3.87 MB | 58.753 ms, 10.48 MB | - | - | | Typed row parsing, shared strings | 9.896 ms, 2.29 MB | - | - | - | +| Typed row parsing, `InternStrings` | 11.58 ms, 2.30 MB | - | - | - | | Workbook writing | 8.527 ms, 4.02 MB | - | 18.100 ms, 4.04 MB | 15.393 ms, 15.84 MB | | Workbook writing, shared strings | 8.416 ms, 4.06 MB | - | - | - | -ExcelReader is ~5.2x faster than Sylvan for raw XLSX reads, allocating ~534x less, and ~2.1x faster than OfficeIMO. For typed parsing it is ~5.2x faster than Sylvan (~5.4x async against async) and ~1.1x faster than OfficeIMO, which allocates less (2.36 MB against 3.87 MB). For XLSX writing, ExcelReader is ~1.8x faster than SpreadCheetah while allocating ~3.9x less memory, and ~2.1x faster than OfficeIMO. The OfficeIMO rows are OfficeIMO.Excel 3.4.5: that release made its XLSX read and typed parse several times faster than the 3.4.4 figures published before (106.861 ms and 107.805 ms). +ExcelReader is ~5.2x faster than Sylvan for raw XLSX reads, allocating ~534x less, and ~2.1x faster than OfficeIMO. For typed parsing it is ~5.2x faster than Sylvan (~5.4x async against async) and ~1.1x faster than OfficeIMO, which allocates less (2.36 MB against 3.87 MB) because it always interns strings. `ExcelReaderOptions.InternStrings` is off by default; turned on, ExcelReader allocates 2.30 MB at 11.58 ms, against OfficeIMO's 11.93 ms in the same `--job Medium` run. For XLSX writing, ExcelReader is ~1.8x faster than SpreadCheetah while allocating ~3.9x less memory, and ~2.1x faster than OfficeIMO. The OfficeIMO rows are OfficeIMO.Excel 3.4.5: that release made its XLSX read and typed parse several times faster than the 3.4.4 figures published before (106.861 ms and 107.805 ms). OfficeIMO.Excel's typed parsing is hand-mapped from its `OpenDataReader`, because `RowsAs` needs an `r` attribute on every row and cell, which the spec makes optional and ExcelReader's writer omits. @@ -70,30 +71,36 @@ For raw CSV reads, all three libraries read text through spans. ExcelReader is ~ `SheetParallel.ParseAsync` and `SheetParallel.AggregateAsync` over a `CsvSheet`, on generated files (8,000,000 rows of three ints for the narrow corpus, 4,300,000 rows of two strings, a date, two decimals and an int for the conversion-heavy one), by `degreeOfParallelism`. The machine has 8 physical / 16 logical cores. -| Dop | Conversion-heavy, typed | Narrow ints, typed | Conversion-heavy, `ref struct` aggregate | Narrow ints, `ref struct` aggregate | -|---:|---:|---:|---:|---:| -| 1 | 587.8 ms, 671.42 MB | 516.9 ms, 245.76 MB | 472.0 ms, 1.20 MB | 463.9 ms, 1.20 MB | -| 2 | 461.6 ms, 680.28 MB | 385.8 ms, 254.80 MB | 267.3 ms, 1.35 MB | 265.6 ms, 1.34 MB | -| 4 | 256.9 ms, 680.39 MB | 217.8 ms, 254.81 MB | 145.2 ms, 1.50 MB | 140.1 ms, 1.51 MB | -| 8 | 210.5 ms, 680.31 MB | 187.8 ms, 254.77 MB | 96.3 ms, 1.69 MB | 93.1 ms, 1.68 MB | -| 16 | 471.2 ms, 680.33 MB | 187.0 ms, 255.23 MB | 74.0 ms, 1.79 MB | 73.3 ms, 1.76 MB | +| Dop | Conversion-heavy, typed | Conversion-heavy, struct + `InternStrings` | Narrow ints, typed | Conversion-heavy, `ref struct` aggregate | Narrow ints, `ref struct` aggregate | +|---:|---:|---:|---:|---:|---:| +| 1 | 611.6 ms, 671.42 MB | 543.9 ms, 1.62 MB | 525.5 ms, 245.76 MB | 475.8 ms, 1.20 MB | 473.4 ms, 1.27 MB | +| 2 | 477.9 ms, 679.58 MB | 431.9 ms, 10.40 MB | 401.5 ms, 254.06 MB | 275.2 ms, 0.90 MB | 266.3 ms, 0.91 MB | +| 4 | 281.3 ms, 679.64 MB | 243.6 ms, 10.83 MB | 234.4 ms, 254.06 MB | 149.7 ms, 1.09 MB | 149.2 ms, 1.07 MB | +| 8 | 226.4 ms, 679.59 MB | 186.1 ms, 11.63 MB | 207.0 ms, 253.92 MB | 104.8 ms, 1.27 MB | 101.8 ms, 1.25 MB | +| 16 | 396.0 ms, 679.61 MB | 181.0 ms, 13.60 MB | 220.1 ms, 254.51 MB | 85.4 ms, 1.31 MB | 87.0 ms, 1.32 MB | -The typed path scales ~2.3x by dop 4 on the conversion-heavy corpus (587.8 to 256.9 ms) and ~2.4x on narrow ints (516.9 to 217.8 ms), then flattens out: narrow ints reach only 187.0 ms at dop 16, and the conversion-heavy corpus gets *slower* past dop 8 (471.2 ms at dop 16 against 210.5 ms at dop 8, standard deviation 23.2 ms). Its ~680 MB is one materialized row object per record, and at dop 16 that costs 42,000 Gen0 plus 41,000 Gen1 collections. +The typed path scales ~2.2x by dop 4 on both corpora (611.6 to 281.3 ms conversion-heavy, 525.5 to 234.4 ms narrow), then flattens out: narrow ints reach 207.0 ms at dop 8 and no better at dop 16, and the conversion-heavy corpus gets *slower* past dop 8 (396.0 ms at dop 16 against 226.4 ms at dop 8, standard deviation 14.1 ms). Its ~680 MB is one materialized row object and two strings per record, and at dop 16 that costs 42,000 Gen0 plus 41,000 Gen1 collections. -The aggregate columns fold each record into a per-partition accumulator through a `ref struct` model, so text columns stay as `ReadOnlySpan` and no row object is ever materialized. They scale better than the typed path — ~4.9x (conversion-heavy) and ~5.0x (narrow) at dop 8, ~6.4x and ~6.3x at dop 16 — because they are not fighting the allocator: at dop 4 the conversion-heavy aggregate is ~1.8x faster than typed while allocating ~450x less (1.50 MB against 680.39 MB), with **zero** garbage collections at every degree. The remaining allocation is per-partition bookkeeping, not per-row. +The struct + `InternStrings` column parses the same rows into a `struct` with `ExcelReaderOptions.InternStrings` on, so there is no row object and each repeated region or country name is one cached string. It never slows down: 186.1 ms at dop 8, 181.0 ms at dop 16, 2.2x faster than the class at dop 16 while allocating ~50x less (13.60 MB, no garbage collections at dop 16). What remains is per-worker caches and the ordered merge's buffers. The interning win depends on the data: the corpus has 4 regions and 6 countries, so every string is a cache hit. Text columns with mostly distinct values (names, ids, free text) allocate a string per cell either way, and on those the struct alone removes only the row object. -The typed path's slowdown past dop 8 is the garbage collector. Every degree allocates the same ~680 MB and runs 42 gen0 collections, but at dop 16 each of them also collects gen1 (41 gen1 collections against 24 at dop 8), and the pauses add up to 100-200 ms against 39 ms. In a console harness on the 4.3 million row corpus, with the default workstation GC, dop 16 took 365 ms; with `DOTNET_gcServer=1` it took 182 ms, faster than dop 8. The aggregate path allocates no row objects and is unaffected. +The aggregate columns fold each record into a per-partition accumulator through a `ref struct` model, so text columns stay as `ReadOnlySpan` and no row object is ever materialized. They scale better than the typed path — ~4.5x (conversion-heavy) and ~4.7x (narrow) at dop 8, ~5.6x and ~5.4x at dop 16 — because they are not fighting the allocator: at dop 4 the conversion-heavy aggregate is ~1.9x faster than typed while allocating ~620x less (1.09 MB against 679.64 MB), with **zero** garbage collections at every degree. The remaining allocation is per-partition bookkeeping, not per-row. -Type conversion is the floor under both. A single-threaded pass that allocates nothing still costs 472.0 ms, so parsing bytes into `DateTime`/`decimal`/`int` — not allocation and not I/O — is what the parallelism is actually buying down. +The typed path's slowdown past dop 8 is the garbage collector. Every degree allocates the same ~680 MB and runs 42 gen0 collections, but at dop 16 each of them also collects gen1 (41 gen1 collections against 24 at dop 8), and the pauses add up to 100-200 ms against 39 ms. In a console harness on the 4.3 million row corpus, with the default workstation GC, dop 16 took 365 ms; with `DOTNET_gcServer=1` it took 182 ms, faster than dop 8. Not allocating per row, as the struct + `InternStrings` and aggregate columns do, avoids it without changing the GC mode. + +Type conversion is the floor under all of them. A single-threaded pass that allocates nothing still costs 475.8 ms, so parsing bytes into `DateTime`/`decimal`/`int` — not allocation and not I/O — is what the parallelism is actually buying down. Against [Sep](https://github.com/nietras/Sep)'s own `ParallelEnumerate` (all cores; 8,000,000 narrow rows, 3,000,000 conversion-heavy rows): -| Corpus | ExcelReader sequential | ExcelReader parallel | Sep sequential | Sep parallel | +| Corpus, row type | ExcelReader sequential | ExcelReader parallel | Sep sequential | Sep parallel | |---|---:|---:|---:|---:| -| Narrow ints | 479.4 ms, 244.15 MB | 189.8 ms, 255.24 MB | 515.1 ms, 244.15 MB | 434.8 ms, 245.60 MB | -| Conversion-heavy | 397.9 ms, 467.31 MB | 330.4 ms, 474.83 MB | 808.7 ms, 467.31 MB | 619.0 ms, 469.93 MB | +| Narrow ints, class | 492.3 ms, 244.15 MB | 227.7 ms, 254.44 MB | 525.1 ms, 244.15 MB | 460.4 ms, 245.89 MB | +| Narrow ints, struct | 397.2 ms, 10.32 KB | 158.0 ms, 11.58 MB | 494.2 ms, 9.34 KB | 263.1 ms, 2.16 MB | +| Conversion-heavy, class | 418.9 ms, 467.31 MB | 286.4 ms, 474.33 MB | 831.0 ms, 467.31 MB | 717.6 ms, 468.34 MB | +| Conversion-heavy, struct + pooled strings | 375.6 ms, 11.37 KB | 121.8 ms, 10.84 MB | 803.5 ms, 10.77 KB | 413.5 ms, 7.76 MB | + +The struct rows give both libraries the same help: the row is a `struct`, and repeated text is pooled, through `ExcelReaderOptions.InternStrings` on ExcelReader's side and `SepToString.PoolPerCol()` (sequential) or `PoolPerColThreadSafe()` (parallel) on Sep's. Sep's parallel struct rows are from a Medium job, since the Short run spread too much (StdDev 47.6 ms on narrow ints); the conversion-heavy one still has a 30.4 ms standard deviation. -Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints (479.4 ms vs. 515.1 ms), and ~2.0x faster on the conversion-heavy corpus, where decimal parsing dominates. In parallel ExcelReader is ~2.3x (narrow) and ~1.9x (conversion-heavy) faster than Sep's, while yielding rows in file order. The conversion-heavy parallel run gains less than the narrow one (330.4 ms against 397.9 ms sequential) because it is bound by the allocator, like the typed dop 16 row above: 28 Gen1 collections per run. +Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints (492.3 ms vs. 525.1 ms), and ~2.0x faster on the conversion-heavy corpus, where decimal parsing dominates. In parallel ExcelReader is ~2.0x (narrow) and ~2.5x (conversion-heavy) faster than Sep's, while yielding rows in file order. With struct rows and pooled strings both libraries speed up and stop collecting, and ExcelReader stays ahead: ~1.2x and ~2.1x sequentially, ~1.7x and ~3.4x in parallel. The class rows gain less from parallelism (286.4 ms against 418.9 ms sequential on the conversion-heavy corpus) because they are bound by the allocator, like the typed dop 16 row above: 28 Gen1 collections per run; the struct rows run 3.1x faster in parallel on the same corpus. As in the table above, pooling pays off here because the conversion-heavy corpus repeats 4 regions and 6 countries. ### Parallel sheet reading @@ -128,17 +135,17 @@ At dop 4 XLSX reads ~3.2x faster than sequentially and XLSB ~2.6x. Dop 2 is a si | 8 | 46.92 ms, 77.61 MB | 53.51 ms, 99.74 MB | | 16 | 49.19 ms, 77.88 MB | 54.30 ms, 100.00 MB | -| CSV, Dop | Aggregate | Typed parse | -|---:|---:|---:| -| 1 | 426.07 ms, 1.99 KB | 543.94 ms, 623.08 MB | -| 2 | 226.22 ms, 6.77 KB | 338.08 ms, 626.87 MB | -| 4 | 118.30 ms, 10.60 KB | 191.11 ms, 626.71 MB | -| 8 | 90.67 ms, 18.32 KB | 189.98 ms, 626.43 MB | -| 16 | 64.16 ms, 34.29 KB | 207.63 ms, 626.57 MB | +| CSV, Dop | Aggregate | Typed parse | Typed parse, struct + `InternStrings` | +|---:|---:|---:|---:| +| 1 | 434.80 ms, 2.84 KB | 541.20 ms, 623.08 MB | 504.24 ms, 10.86 KB | +| 2 | 220.11 ms, 6.79 KB | 369.71 ms, 626.88 MB | 311.99 ms, 4.44 MB | +| 4 | 148.27 ms, 10.61 KB | 201.27 ms, 626.70 MB | 163.29 ms, 4.67 MB | +| 8 | 95.41 ms, 18.32 KB | 169.04 ms, 626.46 MB | 138.99 ms, 5.29 MB | +| 16 | 69.28 ms, 34.10 KB | 250.57 ms, 626.59 MB | 119.77 ms, 7.33 MB | -The XLSX and XLSB aggregates and typed parses each materialize a `Record` and its name string per row, which is where their ~77-107 MB comes from; the CSV aggregate binds a `ref struct` and allocates kilobytes, while the CSV typed parse allocates ~626 MB at every degree, one row object and two strings per record. +The XLSX and XLSB aggregates and typed parses each materialize a `Record` and its name string per row, which is where their ~77-107 MB comes from; the CSV aggregate binds a `ref struct` and allocates kilobytes, while the CSV typed parse allocates ~626 MB at every degree, one row object and two strings per record. The struct + `InternStrings` parse drops both (the corpus repeats 4 regions and 6 countries, see [Parallel CSV](#parallel-csv)) and keeps a few MB of per-worker caches and merge buffers. -XLSX and XLSB reach ~3.3x at dop 4 and then barely improve, as in the real-data table above (the XLSX aggregate gets to 4.9x at dop 16, the XLSB aggregate to 3.4x at dop 8). The CSV aggregate keeps scaling because nothing but the parser runs on the workers: 4.7x at dop 8 and 6.6x at dop 16. The typed CSV parse reaches 2.9x at dop 8 and then loses 9% at dop 16 to the collector (see [Parallel CSV](#parallel-csv)), so for a 4-million-row file the aggregate at dop 16 is ~3.2x faster than the typed parse and allocates no row objects. +XLSX and XLSB reach ~3.3x at dop 4 and then barely improve, as in the real-data table above (the XLSX aggregate gets to 4.9x at dop 16, the XLSB aggregate to 3.4x at dop 8). The CSV aggregate keeps scaling because nothing but the parser runs on the workers: 4.6x at dop 8 and 6.3x at dop 16. The typed CSV parse reaches 3.2x at dop 8 and then loses 48% at dop 16 to the collector (standard deviation 15.2 ms), while the struct + `InternStrings` parse keeps going to 4.2x at dop 16, 2.1x faster than the class. The aggregate is still the fastest, ~1.7x ahead of the struct parse at dop 16, because it never builds the row at all. ### Real data reads diff --git a/global.json b/global.json index c93bc05e..4d4dd03d 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.100", + "version": "10.0.402", "rollForward": "latestMinor", "allowPrerelease": false }, diff --git a/python/README.md b/python/README.md index eb3d9a33..4a9ff9fe 100644 --- a/python/README.md +++ b/python/README.md @@ -5,6 +5,15 @@ Read and write XLSX, XLSB, XLS and CSV through ExcelReader's NativeAOT library. No .NET runtime required — the shared library is self-contained. +## Install + +```bash +pip install excelreader-native +``` + +Wheels ship for Windows x64, macOS (arm64, x64) and Linux (glibc and musl, x64 and arm64). The +package installs as `excelreader-native` and imports as `excelreader`. + ## Install (from source) ```bash @@ -273,6 +282,15 @@ write_workbook( ) ``` +For xlsx and xlsb, `degree_of_parallelism` formats and compresses batches of `batch_rows` rows on +several threads: `0` uses every processor, `1` writes serially, and `None` is serial. A `batch_rows` +of `None` keeps the library default of 8,192. With shared strings on, two parallel runs give the same +content in files that differ byte for byte. xls and csv ignore both fields. + +```python +write_workbook("big.xlsx", table, types, options=WriteOptions(degree_of_parallelism=0)) +``` + **Phase-1 limits, stated plainly:** a single sheet only (no multi-sheet workbooks); the whole table must already be in memory (no streaming/chunked writes); no styling beyond the temporal number formats `xl_write_typed` applies to DATE/TIME/TIMESTAMP columns. `format="auto"` is not accepted — diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index e39b2e50..ca9aa581 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -337,6 +337,8 @@ class NativeWriteOptions(ctypes.Structure): ("csv_quote", ctypes.c_int32), ("date1904", ctypes.c_int32), ("use_shared_strings", ctypes.c_int32), + ("degree_of_parallelism", ctypes.c_int32), + ("batch_rows", ctypes.c_int32), ] @@ -366,6 +368,8 @@ def to_native_write_options(options: WriteOptions) -> NativeWriteOptions: raw.csv_quote = _opt_number(options.csv_quote) raw.date1904 = _opt_state(options.date1904) raw.use_shared_strings = _opt_state(options.use_shared_strings) + raw.degree_of_parallelism = 1 if options.degree_of_parallelism is None else options.degree_of_parallelism + raw.batch_rows = _opt_number(options.batch_rows) return raw diff --git a/python/src/excelreader/types.py b/python/src/excelreader/types.py index e486442f..ae86aa02 100644 --- a/python/src/excelreader/types.py +++ b/python/src/excelreader/types.py @@ -156,6 +156,11 @@ class WriteOptions: apply to CSV output only. `date1904` applies to xls/xlsb only; `use_shared_strings` to xlsx/xlsb only, where it shrinks files with many repeated strings at the cost of a string table. + `degree_of_parallelism` (xlsx/xlsb only) formats and compresses batches of `batch_rows` rows on + several threads: 0 uses every processor, 1 writes serially, and None is serial. A `batch_rows` + of None keeps the library default of 8192. With shared strings on, two parallel runs give the + same content in files that differ byte for byte. + Validation happens on the native side; an out-of-range value or an invalid sheet name raises `ExcelReaderError`. Fields mirror xl_write_options in src/ExcelReader.Native/include/excelreader.h. @@ -166,6 +171,8 @@ class WriteOptions: csv_quote: int | None = None date1904: bool | None = None use_shared_strings: bool | None = None + degree_of_parallelism: int | None = None + batch_rows: int | None = None class StringColumn: diff --git a/python/tests/test_native.py b/python/tests/test_native.py index 291186b6..62b70b76 100644 --- a/python/tests/test_native.py +++ b/python/tests/test_native.py @@ -264,6 +264,16 @@ def test_write_typed_round_trips_through_parse_typed(tmp_path): assert out_path.read_text(encoding="utf-8").replace("\r\n", "\n") == "qty\n3\n7\n" +def test_write_options_default_to_a_serial_write(): + from excelreader import WriteOptions + + assert ctypes.sizeof(_native.NativeWriteOptions) == 40 + raw = _native.to_native_write_options(WriteOptions()) + assert (raw.degree_of_parallelism, raw.batch_rows) == (1, 0) + raw = _native.to_native_write_options(WriteOptions(degree_of_parallelism=0, batch_rows=2)) + assert (raw.degree_of_parallelism, raw.batch_rows) == (0, 2) + + def test_abi_version_matches_the_header(): header = HEADER_PATH.read_text(encoding="utf-8") match = re.search(r"#define XL_ABI_VERSION (\d+)", header) diff --git a/python/tests/test_writer.py b/python/tests/test_writer.py index b7629cf7..8c10c697 100644 --- a/python/tests/test_writer.py +++ b/python/tests/test_writer.py @@ -79,6 +79,35 @@ def test_write_workbook_rejects_an_unknown_extension(source_csv, tmp_path): write_workbook(tmp_path / "out.parquet", table, _TYPES) +@pytest.mark.parametrize("extension", ["xlsx", "xlsb"]) +def test_write_workbook_in_parallel_round_trips(tmp_path, extension): + source = tmp_path / "nullable.csv" + source.write_text("qty\n0\n10\n\n30\n40\n50\n60\n", encoding="utf-8") + schema = [ColumnSpec(ColumnType.I64, name="qty", nullable=True)] + + with open_workbook(source) as workbook: + table = workbook.sheets[0].parse(schema) + + out = tmp_path / f"out.{extension}" + options = WriteOptions(degree_of_parallelism=4, batch_rows=2) + write_workbook(out, table, [spec.type for spec in schema], options=options) + + with open_workbook(out) as workbook: + result = workbook.sheets[0].parse(schema) + + assert result.row_count == 7 + assert [result.columns[0][i] for i in range(7) if i != 2] == [0, 10, 30, 40, 50, 60] + assert result.validity[0][0] & 0b0111_1111 == 0b0111_1011 + + +def test_write_workbook_rejects_a_negative_degree_of_parallelism(source_csv, tmp_path): + with open_workbook(source_csv) as workbook: + table = workbook.sheets[0].parse(_SCHEMA) + + with pytest.raises(ExcelReaderError, match="degree_of_parallelism"): + write_workbook(tmp_path / "out.xlsx", table, _TYPES, options=WriteOptions(degree_of_parallelism=-1)) + + def test_write_workbook_reports_a_native_rejection(source_csv, tmp_path): with open_workbook(source_csv) as workbook: table = workbook.sheets[0].parse(_SCHEMA) diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index ff6f18fe..7d88134d 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -289,6 +289,20 @@ than a wrong answer. toggles. Its format defaults to `Format::Auto`, inferred from the path's extension; an extension it does not recognize is an error - a file being created has no signature bytes to sniff. +For XLSX and XLSB, `degree_of_parallelism` formats and compresses batches of `batch_rows` rows on +several threads: `0` uses every processor, `1` writes serially, and leaving it unset is serial. +Unset `batch_rows` keeps the library default of 8,192. With shared strings on, two parallel runs +give the same content in files that differ byte for byte. XLS, CSV and `WriterHandle` ignore both. + +```rust,no_run +use excelreader::{write_columns, Column, ColumnData, WriteOptions}; +# let ids = [1i64, 2, 3]; +# let columns = [Column::new(Some("id"), ColumnData::I64(&ids))]; + +write_columns("big.xlsx", &columns, &WriteOptions::new().degree_of_parallelism(0))?; +# Ok::<(), excelreader::Error>(()) +``` + Each write has a path form and an `io::Write` form: `write_columns_to`/`write_sheet_to` build the workbook in memory and write it to the writer in one `write_all`. A writer has no extension, so these need an explicit format: @@ -422,6 +436,59 @@ The input is a path (`&str`, `&Path`, `&PathBuf`) or bytes (`&[u8]`); then `head `degree_of_parallelism`, as in `Sheet::parse`. A nonzero code from a callback comes back as `ErrorKind::Callback(code)`; a panic in one is resumed on the calling thread. +## Callback reads + +`Sheet::for_each_row` calls a closure once per row on the calling thread, with cells borrowed from +the library's buffers for that call instead of copied into a cursor: + +```rust,no_run +use excelreader::Workbook; + +let workbook = Workbook::open("report.xlsx")?; +let mut total = 0.0; +workbook.sheet(0)?.for_each_row(|row| { + if let Some(amount) = row.get(1).and_then(|cell| cell.number()) { + total += amount; + } + Ok(()) +})?; +# Ok::<(), excelreader::Error>(()) +``` + +`Sheet::aggregate` folds any sheet, in any format, into one accumulator across several threads. +New code should use it rather than `aggregate_csv`: + +```rust,no_run +use excelreader::{Accumulator, RowRef, Workbook}; + +struct Count(u64); +impl Accumulator for Count { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + self.0 += 1; + Ok(()) + } + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.0 += other.0; + Ok(()) + } +} + +let workbook = Workbook::open("big.xlsx")?; +let rows = workbook.sheet(0)?.aggregate(1, 0, || Count(0))?; +# Ok::<(), excelreader::Error>(()) +``` + +A cell in these callbacks carries text, a number, or both. `as_bytes`/`as_str` return only the +stored text, so they are empty for a cell that carries only a number; `text()` formats that number +itself (shortest round-trip form) and `number()` returns it directly, or parses the text when there +is no number. A positive `Err(code)` comes back as `ErrorKind::Callback(code)`, and a panic is +resumed on the calling thread. Calling other methods on the workbook from inside the callback is +allowed. + +`Sheet::aggregate` seeds an accumulator per chunk of sheet data, not per thread, and may drop a +chunk's accumulator without combining it. XLS, table views and CSV opened from a `Source` or stream +read sequentially. + ## Bounds and panics `TableView::get` returns `Option` and is `None` outside `0..len()`. The accessors the derive's @@ -429,22 +496,6 @@ generated code reads columns through panic on an out-of-range row, a column type match the binding, or a string the native library returned as non-UTF-8 - each is a contract violation rather than recoverable input. -## Parallel aggregation on macOS - -`aggregate_csv` over a source large enough to partition (256 KB and up) can crash the process on -macOS arm64. The fault is inside the native runtime's signal handling, not in this crate: a .NET -thread resumes at address 0 after a signal the runtime sent it, which ends the process with no -message. It happens with any `degree_of_parallelism` above 1, and the same runtime built into a -plain C host does not show it, so a Rust host appears to make it far more likely. - -`Sheet::parse` and `Sheet::to_arrow` with a `degree_of_parallelism` above 1 fan out on XLSX and -XLSB too; this has not been tested on macOS, so treat them the same way. - -Until it is fixed upstream, pass a `degree_of_parallelism` of 1 on macOS for a sequential run, or -keep sources under the 256 KB partitioning floor, where the library never fans out. Linux and -Windows are unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for -this reason. - ## Build notes `build.rs` downloads the native `ExcelReader.Native` binary matching your target from the crate's diff --git a/rust/excelreader/excelreader.def b/rust/excelreader/excelreader.def index 6bd570d3..454ceb9e 100644 --- a/rust/excelreader/excelreader.def +++ b/rust/excelreader/excelreader.def @@ -15,6 +15,7 @@ EXPORTS xl_rows_close xl_rows_next xl_rows_next_view + xl_rows_foreach xl_rows_read_all_decoded xl_rows_read_all_blob xl_free_rows @@ -50,5 +51,6 @@ EXPORTS xl_write_handle_bytes xl_csv_aggregate_file xl_csv_aggregate_memory + xl_aggregate xl_last_error xl_last_error_ptr diff --git a/rust/excelreader/src/aggregate.rs b/rust/excelreader/src/aggregate.rs index 5e76d743..c54a03da 100644 --- a/rust/excelreader/src/aggregate.rs +++ b/rust/excelreader/src/aggregate.rs @@ -7,7 +7,7 @@ //! can unwind into the library, and resumes that panic on the calling thread once the run has //! finished freeing every state. -use crate::ffi::{self, XlCsvAggregation, XlRow, XL_OK}; +use crate::ffi::{self, XlAggregation, XlCellView, XlCsvAggregation, XlRow, XlWorkbook, XL_OK}; use crate::workbook::{self, len_i32, path_str}; use crate::{Error, ErrorKind, OpenOptions, RowRef}; use std::any::Any; @@ -25,7 +25,7 @@ use std::sync::Mutex; /// the two by status; it keys on whether the panic slot actually holds a payload, so an ordinary /// `Err(i32::MAX)` from `accumulate` or `combine` still surfaces as a normal [`Error`] rather than /// being resumed as a panic. -const PANIC_STATUS: i32 = i32::MAX; +pub(crate) const PANIC_STATUS: i32 = i32::MAX; /// What each seeded accumulator is boxed in. The trailing byte keeps the allocation non-zero-sized, /// so `seed` hands the library a distinct pointer per partition even when `A` is zero-sized, as the @@ -59,6 +59,35 @@ pub trait CsvAccumulator: Send { fn combine(&mut self, other: &mut Self) -> Result<(), i32>; } +/// A fold over the rows of any sheet, one instance per chunk, driven by +/// [`Sheet::aggregate`](crate::Sheet::aggregate). +/// +/// `combine` takes `&mut Self` rather than `Self` because the library, not the implementation, owns +/// `other`: it frees that state through `free_state` once `combine` returns. Drain `other`, do not +/// consume it. +/// +/// `accumulate` runs concurrently on worker threads, each on its own instance, which is why `Send` +/// is required and `Sync` is not. An instance is seeded once per chunk of sheet data, not once per +/// thread, and a chunk can be accumulated and then dropped without ever being combined. A nonzero +/// code from `accumulate` fails only that chunk: if the library later re-reads it, the failure is +/// discarded with it and the run continues; otherwise the run ends with that code. A nonzero code +/// from `combine`, or a panic from the seed closure, always ends the run. However it ends, the code +/// is returned to the caller verbatim as [`ErrorKind::Callback`]; use positive codes, since every +/// code the library itself returns is `<= 0`. +/// Sibling workers are never stopped, so both `accumulate` and `combine` must tolerate being called +/// again after returning an error. +/// +/// XLS, table views and CSV opened from a [`Source`](crate::Source) or stream read sequentially, +/// with the same result. +pub trait Accumulator: Send { + /// Folds one row in. The row's cells borrow the library's buffers for this call only; a cell + /// may carry text, a number, or both. + fn accumulate(&mut self, row: RowRef<'_>) -> Result<(), i32>; + + /// Folds `other`'s contents into `self`, leaving `other` for the library to free. + fn combine(&mut self, other: &mut Self) -> Result<(), i32>; +} + /// What [`aggregate_csv`] reads: a CSV file, or CSV bytes already in memory. A `&str` converts to a /// path, never to CSV text - pass `text.as_bytes()` for that. #[derive(Clone, Copy, Debug)] @@ -126,7 +155,6 @@ impl Shared<'_, A, F> { /// `user_data` is the `Shared` this crate handed to `xl_csv_aggregate_*`, alive for the whole call. unsafe extern "C" fn seed_shim(out_state: *mut *mut c_void, user_data: *mut c_void) -> i32 where - A: CsvAccumulator, F: Fn() -> A, { let shared = unsafe { &*(user_data as *const Shared<'_, A, F>) }; @@ -194,7 +222,6 @@ where /// `user_data` is the `Shared` this crate handed to `xl_csv_aggregate_*`, alive for the whole call. unsafe extern "C" fn free_state_shim(state: *mut c_void, user_data: *mut c_void) where - A: CsvAccumulator, F: Fn() -> A, { let shared = unsafe { &*(user_data as *const Shared<'_, A, F>) }; @@ -205,6 +232,94 @@ where } } +/// # Safety +/// `state` is a state a previous `seed_shim` boxed, and `cells` is valid for this call only. +unsafe extern "C" fn view_accumulate_shim( + state: *mut c_void, + cells: *const XlCellView, + count: i32, + user_data: *mut c_void, +) -> i32 +where + A: Accumulator, + F: Fn() -> A, +{ + let shared = unsafe { &*(user_data as *const Shared<'_, A, F>) }; + let accumulator = unsafe { &mut (*(state as *mut State)).value }; + match catch_unwind(AssertUnwindSafe(|| { + accumulator.accumulate(unsafe { RowRef::from_views(cells, count) }) + })) { + Ok(Ok(())) => XL_OK, + Ok(Err(code)) => code, + Err(payload) => shared.store_panic(payload), + } +} + +/// # Safety +/// `acc` and `next` are two distinct states boxed by `seed_shim`; `next` stays the library's to free. +unsafe extern "C" fn view_combine_shim( + acc: *mut c_void, + next: *mut c_void, + user_data: *mut c_void, +) -> i32 +where + A: Accumulator, + F: Fn() -> A, +{ + let shared = unsafe { &*(user_data as *const Shared<'_, A, F>) }; + let accumulator = unsafe { &mut (*(acc as *mut State)).value }; + let other = unsafe { &mut (*(next as *mut State)).value }; + match catch_unwind(AssertUnwindSafe(|| accumulator.combine(other))) { + Ok(Ok(())) => XL_OK, + Ok(Err(code)) => code, + Err(payload) => shared.store_panic(payload), + } +} + +fn raw_view_aggregation(shared: &Shared<'_, A, F>) -> XlAggregation +where + A: Accumulator, + F: Fn() -> A, +{ + XlAggregation { + struct_size: std::mem::size_of::() as i32, + seed: Some(seed_shim::), + accumulate: Some(view_accumulate_shim::), + combine: Some(view_combine_shim::), + free_state: Some(free_state_shim::), + user_data: shared as *const Shared<'_, A, F> as *mut c_void, + } +} + +/// The body of [`Sheet::aggregate`](crate::Sheet::aggregate). +pub(crate) fn aggregate_sheet( + handle: *mut XlWorkbook, + sheet: i32, + header_row: i32, + degree_of_parallelism: i32, + seed: F, +) -> Result +where + A: Accumulator, + F: Fn() -> A + Sync, +{ + workbook::check_abi_version()?; + let shared = shared::(&seed); + let agg = raw_view_aggregation(&shared); + let mut state: *mut c_void = std::ptr::null_mut(); + let status = unsafe { + ffi::xl_aggregate( + handle, + sheet, + &agg, + header_row, + degree_of_parallelism, + &mut state, + ) + }; + finish(shared, status, state) +} + fn raw_aggregation(shared: &Shared<'_, A, F>) -> XlCsvAggregation where A: CsvAccumulator, @@ -321,7 +436,6 @@ where /// crossed a foreign frame. fn finish(shared: Shared<'_, A, F>, status: i32, state: *mut c_void) -> Result where - A: CsvAccumulator, F: Fn() -> A, { let winner = if state.is_null() { @@ -354,7 +468,7 @@ where /// Library statuses (all `<= 0`) carry a `xl_last_error_ptr` message; a positive status is a code one /// of the caller's own callbacks returned, which the library passes through untouched and has no /// message for. -fn status_error(status: i32) -> Error { +pub(crate) fn status_error(status: i32) -> Error { if status > 0 { Error::new( ErrorKind::Callback(status), diff --git a/rust/excelreader/src/ffi.rs b/rust/excelreader/src/ffi.rs index 8cdc9fa7..9b5c8a43 100644 --- a/rust/excelreader/src/ffi.rs +++ b/rust/excelreader/src/ffi.rs @@ -122,7 +122,7 @@ pub struct XlStream { /// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with /// `repr(C)` the 4 bytes of padding after `sheet_name_len` land exactly where a C compiler puts -/// them, giving the 32-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. +/// them, giving the 40-byte x64 layout `tests/ExcelReader.NativeSmoke/smoke.c` static-asserts. /// Build one through `WriteOptions` rather than by hand. #[repr(C)] #[derive(Clone, Copy, Debug)] @@ -134,6 +134,8 @@ pub struct XlWriteOptions { pub csv_quote: i32, pub date1904: i32, pub use_shared_strings: i32, + pub degree_of_parallelism: i32, + pub batch_rows: i32, } #[repr(C)] @@ -205,6 +207,33 @@ pub struct XlRow { pub cells: *mut XlRowCell, } +/// Mirrors `xl_cell_view`. `value` is NULL when `value_len` is 0, and `number` is meaningful only +/// when `has_number` is non-zero; a cell carries text, a number, or both. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct XlCellView { + pub column: i32, + pub cell_type: i32, + pub value_len: i32, + pub has_number: i32, + pub value: *const u8, + pub number: f64, +} + +/// Mirrors `xl_aggregation`: the `xl_csv_aggregation` layout with an `xl_cell_view` accumulate. +/// Same threading rules as `XlCsvAggregation`; build one through `Sheet::aggregate`. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XlAggregation { + pub struct_size: i32, + pub seed: Option i32>, + pub accumulate: + Option i32>, + pub combine: Option i32>, + pub free_state: Option, + pub user_data: *mut c_void, +} + /// Mirrors `xl_csv_aggregation`. Every callback runs on a library worker thread except /// `free_state`, which runs on the calling thread after the run has finished; none may let a /// panic escape into the library - build one through `aggregate` rather than by hand. @@ -477,6 +506,27 @@ extern "C" { out_state: *mut *mut c_void, ) -> c_int; + /// Calls `callback` once per row of `sheet` on the calling thread, with cells borrowed for the + /// call only. A non-`XL_OK` return stops the walk and is returned verbatim. + pub fn xl_rows_foreach( + handle: *mut XlWorkbook, + sheet: i32, + callback: Option i32>, + user_data: *mut c_void, + ) -> c_int; + + /// Folds `sheet` into one caller-owned state across several threads, like + /// `xl_csv_aggregate_memory` over any format. See the contract above `xl_aggregate` in + /// `excelreader.h`. + pub fn xl_aggregate( + handle: *mut XlWorkbook, + sheet: i32, + agg: *const XlAggregation, + header_row: i32, + degree_of_parallelism: i32, + out_state: *mut *mut c_void, + ) -> c_int; + pub fn xl_last_error_ptr(out_len: *mut i32) -> *const u8; } @@ -492,4 +542,13 @@ mod tests { "the linked native library speaks a different ABI revision than this crate" ); } + + #[test] + fn view_structs_match_the_c_layout() { + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::offset_of!(XlCellView, value), 16); + assert_eq!(std::mem::offset_of!(XlCellView, number), 24); + assert_eq!(std::mem::size_of::(), 48); + assert_eq!(std::mem::offset_of!(XlAggregation, accumulate), 16); + } } diff --git a/rust/excelreader/src/foreach.rs b/rust/excelreader/src/foreach.rs new file mode 100644 index 00000000..98ce92af --- /dev/null +++ b/rust/excelreader/src/foreach.rs @@ -0,0 +1,68 @@ +//! Push-style row reads over `xl_rows_foreach`. +//! +//! The library calls back on the calling thread, so the closure needs no `Send` bound and the +//! panic slot no lock. A panic is caught before it can unwind into the library and resumed once +//! `xl_rows_foreach` has returned. + +use crate::aggregate::{status_error, PANIC_STATUS}; +use crate::ffi::{self, XlCellView, XlWorkbook, XL_OK}; +use crate::{Error, RowRef}; +use std::any::Any; +use std::ffi::c_void; +use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; + +struct Walk<'f, F> { + f: &'f mut F, + panic: Option>, +} + +/// # Safety +/// `user_data` is the `Walk` handed to `xl_rows_foreach`, and `cells` is valid for this call only. +unsafe extern "C" fn row_shim( + cells: *const XlCellView, + count: i32, + user_data: *mut c_void, +) -> i32 +where + F: FnMut(RowRef<'_>) -> Result<(), i32>, +{ + let walk = unsafe { &mut *(user_data as *mut Walk<'_, F>) }; + match catch_unwind(AssertUnwindSafe(|| { + (walk.f)(unsafe { RowRef::from_views(cells, count) }) + })) { + Ok(Ok(())) => XL_OK, + Ok(Err(code)) => code, + Err(payload) => { + walk.panic = Some(payload); + PANIC_STATUS + } + } +} + +pub(crate) fn for_each_row(handle: *mut XlWorkbook, sheet: i32, mut f: F) -> Result<(), Error> +where + F: FnMut(RowRef<'_>) -> Result<(), i32>, +{ + crate::workbook::check_abi_version()?; + let mut walk = Walk { + f: &mut f, + panic: None, + }; + let status = unsafe { + ffi::xl_rows_foreach( + handle, + sheet, + Some(row_shim::), + &mut walk as *mut Walk<'_, F> as *mut c_void, + ) + }; + if status == PANIC_STATUS { + if let Some(payload) = walk.panic.take() { + resume_unwind(payload); + } + } + if status != XL_OK { + return Err(status_error(status)); + } + Ok(()) +} diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 91a60ce8..d7804774 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -7,6 +7,7 @@ mod aggregate; mod error; pub(crate) mod ffi; +mod foreach; mod options; mod rows; mod source; @@ -18,7 +19,7 @@ mod writer_handle; #[cfg(feature = "arrow")] mod arrow; -pub use aggregate::{aggregate_csv, CsvAccumulator, CsvInput}; +pub use aggregate::{aggregate_csv, Accumulator, CsvAccumulator, CsvInput}; pub use error::{Error, ErrorKind}; pub use options::{Format, OpenOptions, ReadOptions, WriteOptions}; pub use rows::{AllRows, CellIter, CellRef, CellType, RowCursor, RowRef}; diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 417b99a0..f252f416 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -343,6 +343,8 @@ pub struct WriteOptions { csv_quote: Option, date1904: Option, use_shared_strings: Option, + degree_of_parallelism: Option, + batch_rows: Option, } impl WriteOptions { @@ -378,6 +380,12 @@ impl WriteOptions { /// XLSX/XLSB only; ignored for XLS and CSV. Shrinks files with many repeated strings, at /// the cost of a string table. Library default is `false`. use_shared_strings: bool, + /// XLSX/XLSB only. Batches formatted and compressed at once: `0` uses every processor, + /// `1` writes serially. Unset is serial. With shared strings on, parallel runs give the + /// same content in files that differ byte for byte. + degree_of_parallelism: i32, + /// XLSX/XLSB only. Rows per parallel batch; `0` or unset is the library default (8192). + batch_rows: i32, } /// The `XL_FORMAT_*` to write: the explicit format, else one inferred from `path`. With no path @@ -407,6 +415,8 @@ impl WriteOptions { csv_quote: opt_number(self.csv_quote), date1904: opt_state(self.date1904), use_shared_strings: opt_state(self.use_shared_strings), + degree_of_parallelism: self.degree_of_parallelism.unwrap_or(1), + batch_rows: self.batch_rows.unwrap_or(0), } } } @@ -540,6 +550,18 @@ mod tests { assert_eq!(raw.use_shared_strings, XL_OPT_TRUE); } + #[test] + fn write_options_default_to_a_serial_write() { + let raw = WriteOptions::new().to_raw(); + assert_eq!(raw.struct_size, 40); + assert_eq!(raw.degree_of_parallelism, 1); + assert_eq!(raw.batch_rows, 0); + + let raw = WriteOptions::new().degree_of_parallelism(0).batch_rows(2).to_raw(); + assert_eq!(raw.degree_of_parallelism, 0); + assert_eq!(raw.batch_rows, 2); + } + #[test] fn write_format_is_explicit_or_inferred_from_the_extension() { let auto = WriteOptions::new(); @@ -570,7 +592,7 @@ mod tests { /// side garbage where it expects a length and a pointer. #[test] fn write_options_match_the_c_struct_size() { - assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 40); } #[test] diff --git a/rust/excelreader/src/rows.rs b/rust/excelreader/src/rows.rs index c075528f..257b9734 100644 --- a/rust/excelreader/src/rows.rs +++ b/rust/excelreader/src/rows.rs @@ -1,8 +1,10 @@ //! Row-at-a-time and whole-sheet decoded reads over the C ABI's row APIs. +use std::borrow::Cow; + use crate::ffi::{ - self, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, XL_CELL_FORMULA, - XL_CELL_NUMBER, XL_CELL_STRING, + self, XlCellView, XlRowCell, XL_CELL_BOOL, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR, + XL_CELL_FORMULA, XL_CELL_NUMBER, XL_CELL_STRING, }; use crate::{Error, ErrorKind}; @@ -43,17 +45,20 @@ pub struct CellRef<'a> { pub column: i32, pub cell_type: CellType, value: &'a [u8], + number: Option, } impl<'a> CellRef<'a> { - /// The raw UTF-8 bytes as stored. A `Date` cell carries an Excel serial number as text. + /// The raw UTF-8 bytes as stored. A `Date` cell carries an Excel serial number as text. Empty + /// for a cell that carries only a number - use `text()` or `number()`. #[must_use] pub fn as_bytes(&self) -> &'a [u8] { self.value } - /// The value as a string. Fails when the bytes are not valid UTF-8, which a well-formed - /// workbook never produces. + /// The stored text as a string; empty for a cell that carries only a number - use `text()` or + /// `number()`. Fails when the bytes are not valid UTF-8, which a well-formed workbook never + /// produces. pub fn as_str(&self) -> Result<&'a str, Error> { std::str::from_utf8(self.value).map_err(|err| { Error::new( @@ -62,15 +67,35 @@ impl<'a> CellRef<'a> { ) }) } + + /// The cell's numeric value: the library's own double when the cell carries one, else its + /// text parsed as `f64`. Dates are Excel serials. + #[must_use] + pub fn number(&self) -> Option { + self.number.or_else(|| self.as_str().ok()?.parse().ok()) + } + + /// The cell's text, or its number in shortest round-trip form when it carries only a number. + /// That formatting is this crate's, so it can differ from what `xl_rows_next` writes. + pub fn text(&self) -> Result, Error> { + if !self.value.is_empty() { + return self.as_str().map(Cow::Borrowed); + } + Ok(match self.number { + Some(number) => Cow::Owned(number.to_string()), + None => Cow::Borrowed(""), + }) + } } /// Where a `RowRef`'s cells live. Blob rows come from `xl_rows_next`/`xl_rows_read_all_blob`, -/// decoded rows from a CSV aggregation callback. +/// decoded rows from a CSV aggregation callback, views from `xl_rows_foreach`/`xl_aggregate`. #[derive(Debug, Clone, Copy)] enum RowBacking<'a> { /// The bytes AFTER the leading `int32 cell_count`. Blob(&'a [u8]), Decoded(&'a [XlRowCell]), + Views(&'a [XlCellView]), } /// One row, borrowing from whichever buffer produced it. @@ -112,6 +137,24 @@ impl<'a> RowRef<'a> { } } + /// Wraps the cells handed to an `xl_rows_foreach`/`xl_aggregate` callback. + /// + /// # Safety + /// `cells` must point to `count` initialized `XlCellView` values whose `value` pointers stay + /// valid for `'a`. + pub(crate) unsafe fn from_views(cells: *const XlCellView, count: i32) -> RowRef<'a> { + let len = if count > 0 { count as usize } else { 0 }; + let slice = if len == 0 || cells.is_null() { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(cells, len) } + }; + RowRef { + backing: RowBacking::Views(slice), + len: slice.len(), + } + } + #[must_use] pub fn len(&self) -> usize { self.len @@ -131,6 +174,7 @@ impl<'a> RowRef<'a> { } match self.backing { RowBacking::Decoded(cells) => cell_from_decoded(&cells[index]), + RowBacking::Views(cells) => cell_from_view(&cells[index]), RowBacking::Blob(_) => self.iter().nth(index), } } @@ -177,6 +221,11 @@ impl<'a> Iterator for CellIter<'a> { self.index += 1; Some(cell) } + RowBacking::Views(cells) => { + let cell = cell_from_view(&cells[self.index])?; + self.index += 1; + Some(cell) + } RowBacking::Blob(blob) => { let (cell, next) = cell_from_blob(blob, self.offset)?; self.offset = next; @@ -217,6 +266,7 @@ fn cell_from_blob(blob: &[u8], offset: usize) -> Option<(CellRef<'_>, usize)> { column, cell_type, value, + number: None, }, end, )) @@ -233,6 +283,22 @@ fn cell_from_decoded(raw: &XlRowCell) -> Option> { column: raw.column, cell_type, value, + number: None, + }) +} + +fn cell_from_view(raw: &XlCellView) -> Option> { + let cell_type = CellType::from_raw(raw.cell_type)?; + let value = if raw.value.is_null() || raw.value_len <= 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(raw.value, raw.value_len as usize) } + }; + Some(CellRef { + column: raw.column, + cell_type, + value, + number: (raw.has_number != 0).then_some(raw.number), }) } @@ -438,6 +504,7 @@ fn malformed_all_rows_blob() -> Error { #[cfg(test)] mod tests { use super::*; + use crate::ffi::XlCellView; /// Builds a row blob in the wire format documented on `xl_rows_next`: /// `int32 cell_count`, then per cell `int32 column, int32 type, int32 value_len, bytes`. @@ -503,6 +570,71 @@ mod tests { assert!(row.get(0).is_none()); } + fn view(column: i32, cell_type: i32, text: &[u8], number: Option) -> XlCellView { + XlCellView { + column, + cell_type, + value_len: text.len() as i32, + has_number: number.is_some() as i32, + value: if text.is_empty() { + std::ptr::null() + } else { + text.as_ptr() + }, + number: number.unwrap_or(0.0), + } + } + + #[test] + fn view_number_only_cell_has_empty_bytes_and_formats_its_number() { + let cells = [view(3, XL_CELL_NUMBER, b"", Some(42.5))]; + let row = unsafe { RowRef::from_views(cells.as_ptr(), 1) }; + let cell = row.get(0).unwrap(); + assert_eq!(cell.column, 3); + assert_eq!(cell.as_bytes(), b""); + assert_eq!(cell.number(), Some(42.5)); + assert_eq!(cell.text().unwrap(), "42.5"); + } + + #[test] + fn view_text_and_number_cell_prefers_the_raw_number_and_keeps_the_text() { + let cells = [view(0, XL_CELL_DATE, b"45000", Some(45000.25))]; + let row = unsafe { RowRef::from_views(cells.as_ptr(), 1) }; + let cell = row.get(0).unwrap(); + assert_eq!(cell.as_str().unwrap(), "45000"); + assert_eq!(cell.text().unwrap(), "45000"); + assert_eq!(cell.number(), Some(45000.25)); + } + + #[test] + fn text_only_cell_parses_its_number() { + let cells = [ + view(0, XL_CELL_NUMBER, b"12.5", None), + view(1, XL_CELL_STRING, b"abc", None), + ]; + let row = unsafe { RowRef::from_views(cells.as_ptr(), 2) }; + let cells: Vec> = row.iter().collect(); + assert_eq!(cells[0].number(), Some(12.5)); + assert_eq!(cells[0].text().unwrap(), "12.5"); + assert_eq!(cells[1].number(), None); + } + + #[test] + fn empty_view_row_from_null_is_empty() { + let row = unsafe { RowRef::from_views(std::ptr::null(), 0) }; + assert!(row.is_empty()); + assert_eq!(row.iter().count(), 0); + } + + #[test] + fn blob_cells_report_no_raw_number() { + let bytes = blob(&[(0, XL_CELL_NUMBER, "7")]); + let row = RowRef::from_blob(&bytes).unwrap(); + let cell = row.get(0).unwrap(); + assert_eq!(cell.number(), Some(7.0)); + assert_eq!(cell.text().unwrap(), "7"); + } + #[test] fn unknown_cell_type_is_none() { assert_eq!(CellType::from_raw(99), None); diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index e7d487e0..4071abf6 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -3,7 +3,7 @@ use crate::ffi::{ XL_BUFFER_TOO_SMALL, XL_EOF, XL_OK, }; use crate::{ - AllRows, Date, Error, ErrorKind, OpenOptions, ReadOptions, RowCursor, Time, Timestamp, + AllRows, Date, Error, ErrorKind, OpenOptions, ReadOptions, RowCursor, RowRef, Time, Timestamp, }; use std::marker::PhantomData; use std::path::Path; @@ -336,6 +336,52 @@ impl<'w> Sheet<'w> { RowCursor::open(self.workbook.handle, self.index) } + /// Calls `f` once per row on the calling thread, without copying the row into a cursor buffer. + /// + /// Each row's cells borrow the library's buffers for that call only. A cell may carry text, a + /// number, or both: [`CellRef::text`](crate::CellRef::text) and + /// [`CellRef::number`](crate::CellRef::number) cover all three. A nonzero `Err(code)` stops the + /// walk (`Err(0)` is `XL_OK` and does not); a positive code surfaces as + /// [`ErrorKind::Callback`](crate::ErrorKind::Callback). Other calls + /// on this workbook are allowed from inside `f`. A panic in `f` stops the walk and is resumed + /// here. + pub fn for_each_row(&self, f: F) -> Result<(), Error> + where + F: FnMut(RowRef<'_>) -> Result<(), i32>, + { + crate::foreach::for_each_row(self.workbook.handle, self.index, f) + } + + /// Folds this sheet into one `A` across several threads. + /// + /// `header_row` is 1-based, `0` meaning no header. `degree_of_parallelism` is `0` for every + /// processor, `1` sequential, `n` up to `n` threads. The CSV dialect, password and limits come + /// from the options the workbook was opened with. + /// + /// `seed` is called once per chunk, on worker threads; see [`Accumulator`](crate::Accumulator) + /// for the contract. `seed`, `accumulate` and `combine` never run on the calling thread; each + /// accumulator is dropped on the calling thread once the run has finished. The call blocks + /// until the run finishes, so `seed` may borrow locals. A panic inside any callback aborts the + /// run and is resumed here once the library has freed every state. + pub fn aggregate( + &self, + header_row: i32, + degree_of_parallelism: i32, + seed: F, + ) -> Result + where + A: crate::Accumulator, + F: Fn() -> A + Sync, + { + crate::aggregate::aggregate_sheet( + self.workbook.handle, + self.index, + header_row, + degree_of_parallelism, + seed, + ) + } + /// Every row of this sheet, read in one native call into a buffer the result owns outright. pub fn read_all(&self) -> Result { let cursor = self.rows()?; diff --git a/rust/excelreader/tests/aggregate.rs b/rust/excelreader/tests/aggregate.rs index 96237caf..fc4c66ca 100644 --- a/rust/excelreader/tests/aggregate.rs +++ b/rust/excelreader/tests/aggregate.rs @@ -67,10 +67,6 @@ impl Drop for TempDir { } #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn group_by_sum_over_a_file_is_partitioned_and_exact() { let dir = TempDir::new("file"); let path = dir.0.join("groups.csv"); @@ -94,10 +90,6 @@ fn group_by_sum_over_a_file_is_partitioned_and_exact() { } #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn group_by_sum_over_memory_is_partitioned_and_exact() { let (text, expected) = group_fixture(); let actual = aggregate_csv( @@ -118,10 +110,6 @@ fn group_by_sum_over_memory_is_partitioned_and_exact() { } #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn a_failing_accumulate_returns_its_own_code() { #[derive(Debug)] struct AlwaysFails; @@ -149,10 +137,6 @@ fn a_failing_accumulate_returns_its_own_code() { /// case can violate is asserted separately, in /// `a_zero_sized_accumulator_never_aliases_two_states_in_combine`. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn a_zero_sized_accumulator_gets_a_distinct_state_per_partition() { use std::sync::atomic::{AtomicI64, Ordering}; static ROWS: AtomicI64 = AtomicI64::new(0); @@ -178,10 +162,6 @@ fn a_zero_sized_accumulator_gets_a_distinct_state_per_partition() { /// distinct states". A zero-sized accumulator is the case that breaks it: `Box::into_raw` of a ZST /// returns the same dangling address every time, so `acc` and `next` are one and the same object. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn a_zero_sized_accumulator_never_aliases_two_states_in_combine() { use std::sync::atomic::{AtomicUsize, Ordering}; static COMBINES: AtomicUsize = AtomicUsize::new(0); @@ -218,10 +198,6 @@ fn a_zero_sized_accumulator_never_aliases_two_states_in_combine() { /// Every accumulator the library seeds must be dropped exactly once. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn every_seeded_accumulator_is_dropped_exactly_once() { use std::sync::atomic::{AtomicUsize, Ordering}; static SEEDED: AtomicUsize = AtomicUsize::new(0); @@ -262,10 +238,6 @@ fn every_seeded_accumulator_is_dropped_exactly_once() { /// Quoted records that span lines are what make a guessed partition boundary wrong, so the run must /// still count each record exactly once. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn quoted_multi_line_records_straddling_chunks_are_counted_once() { struct Rows(i64); impl CsvAccumulator for Rows { @@ -327,10 +299,6 @@ impl CsvAccumulator for RowCounter { /// Coverage for a macOS/arm64-only SIGSEGV after a concurrent (`degree_of_parallelism` > 1) run, /// seen only with a symbol-stripped dylib; see `StripSymbols` in ExcelReader.Native.csproj. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn a_call_after_a_partitioned_run_still_succeeds() { let (text, _) = group_fixture(); let first = @@ -346,10 +314,6 @@ fn a_call_after_a_partitioned_run_still_succeeds() { /// the true start is confirmed, and the re-read succeeds without panicking - the stale panic from /// the discarded first attempt must not resume and kill an otherwise-successful run. #[test] -#[cfg_attr( - target_os = "macos", - ignore = "crashes the native runtime on macOS arm64; see \"Parallel aggregation on macOS\" in the crate README" -)] fn a_panic_from_a_discarded_reread_does_not_resume() { struct PanicsOnGarbledRows(i64); impl CsvAccumulator for PanicsOnGarbledRows { diff --git a/rust/excelreader/tests/aggregate_sheet.rs b/rust/excelreader/tests/aggregate_sheet.rs new file mode 100644 index 00000000..e4403760 --- /dev/null +++ b/rust/excelreader/tests/aggregate_sheet.rs @@ -0,0 +1,193 @@ +//! Integration coverage for `Sheet::aggregate` over `xl_aggregate`. + +use excelreader::{ + Accumulator, ErrorKind, Format, OpenOptions, RowRef, Workbook, WriteOptions, WriterHandle, +}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; +use std::sync::Arc; + +fn generated(format: Format, rows: i64) -> Workbook { + let mut bytes = Vec::new(); + let options = WriteOptions::new().format(format); + let mut writer = WriterHandle::new(&mut bytes, &options).expect("open writer"); + writer.start_sheet("S1").expect("start sheet"); + writer.start_row().expect("start row"); + writer.write_str(Some("id")).expect("write"); + writer.write_str(Some("label")).expect("write"); + writer.end_row().expect("end row"); + for i in 1..=rows { + writer.start_row().expect("start row"); + writer.write_i64(Some(i)).expect("write"); + writer + .write_str(Some(&format!("filler-value-{i}"))) + .expect("write"); + writer.end_row().expect("end row"); + } + writer.end_sheet().expect("end sheet"); + writer.finish().expect("finish"); + OpenOptions::new() + .format(format) + .open_bytes(&bytes) + .expect("open") +} + +fn first_number(row: &RowRef<'_>) -> Result { + Ok(row.get(0).ok_or(1)?.number().ok_or(1)? as i64) +} + +#[derive(Default)] +struct Totals { + sum: i64, + rows: i64, + combines: u64, +} + +impl Accumulator for Totals { + fn accumulate(&mut self, row: RowRef<'_>) -> Result<(), i32> { + self.sum += first_number(&row)?; + self.rows += 1; + Ok(()) + } + + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.sum += other.sum; + self.rows += other.rows; + self.combines += other.combines + 1; + Ok(()) + } +} + +#[test] +fn aggregate_sums_every_data_row_once_on_every_format() { + for format in [Format::Xlsx, Format::Xlsb, Format::Xls, Format::Csv] { + let workbook = generated(format, 60_000); + let totals = workbook + .sheet(0) + .unwrap() + .aggregate(1, 4, Totals::default) + .unwrap(); + assert_eq!(totals.rows, 60_000, "{format:?}"); + assert_eq!(totals.sum, 60_000 * 60_001 / 2, "{format:?}"); + if matches!(format, Format::Xlsx | Format::Xlsb) { + assert!(totals.combines >= 1, "{format:?} must partition"); + } + } +} + +struct Live { + live: Arc, +} + +impl Drop for Live { + fn drop(&mut self) { + self.live.fetch_sub(1, Ordering::SeqCst); + } +} + +impl Accumulator for Live { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + Ok(()) + } + + fn combine(&mut self, _other: &mut Self) -> Result<(), i32> { + Ok(()) + } +} + +#[test] +fn every_seeded_accumulator_is_dropped_once() { + let workbook = generated(Format::Xlsx, 60_000); + let live = Arc::new(AtomicUsize::new(0)); + let result = workbook.sheet(0).unwrap().aggregate(1, 4, || { + live.fetch_add(1, Ordering::SeqCst); + Live { live: live.clone() } + }); + drop(result.unwrap()); + assert_eq!(live.load(Ordering::SeqCst), 0); +} + +struct Aborting<'a> { + seen: &'a AtomicI64, +} + +impl Accumulator for Aborting<'_> { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + if self.seen.fetch_add(1, Ordering::SeqCst) >= 20_000 { + return Err(9); + } + Ok(()) + } + + fn combine(&mut self, _other: &mut Self) -> Result<(), i32> { + Ok(()) + } +} + +#[test] +fn a_callback_code_aborts_with_callback_kind() { + let workbook = generated(Format::Xlsx, 60_000); + let seen = AtomicI64::new(0); + let result = workbook + .sheet(0) + .unwrap() + .aggregate(1, 4, || Aborting { seen: &seen }); + assert_eq!(result.err().unwrap().kind(), ErrorKind::Callback(9)); +} + +#[derive(Debug, Default)] +struct Panicking { + rows: i64, +} + +impl Accumulator for Panicking { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + self.rows += 1; + if self.rows == 100 { + panic!("boom"); + } + Ok(()) + } + + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.rows += other.rows; + Ok(()) + } +} + +#[test] +fn a_panic_in_accumulate_is_resumed() { + let workbook = generated(Format::Xlsb, 60_000); + let sheet = workbook.sheet(0).unwrap(); + let payload = catch_unwind(AssertUnwindSafe(|| { + sheet.aggregate(1, 4, Panicking::default) + })) + .expect_err("the panic must be resumed"); + assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom")); +} + +#[derive(Default)] +struct Count(i64); + +impl Accumulator for Count { + fn accumulate(&mut self, _row: RowRef<'_>) -> Result<(), i32> { + self.0 += 1; + Ok(()) + } + + fn combine(&mut self, other: &mut Self) -> Result<(), i32> { + self.0 += other.0; + Ok(()) + } +} + +#[test] +fn header_row_zero_aggregates_the_header_too() { + let workbook = generated(Format::Csv, 10); + let count = workbook + .sheet(0) + .unwrap() + .aggregate(0, 1, Count::default) + .unwrap(); + assert_eq!(count.0, 11); +} diff --git a/rust/excelreader/tests/for_each_row.rs b/rust/excelreader/tests/for_each_row.rs new file mode 100644 index 00000000..288d8baf --- /dev/null +++ b/rust/excelreader/tests/for_each_row.rs @@ -0,0 +1,166 @@ +use excelreader::{ + CellType, ErrorKind, Format, OpenOptions, RowRef, Workbook, WriteOptions, WriterHandle, +}; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +fn fixture(name: &str) -> String { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(|p| p.parent()) + .expect("repo root is two levels above the crate"); + root.join(name).to_string_lossy().into_owned() +} + +fn generated(format: Format) -> Workbook { + let mut bytes = Vec::new(); + let options = WriteOptions::new().format(format); + let mut writer = WriterHandle::new(&mut bytes, &options).expect("open writer"); + writer.start_sheet("S1").expect("start sheet"); + writer.start_row().expect("start row"); + writer.write_str(Some("id")).expect("write"); + writer.write_str(Some("label")).expect("write"); + writer.end_row().expect("end row"); + for i in 1..=200i64 { + writer.start_row().expect("start row"); + writer.write_i64(Some(i)).expect("write"); + writer + .write_str(Some(&format!("label-{i}"))) + .expect("write"); + writer.write_f64(Some(i as f64 * 1e20)).expect("write"); + writer.end_row().expect("end row"); + } + writer.end_sheet().expect("end sheet"); + writer.finish().expect("finish"); + OpenOptions::new() + .format(format) + .open_bytes(&bytes) + .expect("open") +} + +type Cell = (i32, CellType, Option, String); + +fn collect(row: RowRef<'_>) -> Vec { + row.iter() + .map(|cell| { + ( + cell.column, + cell.cell_type, + cell.number(), + cell.as_str().unwrap().to_owned(), + ) + }) + .collect() +} + +fn assert_same_as_cursor(workbook: &Workbook) { + let sheet = workbook.sheet(0).expect("sheet 0"); + let mut pushed: Vec> = Vec::new(); + sheet + .for_each_row(|row| { + pushed.push(collect(row)); + Ok(()) + }) + .expect("for_each_row"); + + let mut pulled: Vec> = Vec::new(); + let mut cursor = sheet.rows().expect("cursor"); + while let Some(row) = cursor.next_row() { + pulled.push(collect(row.expect("row"))); + } + + assert_eq!(pushed.len(), pulled.len()); + for (left, right) in pushed.iter().zip(&pulled) { + assert_eq!(left.len(), right.len()); + for (a, b) in left.iter().zip(right) { + assert_eq!((a.0, a.1), (b.0, b.1)); + if !a.3.is_empty() && !b.3.is_empty() { + assert_eq!(a.3, b.3); + } else { + assert_eq!(a.2, b.2); + } + } + } +} + +#[test] +fn for_each_row_matches_the_cursor_on_every_format() { + assert_same_as_cursor(&Workbook::open(fixture("RealExcel.xlsx")).expect("open")); + assert_same_as_cursor(&Workbook::open(fixture("RealExcel.xlsb")).expect("open")); + assert_same_as_cursor(&generated(Format::Xlsb)); + assert_same_as_cursor(&generated(Format::Xls)); + assert_same_as_cursor(&generated(Format::Csv)); +} + +#[test] +fn a_positive_callback_code_stops_the_walk() { + let workbook = Workbook::open(fixture("RealExcel.xlsx")).expect("open"); + let mut count = 0; + let result = workbook.sheet(0).expect("sheet 0").for_each_row(|_| { + count += 1; + if count == 3 { + Err(7) + } else { + Ok(()) + } + }); + assert_eq!(result.unwrap_err().kind(), ErrorKind::Callback(7)); + assert_eq!(count, 3); +} + +#[test] +fn a_non_positive_callback_code_is_an_error() { + let workbook = Workbook::open(fixture("RealExcel.xlsx")).expect("open"); + let result = workbook + .sheet(0) + .expect("sheet 0") + .for_each_row(|_| Err(-1)); + assert!(result.is_err()); +} + +#[test] +fn a_nested_call_inside_the_callback_works() { + let workbook = Workbook::open(fixture("RealExcel.xlsx")).expect("open"); + let sheet = workbook.sheet(0).expect("sheet 0"); + let name = sheet.name().expect("name"); + sheet + .for_each_row(|_| { + assert_eq!(sheet.name().expect("nested name"), name); + Ok(()) + }) + .expect("for_each_row"); +} + +#[test] +fn a_panic_is_resumed_and_the_workbook_stays_usable() { + let workbook = Workbook::open(fixture("RealExcel.xlsx")).expect("open"); + let sheet = workbook.sheet(0).expect("sheet 0"); + let payload = catch_unwind(AssertUnwindSafe(|| sheet.for_each_row(|_| panic!("boom")))) + .expect_err("the panic must be resumed"); + assert_eq!(payload.downcast_ref::<&str>(), Some(&"boom")); + assert!(sheet.for_each_row(|_| Ok(())).is_ok()); +} + +#[test] +fn number_only_cells_format_their_text() { + let workbook = Workbook::open(fixture("RealExcel.xlsb")).expect("open"); + let mut seen = 0; + workbook + .sheet(0) + .expect("sheet 0") + .for_each_row(|row| { + for cell in row.iter() { + if cell.as_bytes().is_empty() { + if let Some(number) = cell.number() { + assert_eq!(cell.text().unwrap(), number.to_string()); + seen += 1; + } + } + } + Ok(()) + }) + .expect("for_each_row"); + assert!( + seen > 0, + "the fixture must have a cell that carries only a number" + ); +} diff --git a/rust/excelreader/tests/write_typed.rs b/rust/excelreader/tests/write_typed.rs index da4aa26c..3baa0283 100644 --- a/rust/excelreader/tests/write_typed.rs +++ b/rust/excelreader/tests/write_typed.rs @@ -166,6 +166,65 @@ fn writing_to_a_writer_needs_an_explicit_format() { assert_eq!(error.kind(), ErrorKind::InvalidArgument); } +#[test] +fn write_columns_in_parallel_round_trips() { + let offsets = [0i32, 1, 2, 3, 4, 5, 6, 7]; + let blob = b"abcdefg"; + let inteiros = [0i64, 10, 20, 30, 40, 50, 60]; + let validity = [0b0111_1011u8]; + let columns = [ + Column::new( + Some("texto"), + ColumnData::Str { + offsets: &offsets, + data: blob, + }, + ), + Column::new(Some("inteiro"), ColumnData::I64(&inteiros)).validity(&validity), + ]; + + #[derive(Default, Debug, ExcelMapper)] + struct ParallelRow { + #[excel(name = "texto")] + texto: String, + #[excel(name = "inteiro")] + inteiro: Option, + } + + for format in [Format::Xlsx, Format::Xlsb] { + let mut bytes = Vec::new(); + let options = WriteOptions::new() + .format(format) + .degree_of_parallelism(4) + .batch_rows(2); + write_columns_to(&mut bytes, &columns, &options).expect("a parallel write must succeed"); + + let workbook = OpenOptions::new() + .format(format) + .open_bytes(&bytes) + .expect("the parallel output must open"); + let table = workbook + .sheet(0) + .expect("sheet 0") + .parse::(&ReadOptions::new()) + .expect("the parallel output must parse back"); + let rows: Vec<(String, Option)> = + table.iter().map(|row| (row.texto, row.inteiro)).collect(); + assert_eq!( + rows, + vec![ + ("a".to_string(), Some(0)), + ("b".to_string(), Some(10)), + ("c".to_string(), None), + ("d".to_string(), Some(30)), + ("e".to_string(), Some(40)), + ("f".to_string(), Some(50)), + ("g".to_string(), Some(60)), + ] + ); + } +} + #[test] fn write_columns_writes_nulls_from_the_validity_bitmap() { let nomes_offsets = [0i32, 3, 8, 8]; diff --git a/src/ExcelReader.Core/Crypto/CfbContainer.cs b/src/ExcelReader.Core/Crypto/CfbContainer.cs index 3450732b..a08dda89 100644 --- a/src/ExcelReader.Core/Crypto/CfbContainer.cs +++ b/src/ExcelReader.Core/Crypto/CfbContainer.cs @@ -466,6 +466,10 @@ internal static byte[] ReadMiniStream(ReadOnlySpan miniStream, ReadOnlySpa visited[sector] = true; sector = miniFat[sector]; } + if (written < result.Length) + { + throw new InvalidDataException("The OLE mini stream chain ended early."); + } return result; } diff --git a/src/ExcelReader.Core/ExcelEpoch.cs b/src/ExcelReader.Core/ExcelEpoch.cs index b9684df1..fe2c84f5 100644 --- a/src/ExcelReader.Core/ExcelEpoch.cs +++ b/src/ExcelReader.Core/ExcelEpoch.cs @@ -13,30 +13,27 @@ internal static double SerialToOADate(double serial, bool date1904) return isLessThan60 ? serial + 1.0 : serial; } - private static void ThrowIfNegative(double value, string paramName, string message) + internal static double OADateToSerial(double oadate, bool date1904) { - if (value >= 0.0) + if (!TryOADateToSerial(oadate, date1904, out double serial)) { - return; + throw new ArgumentOutOfRangeException(nameof(oadate), serial, "Dates before the workbook epoch cannot be written to Excel."); } - throw new ArgumentOutOfRangeException(paramName, value, message); + return serial; } - internal static double OADateToSerial(double oadate, bool date1904) + /// False for a date before the workbook epoch, which has no serial. + internal static bool TryOADateToSerial(double oadate, bool date1904, out double serial) { - double serial = oadate; if (date1904) { - serial -= 1462.0; - ThrowIfNegative(serial, nameof(oadate), "Dates before the workbook epoch cannot be written to Excel."); - return serial; + serial = oadate - 1462.0; } - if (oadate < 61.0) + else { - serial--; + serial = oadate < 61.0 ? oadate - 1.0 : oadate; } - ThrowIfNegative(serial, nameof(oadate), "Dates before the workbook epoch cannot be written to Excel."); - return serial; + return serial >= 0.0; } } } diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkPlan.cs b/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkPlan.cs index 7de52ef5..1aac8370 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkPlan.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkPlan.cs @@ -45,6 +45,11 @@ internal static CsvChunkPlan Create(long dataStart, long dataLength, int degreeO return new CsvChunkPlan(dataStart, dataStart + dataLength, chunkSize, count); } + internal CsvChunkPlan Restart() + { + return new CsvChunkPlan(_dataStart, _dataEnd, _chunkSize, _count); + } + internal bool TryTakeNext(out CsvChunk chunk) { int index = Interlocked.Increment(ref _next); diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkWorker.cs b/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkWorker.cs index a61b39ab..347cb8a5 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkWorker.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/CsvChunkWorker.cs @@ -2,6 +2,7 @@ using ExcelReader.Core.Parser.Internal; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Csv; +using ExcelReader.Core.Reader.Internal; using Microsoft.Win32.SafeHandles; namespace ExcelReader.Core.Parser.ParallelCsv @@ -97,7 +98,8 @@ internal CsvChunkResult(int index, List models, long actualStart, long resolv internal ExcelParseException? Failure { get; set; } - internal int FailureRowInChunk { get; set; } + /// Records read before the chunk ended or failed, blank ones included, so rows can be numbered. + internal int Records { get; set; } } internal static class CsvChunkWorker @@ -114,6 +116,7 @@ internal static ValueTask> ParseAsync( CsvReaderOptions readerOptions, ExcelParserOptions config, List models, + Utf8StringCache? contentCache, CancellationToken ct) { long start = confirmedStart ?? GuessStart(source, chunk, readerOptions.Quote); @@ -123,7 +126,7 @@ internal static ValueTask> ParseAsync( } CsvReaderOptions chunkOptions = readerOptions with { DetectEncodingFromByteOrderMark = start == 0 }; - return ParseFromAsync(source, chunk, start, models, map, info, chunkOptions, config, ct); + return ParseFromAsync(source, chunk, start, models, map, info, chunkOptions, config, contentCache, ct); } private static async ValueTask> ParseFromAsync( @@ -135,14 +138,15 @@ private static async ValueTask> ParseFromAsync( TypeMapInfo info, CsvReaderOptions options, ExcelParserOptions config, + Utf8StringCache? contentCache, CancellationToken ct) { var projector = new CsvRowProjector(info, map, config.Culture, config.ThrowOnParseFailure); - (long resolvedNextStart, ExcelParseException? failure, int failureRow) outcome; + (long resolvedNextStart, ExcelParseException? failure, int records) outcome; if (source.IsMemory) { - var rows = new CsvReader.Enumerator(source.SliceAt(start), options, ct); + var rows = new CsvReader.Enumerator(source.SliceAt(start), options, contentCache, ct); await using (rows.ConfigureAwait(false)) { outcome = await ConsumeAsync(rows, start, chunk, projector, models).ConfigureAwait(false); @@ -150,13 +154,13 @@ private static async ValueTask> ParseFromAsync( return new CsvChunkResult(chunk.Index, models, start, outcome.resolvedNextStart) { Failure = outcome.failure, - FailureRowInChunk = outcome.failureRow, + Records = outcome.records, }; } Stream partitionStream = source.OpenAt(start); try { - var outerRows = new CsvReader.Enumerator(partitionStream, options, ct); + var outerRows = new CsvReader.Enumerator(partitionStream, options, contentCache, ct); await using (outerRows.ConfigureAwait(false)) { outcome = await ConsumeAsync(outerRows, start, chunk, projector, models).ConfigureAwait(false); @@ -169,11 +173,11 @@ private static async ValueTask> ParseFromAsync( return new CsvChunkResult(chunk.Index, models, start, outcome.resolvedNextStart) { Failure = outcome.failure, - FailureRowInChunk = outcome.failureRow, + Records = outcome.records, }; } - private static async ValueTask<(long resolvedNextStart, ExcelParseException? failure, int failureRow)> ConsumeAsync( + private static async ValueTask<(long resolvedNextStart, ExcelParseException? failure, int records)> ConsumeAsync( CsvReader.Enumerator rows, long start, CsvChunk chunk, @@ -182,7 +186,7 @@ private static async ValueTask> ParseFromAsync( { long resolvedNextStart = long.MaxValue; ExcelParseException? failure = null; - int failureRow = 0; + int records = 0; while (await rows.MoveNextAsync().ConfigureAwait(false)) { long absolute = start + rows.CurrentRecordStart; @@ -202,11 +206,11 @@ private static async ValueTask> ParseFromAsync( catch (ExcelParseException ex) { failure = ex; - failureRow = models.Count; break; } + records++; } - return (resolvedNextStart, failure, failureRow); + return (resolvedNextStart, failure, records); } internal static long GuessStart(CsvChunkSource source, CsvChunk chunk, byte quote) diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs index 964692ab..e6b6d84b 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvEnumerable.cs @@ -60,7 +60,7 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellati var lists = new ConcurrentBag>(); MergeState state = StartWorkers(slots, lists, ring, ct); Task allWorkers = Task.WhenAll(state.Workers); - long rowsEmitted = 0; + long recordsBefore = 0; try { @@ -75,14 +75,14 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellati int count = models.Count; for (int m = 0; m < count; m++) { - rowsEmitted++; yield return models[m]; } models.Clear(); lists.Add(models); - confirmedNextStart = AdvanceOrThrow(result, rowsEmitted); + confirmedNextStart = AdvanceOrThrow(result, recordsBefore); + recordsBefore += result.Records; if (confirmedNextStart >= long.MaxValue) { break; @@ -97,11 +97,11 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellati } } - private long AdvanceOrThrow(CsvChunkResult result, long rowsEmitted) + private long AdvanceOrThrow(CsvChunkResult result, long recordsBefore) { if (result.Failure is not null) { - throw RenumberFailure(result.Failure, _config.HeaderRow + rowsEmitted + 1); + throw RenumberFailure(result.Failure, _config.HeaderRow + recordsBefore + result.Records + 1); } return result.ResolvedNextStart; } @@ -109,7 +109,8 @@ private long AdvanceOrThrow(CsvChunkResult result, long rowsEmitted) private readonly record struct MergeState( CsvChunkResult?[] Results, TaskCompletionSource[] Ready, - Task[] Workers); + Task[] Workers, + CsvChunkPlan Plan); private MergeState StartWorkers(SemaphoreSlim slots, ConcurrentBag> lists, int ring, CancellationToken ct) { @@ -120,15 +121,17 @@ private MergeState StartWorkers(SemaphoreSlim slots, ConcurrentBag> list slot = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); } - Task[] workers = new Task[Math.Min(_dop, _plan.Count)]; + // Each enumeration hands out the chunks again; a plan already drained would leave the merge waiting forever. + CsvChunkPlan plan = _plan.Restart(); + Task[] workers = new Task[Math.Min(_dop, plan.Count)]; foreach (ref Task worker in workers.AsSpan()) { worker = Task.Run(body, ct); } - return new MergeState(results, ready, workers); + return new MergeState(results, ready, workers, plan); Task body() { - return RunWorkerAsync(results, ready, slots, lists, ring, ct); + return RunWorkerAsync(plan, results, ready, slots, lists, ring, ct); } } @@ -150,7 +153,7 @@ private async Task> TakeChunkAsync( List reuse = result.Models; reuse.Clear(); result = await CsvChunkWorker.ParseAsync( - _source, _plan[i], confirmedNextStart, _map, _info, _readerOptions, _config, reuse, ct).ConfigureAwait(false); + _source, state.Plan[i], confirmedNextStart, _map, _info, _readerOptions, _config, reuse, contentCache: null, ct).ConfigureAwait(false); } state.Results[slot] = null; @@ -160,6 +163,7 @@ private async Task> TakeChunkAsync( } private async Task RunWorkerAsync( + CsvChunkPlan plan, CsvChunkResult?[] results, TaskCompletionSource[] ready, SemaphoreSlim slots, @@ -168,13 +172,15 @@ private async Task RunWorkerAsync( CancellationToken ct) { int claimed = -1; + // One per worker: a cache per chunk would be allocated, and start cold, every 64 KB of input. + Utf8StringCache? contentCache = _readerOptions.InternStrings ? new Utf8StringCache() : null; try { while (true) { ct.ThrowIfCancellationRequested(); await slots.WaitAsync(ct).ConfigureAwait(false); - if (!_plan.TryTakeNext(out CsvChunk chunk)) + if (!plan.TryTakeNext(out CsvChunk chunk)) { slots.Release(); return; @@ -182,7 +188,7 @@ private async Task RunWorkerAsync( claimed = chunk.Index; long? confirmed = chunk.Index == 0 ? _firstDataRecordOffset : null; CsvChunkResult result = await CsvChunkWorker.ParseAsync( - _source, chunk, confirmed, _map, _info, _readerOptions, _config, lists.TryTake(out List? pooled) ? pooled : [], ct).ConfigureAwait(false); + _source, chunk, confirmed, _map, _info, _readerOptions, _config, lists.TryTake(out List? pooled) ? pooled : [], contentCache, ct).ConfigureAwait(false); results[chunk.Index % ring] = result; ready[chunk.Index % ring].TrySetResult(true); } diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvProcessor.cs b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvProcessor.cs index 31e1e16b..fe054d8a 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvProcessor.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsvProcessor.cs @@ -180,7 +180,7 @@ private static void ParseMemoryChunk( CsvChunkSource source, CsvChunk chunk, ChunkOutcome outcome, CsvReaderOptions chunkOptions, RowAction accumulate, CancellationToken ct) { - var rows = new CsvReader.Enumerator(source.SliceAt(outcome.ActualStart), chunkOptions, ct); + var rows = new CsvReader.Enumerator(source.SliceAt(outcome.ActualStart), chunkOptions, contentCache: null, ct); try { while (rows.MoveNext()) @@ -208,7 +208,7 @@ private static async ValueTask> ParseFileChunkAsync Stream partition = source.OpenAt(outcome.ActualStart); await using (partition.ConfigureAwait(false)) { - var rows = new CsvReader.Enumerator(partition, chunkOptions, ct); + var rows = new CsvReader.Enumerator(partition, chunkOptions, contentCache: null, ct); await using (rows.ConfigureAwait(false)) { try diff --git a/src/ExcelReader.Core/Parser/ParallelCsv/RangedFileStream.cs b/src/ExcelReader.Core/Parser/ParallelCsv/RangedFileStream.cs index 6e5b2310..d2e2f681 100644 --- a/src/ExcelReader.Core/Parser/ParallelCsv/RangedFileStream.cs +++ b/src/ExcelReader.Core/Parser/ParallelCsv/RangedFileStream.cs @@ -1,3 +1,4 @@ +using System.Runtime.CompilerServices; using Microsoft.Win32.SafeHandles; namespace ExcelReader.Core.Parser.ParallelCsv @@ -42,6 +43,8 @@ public override int Read(byte[] buffer, int offset, int count) return Read(buffer.AsSpan(offset, count)); } + // Pooled: a state machine box per chunk read was the parallel parse's largest allocation after the rows. + [AsyncMethodBuilder(typeof(PoolingAsyncValueTaskMethodBuilder<>))] public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) { int n = await RandomAccess.ReadAsync(_handle, buffer, _offset, cancellationToken).ConfigureAwait(false); diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs index 4fbc2f71..5a60378c 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs @@ -73,35 +73,37 @@ private struct FieldState public int MatStart; } - internal Enumerator(Stream stream, CsvReaderOptions options, CancellationToken ct = default) - : this(stream, options, ownsSource: false, lease: null, ct) + internal Enumerator(Stream stream, CsvReaderOptions options, Utf8StringCache? contentCache, CancellationToken ct) + : this(stream, options, ownsSource: false, lease: null, contentCache, ct) { } - internal Enumerator(Stream stream, CsvReaderOptions options, bool ownsSource, ReaderLifetime? lease, CancellationToken ct) + internal Enumerator(Stream stream, CsvReaderOptions options, bool ownsSource, ReaderLifetime? lease, + Utf8StringCache? contentCache, CancellationToken ct) : base(stream, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), 64 * 1024, ownsSource, ct) { _delimiter = options.Delimiter; _quote = options.Quote; _stripBom = options.DetectEncodingFromByteOrderMark; - _contentCache = options.InternStrings ? new Utf8StringCache() : null; + _contentCache = contentCache ?? (options.InternStrings ? new Utf8StringCache() : null); _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); _lease = lease; } - internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, CancellationToken ct = default) - : this(content, options, lease: null, ct) + internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, Utf8StringCache? contentCache, CancellationToken ct) + : this(content, options, lease: null, contentCache, ct) { } - internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, ReaderLifetime? lease, CancellationToken ct) + internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, ReaderLifetime? lease, + Utf8StringCache? contentCache, CancellationToken ct) : base(content, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), ct) { _delimiter = options.Delimiter; _quote = options.Quote; _stripBom = options.DetectEncodingFromByteOrderMark; - _contentCache = options.InternStrings ? new Utf8StringCache() : null; + _contentCache = contentCache ?? (options.InternStrings ? new Utf8StringCache() : null); _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); _lease = lease; diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs index c4e9f72d..1e0c07bd 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs @@ -235,11 +235,11 @@ internal Enumerator OpenSheet(CancellationToken ct = default) { if (_source is not null) { - return new Enumerator(OpenSourceStream(_source), _options, ownsSource: true, Lifetime, ct); + return new Enumerator(OpenSourceStream(_source), _options, ownsSource: true, Lifetime, contentCache: null, ct); } if (_stream is null) { - return new Enumerator(_memory, _options, Lifetime, ct); + return new Enumerator(_memory, _options, Lifetime, contentCache: null, ct); } if (!_stream.TryBeginPass()) { @@ -249,7 +249,7 @@ internal Enumerator OpenSheet(CancellationToken ct = default) } try { - return new Enumerator(_stream, _options, ownsSource: true, Lifetime, ct); + return new Enumerator(_stream, _options, ownsSource: true, Lifetime, contentCache: null, ct); } catch { diff --git a/src/ExcelReader.Core/Reader/Internal/FastDouble.cs b/src/ExcelReader.Core/Reader/Internal/FastDouble.cs index f602ec77..6fd94f25 100644 --- a/src/ExcelReader.Core/Reader/Internal/FastDouble.cs +++ b/src/ExcelReader.Core/Reader/Internal/FastDouble.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Numerics; using System.Runtime.CompilerServices; @@ -54,19 +55,11 @@ private static bool TryParse(ReadOnlySpan s, int maxDigits, out double val if (i < s.Length && s[i] == (byte)'.') { int fractionStart = ++i; - for (; i < s.Length; i++) - { - uint d = (uint)(s[i] - (byte)'0'); - if (d > 9) - { - break; - } - mantissa = (mantissa * 10) + d; - } + i = ReadFractionDigits(s, i, ref mantissa); scale = i - fractionStart; } int allDigits = integerDigits + scale; - if (allDigits == 0 || (allDigits > maxDigits && SignificantDigits(s[integerStart..i]) > maxDigits)) + if (allDigits == 0 || HasTooManyDigits(s[integerStart..i], allDigits, maxDigits)) { return false; } @@ -155,6 +148,73 @@ private static bool EiselLemire(ulong w, int q, bool neg, out double value) return true; } + // fast_float's digit loop: eight digits per step while the next eight bytes are all digits, then one + // four-digit step. Integer parts stay byte by byte, since most are too short for a step to pay off. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ReadFractionDigits(ReadOnlySpan s, int i, ref ulong mantissa) + { + while (s.Length - i >= 8) + { + ulong eight = BinaryPrimitives.ReadUInt64LittleEndian(s[i..]); + if ((((eight + 0x4646464646464646) | (eight - 0x3030303030303030)) & 0x8080808080808080) != 0) + { + break; + } + mantissa = (mantissa * 100_000_000) + ParseEightDigits(eight); + i += 8; + } + if (s.Length - i >= 4) + { + uint four = BinaryPrimitives.ReadUInt32LittleEndian(s[i..]); + if ((((four + 0x46464646) | (four - 0x30303030)) & 0x80808080) == 0) + { + mantissa = (mantissa * 10_000) + ParseFourDigits(four); + i += 4; + } + } + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + return i; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ParseEightDigits(ulong eight) + { + eight -= 0x3030303030303030; + eight = (eight * 10) + (eight >> 8); + return (((eight & 0x000000FF000000FF) * 0x000F424000000064) + (((eight >> 16) & 0x000000FF000000FF) * 0x0000271000000001)) >> 32; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ParseFourDigits(uint four) + { + four -= 0x30303030; + four = (four * 10) + (four >> 8); + return (((four & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; + } + + // A leading non-zero digit makes every digit significant, so only leading zeros need the count. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool HasTooManyDigits(ReadOnlySpan number, int allDigits, int maxDigits) + { + if (allDigits <= maxDigits) + { + return false; + } + if (number[0] is >= (byte)'1' and <= (byte)'9') + { + return true; + } + return SignificantDigits(number) > maxDigits; + } + [MethodImpl(MethodImplOptions.NoInlining)] private static int SignificantDigits(ReadOnlySpan number) { diff --git a/src/ExcelReader.Core/Reader/Row.cs b/src/ExcelReader.Core/Reader/Row.cs index 094dd042..82f285b0 100644 --- a/src/ExcelReader.Core/Reader/Row.cs +++ b/src/ExcelReader.Core/Reader/Row.cs @@ -32,6 +32,14 @@ internal Row(ReadOnlySpan cells, ReadOnlySpan rowValues, ReadOnl /// Enumerates only the populated cells in this row, in ascending column order. public RowCellEnumerator Cells => new(_cells, _rowValues, _shared, _rowBuffer, _sharedStringCache, _contentCache); + internal ReadOnlySpan Descriptors => _cells; + + internal ReadOnlySpan RowValues => _rowValues; + + internal ReadOnlySpan Shared => _shared; + + internal ReadOnlySpan RowBuffer => _rowBuffer; + internal bool IsEmptyRecord => _cells.IsEmpty || (_cells.Length == 1 && _cells[0].Column == 0 && _cells[0].Type == CellType.Empty); diff --git a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs index 7452110d..8e0830ee 100644 --- a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs +++ b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs @@ -8,7 +8,6 @@ namespace ExcelReader.Core.Reader.Xls [ExcludeFromCodeCoverage(Justification = "Exercised through XlsWorkbook integration tests; guard-rail branches are corrupt-OLE only.")] internal sealed class WorkbookStream : IDisposable { - private const int HeaderSize = 512; private readonly ByteSource? _source; private readonly int[] _chain; @@ -98,7 +97,7 @@ internal void ResolveChainedRun(long pos, out long runStart, out long runEnd, ou } runStart = (long)first * SectorSize; runEnd = Math.Min(Length, (long)(last + 1) * SectorSize); - bufferOffset = HeaderSize + ((long)_chain[first] * SectorSize); + bufferOffset = ((long)_chain[first] + 1) * SectorSize; if (_chain[first] < 0 || bufferOffset < 0 || bufferOffset + (runEnd - runStart) > _fileLength) { throw new InvalidDataException("The OLE sector chain points past the end of the buffer."); @@ -130,7 +129,7 @@ private void RequireChainIndex(int chainIndex) private ReadOnlySpan FileSlice(int sector, int within, int len) { - long offset = HeaderSize + ((long)sector * SectorSize) + within; + long offset = (((long)sector + 1) * SectorSize) + within; if (sector < 0 || offset < 0 || offset + len > _fileLength) { throw new InvalidDataException("The OLE sector chain points past the end of the buffer."); @@ -154,8 +153,10 @@ internal int LoadSectors(int chainIndex, Span dest) } count++; } - long offset = HeaderSize + ((long)_chain[chainIndex] * SectorSize); - _source!.ReadExactly(offset, dest[..(count * SectorSize)]); + long offset = ((long)_chain[chainIndex] + 1) * SectorSize; + // The stream's last sector may end with the file rather than be padded to a whole sector. + int length = (int)Math.Min(count * SectorSize, Length - ((long)chainIndex * SectorSize)); + _source!.ReadExactly(offset, dest[..length]); return count; } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs index a36b7ac1..7d00d310 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxWorkbook.Enumerator.cs @@ -363,6 +363,15 @@ private bool ParseRowInWindow() ReadOnlySpan rowEnd = _ns is null ? ""u8)) + { + _pos = p + 6; + return true; + } + } int lt = p < len && buf[p] == (byte)'<' ? p : IndexOfBounded(buf, len, p, (byte)'<'); if (lt < 0) { @@ -485,6 +494,185 @@ private int ParseCellSpan(byte[] buf, int len, int p) return cEnd + cClose.Length; } + // Consumes a run of cells in the shape Excel writes, .. or , + // and stops at the first anything else, before touching any state, so ParseCellSpan takes it from there. + private int ParseCanonicalCells(ReadOnlySpan buf, int p) + { + int nextCol = _nextCol; + while (true) + { + int end = TryParseCanonicalCell(buf, p, ref nextCol); + if (end < 0) + { + break; + } + p = end; + } + _nextCol = nextCol; + return p; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int TryParseCanonicalCell(ReadOnlySpan buf, int p, ref int nextCol) + { + if (buf.Length - p < 6 || buf[p] != (byte)'<' || buf[p + 1] != (byte)'c') + { + return -1; + } + int i = ScanCanonicalAttributes(buf, p + 2, out int col, out int style, out Kind kind); + if (i < 0) + { + return -1; + } + if (col < 0) + { + col = nextCol; + } + if (buf[i] == (byte)'/') + { + if (buf[i + 1] != (byte)'>') + { + return -1; + } + nextCol = col + 1; + return i + 2; + } + if (kind == Kind.Inline || !buf.Slice(i, 4).SequenceEqual(">"u8)) + { + return -1; + } + int start = i + 4; + int v = ScanCanonicalValue(buf, start, kind, out int sharedIndex); + if (v < 0 || buf.Length - v < 8 || !buf.Slice(v, 8).SequenceEqual(""u8)) + { + return -1; + } + nextCol = col + 1; + EmitCanonicalCell(kind, buf, start, v, col, style, sharedIndex); + return v + 8; + } + + // A shared string's value must be its index in at most 9 digits; any other value runs to the next markup. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ScanCanonicalValue(ReadOnlySpan buf, int start, Kind kind, out int sharedIndex) + { + sharedIndex = 0; + int v = start; + if (kind != Kind.Shared) + { + while ((uint)v < (uint)buf.Length && buf[v] != (byte)'<') + { + v++; + } + return v; + } + while ((uint)v < (uint)buf.Length && (uint)(buf[v] - '0') <= 9) + { + sharedIndex = (sharedIndex * 10) + (buf[v] - '0'); + v++; + } + if (v - start is 0 or > 9) + { + return -1; + } + return v; + } + + // Returns the index of the byte after the attributes, which leaves at least 6 bytes in buf, or -1. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ScanCanonicalAttributes(ReadOnlySpan buf, int i, out int col, out int style, out Kind kind) + { + col = -1; + style = 0; + kind = Kind.Number; + while (true) + { + if (buf.Length - i < 6) + { + return -1; + } + if (buf[i] != (byte)' ') + { + return i; + } + if (buf[i + 2] != (byte)'=' || buf[i + 3] != (byte)'"') + { + return -1; + } + byte name = buf[i + 1]; + i += 4; + int valueStart = i; + switch (name) + { + case (byte)'r': + int letters = 0; + while ((uint)i < (uint)buf.Length && (uint)(buf[i] - 'A') <= 25) + { + letters = (letters * 26) + (buf[i] - 'A' + 1); + i++; + } + if (i - valueStart is 0 or > 3) + { + return -1; + } + col = letters - 1; + while ((uint)i < (uint)buf.Length && (uint)(buf[i] - '0') <= 9) + { + i++; + } + break; + case (byte)'s': + int value = 0; + while ((uint)i < (uint)buf.Length && (uint)(buf[i] - '0') <= 9) + { + value = (value * 10) + (buf[i] - '0'); + i++; + } + if (i - valueStart is 0 or > 9) + { + return -1; + } + style = value; + break; + case (byte)'t': + while ((uint)i < (uint)buf.Length && buf[i] != (byte)'"') + { + i++; + } + kind = ClassifyKind(buf[valueStart..i]); + break; + default: + return -1; + } + if ((uint)i >= (uint)buf.Length || buf[i] != (byte)'"') + { + return -1; + } + i++; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void EmitCanonicalCell(Kind kind, ReadOnlySpan buf, int start, int end, int col, int style, int sharedIndex) + { + switch (kind) + { + case Kind.Shared: + var (sharedStart, sharedLength, validIndex) = WorkbookLookups.SharedAt(_sharedOffsets, sharedIndex); + _acc.Add(col, sharedStart, sharedLength, CellType.ExcelString, style, CellValueSource.Shared, sharedIndex: validIndex); + break; + case Kind.Number: + ReadOnlySpan number = buf[start..end]; + CellType cellType = WorkbookLookups.IsDateStyle(_styleIsDate, style) ? CellType.Date : CellType.Number; + bool hasNumber = FastDouble.TryParseLossless(number, out double parsed); + _acc.Add(col, start, number.Length, cellType, style, CellValueSource.RowBuffer, number: parsed, hasNumber: hasNumber); + break; + default: + EmitScalarValueFast(kind, buf[start..end], start, col, style); + break; + } + } + private int TryEmitSharedIndex(byte[] buf, int len, int valueStart, int col, int style) { int i = valueStart; @@ -796,6 +984,7 @@ private static bool IsXmlSpace(byte b) private enum Kind { Number, Shared, Inline, Bool, Error, Formula, IsoDate } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Kind ClassifyKind(ReadOnlySpan t) { return t.Length switch @@ -900,8 +1089,14 @@ private void EmitIsoDate(ReadOnlySpan v, int col, int style) { if (TryParseIsoDate(v, out DateTime dt)) { - // ponytail: dt.ToOADate() is exact for dates >= 1900-03-01, which every real t="d" - double serial = dt.ToOADate(); + if (!ExcelEpoch.TryOADateToSerial(dt.ToOADate(), _reader.IsDate1904, out double serial)) + { + // No serial exists before the workbook epoch; Cell.TryGetDateTime parses the kept text instead. + int textStart = _acc.ValueLength; + AppendRaw(v); + _acc.Add(col, textStart, _acc.ValueLength - textStart, CellType.Date, style, CellValueSource.RowValues); + return; + } int start = _acc.ValueLength; Span dst = _acc.ReserveValueSpan(32); CellFormatter.TryFormatDouble(serial, dst, out int written); @@ -931,11 +1126,15 @@ private static bool TryParseIsoDate(ReadOnlySpan utf8, out DateTime value) { chars[i] = (char)utf8[i]; } - const DateTimeStyles dateParseFlag = DateTimeStyles.RoundtripKind | DateTimeStyles.AllowWhiteSpaces; - if (!DateTime.TryParse(chars[..utf8.Length], CultureInfo.InvariantCulture, dateParseFlag, out value)) + // A cell has no time zone: keep the clock time as written and drop any offset, so the + // result never depends on the machine's zone. + const DateTimeStyles dateParseFlag = DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces; + if (!DateTimeOffset.TryParse(chars[..utf8.Length], CultureInfo.InvariantCulture, dateParseFlag, out DateTimeOffset parsed)) { + value = default; return false; } + value = parsed.DateTime; return value.Year >= 100 || RejectDate(out value); } diff --git a/src/ExcelReader.Core/Reader/Zip/Inflate/HuffmanTableBuilder.cs b/src/ExcelReader.Core/Reader/Zip/Inflate/HuffmanTableBuilder.cs index e9fccbd3..f4003007 100644 --- a/src/ExcelReader.Core/Reader/Zip/Inflate/HuffmanTableBuilder.cs +++ b/src/ExcelReader.Core/Reader/Zip/Inflate/HuffmanTableBuilder.cs @@ -5,12 +5,22 @@ namespace ExcelReader.Core.Reader.Zip.Inflate { internal static class HuffmanTableBuilder { + // A method with both a loop and a stackalloc cannot use on-stack replacement, so the JIT compiles it + // fully optimized on first call; keeping the stackallocs here, loop-free, lets the rest start at tier 0. internal static bool TryBuild( Span table, ReadOnlySpan lens, ReadOnlySpan results, int tableBits, int maxLen, Span sorted, bool shrink, out int tableBitsUsed) { Span lenCounts = stackalloc int[InflateFormat.MaxCodewordLength + 2]; + Span offsets = stackalloc int[InflateFormat.MaxCodewordLength + 2]; lenCounts.Clear(); + return TryBuild(table, lens, results, tableBits, maxLen, sorted, shrink, lenCounts, offsets, out tableBitsUsed); + } + + private static bool TryBuild( + Span table, ReadOnlySpan lens, ReadOnlySpan results, int tableBits, int maxLen, + Span sorted, bool shrink, Span lenCounts, Span offsets, out int tableBitsUsed) + { foreach (byte len in lens) { lenCounts[len]++; @@ -25,7 +35,7 @@ internal static bool TryBuild( } tableBitsUsed = tableBits; - uint codespaceUsed = SortSymbols(lens, lenCounts, maxLen, sorted); + uint codespaceUsed = SortSymbols(lens, lenCounts, offsets, maxLen, sorted); ReadOnlySpan used = sorted[lenCounts[0]..lens.Length]; uint codespace = 1U << maxLen; if (codespaceUsed > codespace) @@ -40,9 +50,8 @@ internal static bool TryBuild( return true; } - private static uint SortSymbols(ReadOnlySpan lens, ReadOnlySpan lenCounts, int maxLen, Span sorted) + private static uint SortSymbols(ReadOnlySpan lens, ReadOnlySpan lenCounts, Span offsets, int maxLen, Span sorted) { - Span offsets = stackalloc int[InflateFormat.MaxCodewordLength + 2]; offsets[0] = 0; offsets[1] = lenCounts[0]; uint codespaceUsed = 0; @@ -72,7 +81,9 @@ private static bool TryFillIncomplete( } symbol = used[0]; } - mainTable.Fill(Entry(results[symbol], 1)); + // Span.Fill would JIT a generic SpanHelpers.Fill on the first file; the table is a power of two. + mainTable[0] = Entry(results[symbol], 1); + Replicate(mainTable, 1, mainTable.Length); return true; } diff --git a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.FastLoop.cs b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.FastLoop.cs index ee9aa369..5deee154 100644 --- a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.FastLoop.cs +++ b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.FastLoop.cs @@ -22,8 +22,9 @@ private unsafe LoopStatus RunFastLoop(byte[] input, ref int inPos, int inLimit, { fixed (byte* inBase = input) fixed (byte* outBase = window) - fixed (uint* tables = _tables) + fixed (byte* tableBytes = _tableBytes) { + uint* tables = (uint*)tableBytes; byte* inNext = inBase + inPos; byte* outNext = outBase + outPos; LoopStatus status = FastLoop(ref inNext, inBase + inLimit, outBase, ref outNext, outBase + outLimit, tables); diff --git a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.cs b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.cs index db517fa3..9f61042c 100644 --- a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.cs +++ b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateDecoder.cs @@ -1,5 +1,7 @@ using System.Buffers; using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace ExcelReader.Core.Reader.Zip.Inflate { @@ -29,7 +31,10 @@ internal sealed partial class InflateDecoder : IDisposable private const int LensSize = InflateFormat.MaxLitlenSymbols + InflateFormat.MaxOffsetSymbols + InflateFormat.MaxRepeat; private const ulong PrecodeMask = (1UL << InflateFormat.PrecodeTableBits) - 1; - private readonly uint[] _tables = ArrayPool.Shared.Rent(InflateFormat.LitlenTableSize + InflateFormat.OffsetTableSize); + private const int TableEntries = InflateFormat.LitlenTableSize + InflateFormat.OffsetTableSize; + + // Rented as bytes: ArrayPool comes precompiled with the runtime, ArrayPool is JIT-compiled on the first file. + private readonly byte[] _tableBytes = ArrayPool.Shared.Rent(TableEntries * sizeof(uint)); private ulong _litlenMask; private bool _staticCodesLoaded; private ulong _bitbuf; @@ -81,7 +86,7 @@ public void Dispose() return; } _disposed = true; - ArrayPool.Shared.Return(_tables); + ArrayPool.Shared.Return(_tableBytes); } private bool TryReadHeader(byte[] input, ref int inPos, int inEnd, bool inputEnded) @@ -220,6 +225,23 @@ private uint PopBits(int count) return value; } + // Span.Fill and InitBlock end in a Vector memset that is never precompiled, so either would be + // JIT-compiled on the first file; the spans filled here are at most a few hundred bytes. + internal static void FillBytes(Span span, byte value) + { + ref byte start = ref MemoryMarshal.GetReference(span); + ulong pattern = value * 0x0101_0101_0101_0101UL; + int i = 0; + for (; i + sizeof(ulong) <= span.Length; i += sizeof(ulong)) + { + Unsafe.WriteUnaligned(ref Unsafe.Add(ref start, i), pattern); + } + for (; i < span.Length; i++) + { + Unsafe.Add(ref start, i) = value; + } + } + private void LoadStaticCodes() { if (_staticCodesLoaded) @@ -227,11 +249,11 @@ private void LoadStaticCodes() return; } Span lens = stackalloc byte[InflateFormat.MaxLitlenSymbols + InflateFormat.MaxOffsetSymbols]; - lens[..144].Fill(8); - lens[144..256].Fill(9); - lens[256..280].Fill(7); - lens[280..288].Fill(8); - lens[288..].Fill(5); + FillBytes(lens[..144], 8); + FillBytes(lens[144..256], 9); + FillBytes(lens[256..280], 7); + FillBytes(lens[280..288], 8); + FillBytes(lens[288..], 5); BuildCodeTables(lens, InflateFormat.MaxLitlenSymbols, InflateFormat.MaxOffsetSymbols); _staticCodesLoaded = true; } @@ -286,7 +308,7 @@ private void ReadCodeLengths(ReadOnlySpan input, ref int inPos, ReadOnlySp continue; } int repeat = ReadRepeat(symbol, count, lens, out byte value); - lens.Slice(count, repeat).Fill(value); + FillBytes(lens.Slice(count, repeat), value); count += repeat; } if (count != total) @@ -314,17 +336,19 @@ private int ReadRepeat(uint symbol, int count, ReadOnlySpan lens, out byte return 3 + (int)PopBits(2); } + private Span Tables => MemoryMarshal.Cast(_tableBytes.AsSpan(0, TableEntries * sizeof(uint))); + private void BuildCodeTables(ReadOnlySpan lens, int litlenSymbols, int offsetSymbols) { Span sorted = stackalloc ushort[InflateFormat.MaxLitlenSymbols]; if (!HuffmanTableBuilder.TryBuild( - _tables.AsSpan(InflateFormat.LitlenTableSize, InflateFormat.OffsetTableSize), lens.Slice(litlenSymbols, offsetSymbols), InflateFormat.OffsetResults, + Tables.Slice(InflateFormat.LitlenTableSize, InflateFormat.OffsetTableSize), lens.Slice(litlenSymbols, offsetSymbols), InflateFormat.OffsetResults, InflateFormat.OffsetTableBits, InflateFormat.MaxCodewordLength, sorted, shrink: false, out _)) { throw new InvalidDataException("A deflate block has an invalid offset code."); } if (!HuffmanTableBuilder.TryBuild( - _tables.AsSpan(0, InflateFormat.LitlenTableSize), lens[..litlenSymbols], InflateFormat.LitlenResults, + Tables[..InflateFormat.LitlenTableSize], lens[..litlenSymbols], InflateFormat.LitlenResults, InflateFormat.LitlenTableBits, InflateFormat.MaxCodewordLength, sorted, shrink: true, out int litlenBits)) { throw new InvalidDataException("A deflate block has an invalid literal/length code."); diff --git a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateStream.cs b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateStream.cs index 23a21d27..2d335554 100644 --- a/src/ExcelReader.Core/Reader/Zip/Inflate/InflateStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/Inflate/InflateStream.cs @@ -31,7 +31,7 @@ internal sealed class InflateStream : Stream internal InflateStream(Stream inner) { _inner = inner; - _window.AsSpan(CanaryStart, CanarySize).Fill(CanaryByte); + InflateDecoder.FillBytes(_window.AsSpan(CanaryStart, CanarySize), CanaryByte); } public override bool CanRead => !_disposed; diff --git a/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs b/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs index 5d8eea70..bf5a3885 100644 --- a/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/PrefetchStream.cs @@ -209,10 +209,12 @@ private async Task ProduceAsync(CancellationToken token) { break; } - if (_expectedCrc is uint expected) + // Only the declared bytes are summed, so an entry that inflates past its size still has them checked. + if (_expectedCrc is uint expected && produced < _declaredLength) { - crc = Crc32.Append(crc, pending.AsSpan(0, read)); - produced += read; + int counted = (int)Math.Min(read, _declaredLength - produced); + crc = Crc32.Append(crc, pending.AsSpan(0, counted)); + produced += counted; if (produced == _declaredLength && crc != expected) { throw new InvalidDataException(LimitedReadStream.ChecksumMismatch); diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index e1215363..0c52640f 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -62,6 +62,7 @@ internal sealed class ZipIndex : IDisposable private static ReadOnlySpan EocdSignatureBytes => [0x50, 0x4B, 0x05, 0x06]; private const int TailWindowSize = 65535 + EocdFixedSize + Zip64LocatorSize; + private const int SmallDirectoryEntries = 64; private readonly ByteSource _source; private readonly ReadOnlyMemory _file; @@ -254,10 +255,17 @@ private static ZipIndex Build(ByteSource source, ReadOnlyMemory directory, private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, ZipEntryRef[] entries, int count) { - if (count <= 1) + // A package has a few dozen parts; comparing pairs skips JIT-compiling a sort on every process's first open. + if (count <= SmallDirectoryEntries) { + ThrowIfDuplicateEntryNamesByPairs(directory.Span, entries.AsSpan(0, count)); return; } + ThrowIfDuplicateEntryNamesBySort(directory, entries, count); + } + + private static void ThrowIfDuplicateEntryNamesBySort(ReadOnlyMemory directory, ZipEntryRef[] entries, int count) + { int[] order = ArrayPool.Shared.Rent(count); try { @@ -290,6 +298,21 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, Z } } + private static void ThrowIfDuplicateEntryNamesByPairs(ReadOnlySpan directory, ReadOnlySpan entries) + { + for (int i = 1; i < entries.Length; i++) + { + ReadOnlySpan name = directory.Slice(entries[i].NameStart, entries[i].NameLength); + for (int j = 0; j < i; j++) + { + if (entries[j].NameLength == name.Length && directory.Slice(entries[j].NameStart, name.Length).SequenceEqual(name)) + { + throw new InvalidDataException("The ZIP central directory contains a duplicate entry name."); + } + } + } + } + private void ThrowIfDisposed() { ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); diff --git a/src/ExcelReader.Core/Writer/Internal/StyleTable.cs b/src/ExcelReader.Core/Writer/Internal/StyleTable.cs index b8a11dc7..063f58b7 100644 --- a/src/ExcelReader.Core/Writer/Internal/StyleTable.cs +++ b/src/ExcelReader.Core/Writer/Internal/StyleTable.cs @@ -39,6 +39,10 @@ internal int Add(CellStyle style) throw new ArgumentException( $"A number format cannot be longer than {MaxNumberFormatLength} characters ({style.NumberFormat.Length} chars).", nameof(style)); } + if (style.NumberFormat is { } numberFormat && WriterStateGuard.HasUnwritableCharacter(numberFormat)) + { + throw new ArgumentException("A number format cannot contain control characters, U+FFFE or U+FFFF.", nameof(style)); + } ClaimNumberFormat(style.NumberFormat); index = _styles.Count; _styles.Add(style); diff --git a/src/ExcelReader.Core/Writer/Internal/WriterStateGuard.cs b/src/ExcelReader.Core/Writer/Internal/WriterStateGuard.cs index 700ae88e..3e1425e7 100644 --- a/src/ExcelReader.Core/Writer/Internal/WriterStateGuard.cs +++ b/src/ExcelReader.Core/Writer/Internal/WriterStateGuard.cs @@ -82,7 +82,7 @@ private static void ValidateSheetName(string name) } } - private static bool HasUnwritableCharacter(string name) + internal static bool HasUnwritableCharacter(string name) { foreach (char c in name) { diff --git a/src/ExcelReader.Core/Writer/Parallel/BatchRunner.cs b/src/ExcelReader.Core/Writer/Parallel/BatchRunner.cs index f9f19b1b..35d5170d 100644 --- a/src/ExcelReader.Core/Writer/Parallel/BatchRunner.cs +++ b/src/ExcelReader.Core/Writer/Parallel/BatchRunner.cs @@ -25,7 +25,7 @@ internal sealed class BatchRunner internal BatchRunner(ParallelWriteOptions options, int rowsBefore, Func format, Func emit, CancellationToken ct) { - _degree = options.DegreeOfParallelism == 0 ? Environment.ProcessorCount : options.DegreeOfParallelism; + _degree = options.EffectiveDegree; _batchRows = Math.Min(options.BatchRows, ExcelLimits.MaxRows); _rowsBefore = rowsBefore; _format = format; @@ -102,14 +102,7 @@ private async ValueTask DispatchAsync() _batch = ArrayPool.Shared.Rent(_batchRows); _count = 0; _rowsBefore += count; - if (_degree == 1) - { - _inFlight.Enqueue(Task.FromResult(Format(batch, count, rowsBefore))); - } - else - { - _inFlight.Enqueue(Task.Run(() => Format(batch, count, rowsBefore), CancellationToken.None)); - } + _inFlight.Enqueue(Task.Run(() => Format(batch, count, rowsBefore), CancellationToken.None)); } private BatchSegment Format(T[] batch, int count, int rowsBefore) diff --git a/src/ExcelReader.Core/Writer/Parallel/ParallelRecordWriter.cs b/src/ExcelReader.Core/Writer/Parallel/ParallelRecordWriter.cs index b500abcb..027eee9c 100644 --- a/src/ExcelReader.Core/Writer/Parallel/ParallelRecordWriter.cs +++ b/src/ExcelReader.Core/Writer/Parallel/ParallelRecordWriter.cs @@ -18,7 +18,7 @@ internal static async ValueTask WriteRecordsAsync(TSheet sheet, internal static ValueTask WriteRowsAsync(TSheet sheet, RecordSource records, Action writeRow, ParallelWriteOptions? options, CancellationToken ct) - where TSheet : class, IBatchSheet + where TSheet : class, ISheetWriter, IBatchSheet where TRow : IRowWriter { return RunAsync(sheet, records, writeRow, Resolve(options), ct); @@ -45,9 +45,14 @@ private static ParallelWriteOptions Resolve(ParallelWriteOptions? options) private static async ValueTask RunAsync(TSheet sheet, RecordSource records, Action writeRow, ParallelWriteOptions options, CancellationToken ct) - where TSheet : class, IBatchSheet + where TSheet : class, ISheetWriter, IBatchSheet where TRow : IRowWriter { + if (options.EffectiveDegree == 1) + { + await RunSeriallyAsync(sheet, records, writeRow, ct).ConfigureAwait(false); + return; + } int rowsBefore = await sheet.BeginSegmentsAsync(ct).ConfigureAwait(false); var workers = new ConcurrentBag(); try @@ -72,6 +77,28 @@ private static async ValueTask RunAsync(TSheet sheet, RecordSou sheet.EndSegments(); } + // Batching only pays when batches overlap; one worker would add the cutting and sync flushes for nothing. + private static async ValueTask RunSeriallyAsync(TSheet sheet, RecordSource records, + Action writeRow, CancellationToken ct) + where TSheet : class, ISheetWriter, IBatchSheet + where TRow : IRowWriter + { + try + { + if (records.Sync is not null) + { + await sheet.WriteRecordsAsync(records.Sync, writeRow, ct).ConfigureAwait(false); + return; + } + await sheet.WriteRecordsAsync(records.Async!, writeRow, ct).ConfigureAwait(false); + } + catch + { + await sheet.FailSegmentsAsync().ConfigureAwait(false); + throw; + } + } + private static BatchSegment FormatBatch(TSheet sheet, ConcurrentBag workers, Action writeRow, T[] batch, int count, int rowsBefore) where TSheet : class, IBatchSheet diff --git a/src/ExcelReader.Core/Writer/ParallelWriteOptions.cs b/src/ExcelReader.Core/Writer/ParallelWriteOptions.cs index 5d0d0760..98c01d7e 100644 --- a/src/ExcelReader.Core/Writer/ParallelWriteOptions.cs +++ b/src/ExcelReader.Core/Writer/ParallelWriteOptions.cs @@ -10,11 +10,20 @@ public sealed class ParallelWriteOptions /// /// Gets the most batches formatted and compressed at once. 0, the default, means - /// . With 1 the batches run on the calling flow, - /// without the thread pool. + /// . With 1, or 0 on a single processor, the + /// rows are written one by one on the calling flow, as a serial write does, and + /// is ignored. /// public int DegreeOfParallelism { get; init; } + internal int EffectiveDegree + { + get + { + return DegreeOfParallelism == 0 ? Environment.ProcessorCount : DegreeOfParallelism; + } + } + /// /// Gets the number of rows per batch. Each batch in flight holds its formatted rows and their /// compressed form, so this and bound the memory used. diff --git a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs index 23af5443..437e857e 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs @@ -300,10 +300,18 @@ private static void ValidateColumn(int col) private XlsSheetWriter CreateContinuation() { - string suffix = $" ({_owner.SheetCount + 1})"; - string contName = _baseName.Length + suffix.Length <= MaxSheetNameLength - ? _baseName + suffix - : _baseName[..(MaxSheetNameLength - suffix.Length)] + suffix; + string contName; + for (int n = _owner.SheetCount + 1; ; n++) + { + string suffix = $" ({n})"; + contName = _baseName.Length + suffix.Length <= MaxSheetNameLength + ? _baseName + suffix + : _baseName[..(MaxSheetNameLength - suffix.Length)] + suffix; + if (_owner.TryClaimSheetName(contName)) + { + break; + } + } var cont = new XlsSheetWriter(_owner, contName, _date1904, Visibility, isContinuation: true, baseName: _baseName); cont.EnsureStarted(); return cont; diff --git a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs index d9f429ae..7f332b23 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsWorkbookWriter.cs @@ -72,6 +72,11 @@ public XlsSheetWriter AddSheet(string name, ExcelSheetVisibility visibility) internal int SheetCount => _sheets.Count; + internal bool TryClaimSheetName(string name) + { + return _sheetNames.Add(name); + } + /// public int AddStyle(CellStyle style) { diff --git a/src/ExcelReader.Core/Writer/Zip/Crc32.cs b/src/ExcelReader.Core/Writer/Zip/Crc32.cs index baccde2a..8d77c7eb 100644 --- a/src/ExcelReader.Core/Writer/Zip/Crc32.cs +++ b/src/ExcelReader.Core/Writer/Zip/Crc32.cs @@ -152,9 +152,8 @@ private static uint Finish(Vector128 folded, ref byte source, int positio position += 16; } data = data[position..]; - Span remainder = stackalloc byte[16]; - folded.AsByte().CopyTo(remainder); - return TableUpdate(0, remainder); + // Not a stackalloc: with one, this loop would be compiled fully optimized on first call instead of at tier 0. + return TableUpdate(0, MemoryMarshal.AsBytes(new ReadOnlySpan>(in folded))); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs b/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs index 80b41fe5..7a977bde 100644 --- a/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs +++ b/src/ExcelReader.Core/Writer/Zip/DeflateEncoder.cs @@ -26,7 +26,7 @@ internal sealed class DeflateEncoder : IDisposable // each symbol writes ahead of the position it advances to. private const int MaxEncodedBlock = (MaxBlockLength * 9 / 8) + 24; - private byte[] _window; + private byte[] _window = []; private uint[] _table; private int _encoded; private int _end; @@ -36,14 +36,31 @@ internal sealed class DeflateEncoder : IDisposable internal DeflateEncoder() { - _window = ArrayPool.Shared.Rent(WindowCapacity); _table = ArrayPool.Shared.Rent(1 << HashBits); Array.Clear(_table, 0, 1 << HashBits); } + /// + /// Same output as then on a new encoder, but matches + /// are read from itself, so no window is rented or filled. + /// + internal static void CompressSegment(ReadOnlySpan input, BiffBuffer output) + { + using var encoder = new DeflateEncoder(); + while (input.Length - encoder._encoded >= MaxBlockLength) + { + encoder.EncodeBlock(input, encoder._encoded + MaxBlockLength, final: false, output); + } + encoder.FlushSync(input, output); + } + internal void Write(ReadOnlySpan input, BiffBuffer output) { - ObjectDisposedException.ThrowIf(_window.Length == 0, this); + ObjectDisposedException.ThrowIf(_table.Length == 0, this); + if (_window.Length == 0) + { + _window = ArrayPool.Shared.Rent(WindowCapacity); + } while (!input.IsEmpty) { if (_end == WindowCapacity) @@ -56,14 +73,14 @@ internal void Write(ReadOnlySpan input, BiffBuffer output) input = input[take..]; while (_end - _encoded >= MaxBlockLength) { - EncodeBlock(_encoded + MaxBlockLength, final: false, output); + EncodeBlock(_window, _encoded + MaxBlockLength, final: false, output); } } } internal void Finish(BiffBuffer output) { - EncodeBlock(_end, final: true, output); + EncodeBlock(_window, _end, final: true, output); Span tail = output.GetSpan(sizeof(ulong)); BinaryPrimitives.WriteUInt64LittleEndian(tail, _bits); output.Advance((_bitCount + 7) / 8); @@ -77,28 +94,36 @@ internal void Finish(BiffBuffer output) /// internal void FlushSync(BiffBuffer output) { - EncodeBlock(_end, final: false, output); - Span tail = output.GetSpan(sizeof(ulong) + 4); - int headerBytes = (_bitCount + 3 + 7) / 8; - BinaryPrimitives.WriteUInt64LittleEndian(tail, _bits); - BinaryPrimitives.WriteUInt32LittleEndian(tail[headerBytes..], 0xFFFF_0000); - output.Advance(headerBytes + 4); - _bits = 0; - _bitCount = 0; + FlushSync(_window.AsSpan(0, _end), output); } public void Dispose() { - if (_window.Length == 0) + if (_table.Length == 0) { return; } - ArrayPool.Shared.Return(_window); + if (_window.Length > 0) + { + ArrayPool.Shared.Return(_window); + } ArrayPool.Shared.Return(_table); _window = []; _table = []; } + private void FlushSync(ReadOnlySpan source, BiffBuffer output) + { + EncodeBlock(source, source.Length, final: false, output); + Span tail = output.GetSpan(sizeof(ulong) + 4); + int headerBytes = (_bitCount + 3 + 7) / 8; + BinaryPrimitives.WriteUInt64LittleEndian(tail, _bits); + BinaryPrimitives.WriteUInt32LittleEndian(tail[headerBytes..], 0xFFFF_0000); + output.Advance(headerBytes + 4); + _bits = 0; + _bitCount = 0; + } + // Keeps the last 32 KB already encoded as history. The hash table holds absolute positions, so // moving the window only moves _windowStart; it never rewrites the table. private void Slide() @@ -110,7 +135,7 @@ private void Slide() _end -= keepFrom; } - private void EncodeBlock(int blockEnd, bool final, BiffBuffer output) + private void EncodeBlock(ReadOnlySpan source, int blockEnd, bool final, BiffBuffer output) { Span destination = output.GetSpan(MaxEncodedBlock); int length = blockEnd - _encoded; @@ -121,8 +146,8 @@ private void EncodeBlock(int blockEnd, bool final, BiffBuffer output) }; // Up to 31 pending bits plus the 3 header bits can exceed 32; flush so a 31-bit symbol still fits. FlushWord(destination, ref state); - int next = EncodeMatches(blockEnd, destination, ref state); - EncodeLiterals(next, blockEnd, destination, ref state); + int next = EncodeMatches(source, blockEnd, destination, ref state); + EncodeLiterals(source, next, blockEnd, destination, ref state); state.Count += 7; FlushWord(destination, ref state); long fixedBits = (8L * state.Written) + state.Count - _bitCount; @@ -134,7 +159,7 @@ private void EncodeBlock(int blockEnd, bool final, BiffBuffer output) } else { - output.Advance(WriteStored(destination, length, final)); + output.Advance(WriteStored(source, destination, length, final)); } _encoded = blockEnd; } @@ -145,14 +170,14 @@ private long StoredBits(int length) return 3 + padding + 32 + (8L * length); } - private int WriteStored(Span destination, int length, bool final) + private int WriteStored(ReadOnlySpan source, Span destination, int length, bool final) { ulong header = _bits | ((final ? 1UL : 0UL) << _bitCount); int headerBytes = (_bitCount + 3 + 7) / 8; BinaryPrimitives.WriteUInt64LittleEndian(destination, header); BinaryPrimitives.WriteUInt16LittleEndian(destination[headerBytes..], (ushort)length); BinaryPrimitives.WriteUInt16LittleEndian(destination[(headerBytes + 2)..], (ushort)~length); - _window.AsSpan(_encoded, length).CopyTo(destination[(headerBytes + 4)..]); + source.Slice(_encoded, length).CopyTo(destination[(headerBytes + 4)..]); _bits = 0; _bitCount = 0; return headerBytes + 4 + length; @@ -161,12 +186,12 @@ private int WriteStored(Span destination, int length, bool final) // Called once per block, so tiered compilation would run the loop as on-stack-replacement code // for the first blocks of a process. [MethodImpl(MethodImplOptions.AggressiveOptimization)] - private int EncodeMatches(int blockEnd, Span destination, ref BitState state) + private int EncodeMatches(ReadOnlySpan source, int blockEnd, Span destination, ref BitState state) { - byte[] windowArray = _window; uint[] tableArray = _table; - ObjectDisposedException.ThrowIf(windowArray.Length < WindowCapacity || tableArray.Length < 1 << HashBits, this); - ref byte window = ref MemoryMarshal.GetArrayDataReference(windowArray); + ObjectDisposedException.ThrowIf(tableArray.Length < 1 << HashBits, this); + ArgumentOutOfRangeException.ThrowIfGreaterThan(blockEnd, source.Length); + ref byte window = ref MemoryMarshal.GetReference(source); ref uint table = ref MemoryMarshal.GetArrayDataReference(tableArray); ref byte output = ref MemoryMarshal.GetReference(destination); ref uint literals = ref MemoryMarshal.GetArrayDataReference(FixedHuffmanCodes.Literals); @@ -217,11 +242,11 @@ private int EncodeMatches(int blockEnd, Span destination, ref BitState sta return index; } - private void EncodeLiterals(int index, int blockEnd, Span destination, ref BitState state) + private static void EncodeLiterals(ReadOnlySpan source, int index, int blockEnd, Span destination, ref BitState state) { for (; index < blockEnd; index++) { - uint literal = FixedHuffmanCodes.Literals[_window[index]]; + uint literal = FixedHuffmanCodes.Literals[source[index]]; state.Bits |= (ulong)(literal >> FixedHuffmanCodes.LengthShift) << state.Count; state.Count += (int)(literal & FixedHuffmanCodes.LengthMask); FlushWord(destination, ref state); diff --git a/src/ExcelReader.Core/Writer/Zip/SegmentCompressor.cs b/src/ExcelReader.Core/Writer/Zip/SegmentCompressor.cs index af4780af..bc882bd8 100644 --- a/src/ExcelReader.Core/Writer/Zip/SegmentCompressor.cs +++ b/src/ExcelReader.Core/Writer/Zip/SegmentCompressor.cs @@ -29,9 +29,7 @@ internal static void Compress(ReadOnlySpan input, CompressionLevel level, // DeflateEncoder writes and compares native-order words, which is only deflate's bit order on little-endian. if (level == CompressionLevel.Fastest && BitConverter.IsLittleEndian) { - using var encoder = new DeflateEncoder(); - encoder.Write(input, output); - encoder.FlushSync(output); + DeflateEncoder.CompressSegment(input, output); return; } CompressWithZlib(input, level, output); diff --git a/src/ExcelReader.Native/Csv/CsvAggregateApi.Workbook.cs b/src/ExcelReader.Native/Csv/CsvAggregateApi.Workbook.cs new file mode 100644 index 00000000..a8bc07bb --- /dev/null +++ b/src/ExcelReader.Native/Csv/CsvAggregateApi.Workbook.cs @@ -0,0 +1,52 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Native.Reading; + +namespace ExcelReader.Native.Csv +{ + internal static unsafe partial class CsvAggregateApi + { + internal static int AggregateWorkbook( + NativeHandle? handle, int sheet, NativeCsvAggregationRaw aggregation, + int headerRow, int degreeOfParallelism, out nint result) + { + result = 0; + if (handle is null) + { + return NativeStatus.InvalidHandle; + } + + NativeApi.ClearLastError(); + if (headerRow < 0 || degreeOfParallelism < 0) + { + NativeApi.SetLastError( + $"header_row and degree_of_parallelism must not be negative; got {headerRow} and {degreeOfParallelism}."); + return NativeStatus.InvalidArgument; + } + int status = ReadApi.ResolveSheet(handle, sheet, out IExcelSheet? resolved); + if (status != NativeStatus.Ok) + { + return status; + } + + CsvAggregateContext context = new(aggregation); + SheetParallelOptions options = new() { DegreeOfParallelism = degreeOfParallelism, HeaderRow = headerRow }; + RowAggregation rows = Build(context, (ref CsvAggregateState state, Row row) => + { + state.Views ??= new CellViewBuffer(); + int status; + fixed (byte* rowValues = row.RowValues) + fixed (byte* shared = row.Shared) + fixed (byte* rowBuffer = row.RowBuffer) + { + NativeCellView* cells = state.Views.Fill(row.Descriptors, rowValues, shared, rowBuffer); + status = context.AccumulateView((void*)state.Native, cells, row.Descriptors.Length, context.UserData); + } + if (status != NativeStatus.Ok) + { + CsvAggregateContext.Abort(status); + } + }); + return Run(context, Task.Run(() => SheetParallel.AggregateAsync(resolved!, rows, options, CancellationToken.None)), out result); + } + } +} diff --git a/src/ExcelReader.Native/Csv/CsvAggregateApi.cs b/src/ExcelReader.Native/Csv/CsvAggregateApi.cs index 6e56156b..9d5b620e 100644 --- a/src/ExcelReader.Native/Csv/CsvAggregateApi.cs +++ b/src/ExcelReader.Native/Csv/CsvAggregateApi.cs @@ -25,6 +25,8 @@ private sealed class CsvAggregateContext(NativeCsvAggregationRaw raw) (delegate* unmanaged)raw.Seed; internal readonly delegate* unmanaged Accumulate = (delegate* unmanaged)raw.Accumulate; + internal readonly delegate* unmanaged AccumulateView = + (delegate* unmanaged)raw.Accumulate; internal readonly delegate* unmanaged Combine = (delegate* unmanaged)raw.Combine; internal readonly delegate* unmanaged FreeState = @@ -103,6 +105,19 @@ internal static int AggregateCsvMemory( } private static RowAggregation Build(CsvAggregateContext context) + { + return Build(context, (ref state, row) => + { + NativeRow native = state.WriteRow(row); + int status = context.Accumulate((void*)state.Native, &native, context.UserData); + if (status != NativeStatus.Ok) + { + CsvAggregateContext.Abort(status); + } + }); + } + + private static RowAggregation Build(CsvAggregateContext context, RowAction accumulate) { return new RowAggregation { @@ -119,15 +134,7 @@ private static RowAggregation Build(CsvAggregateContext conte } return state; }, - Accumulate = (ref CsvAggregateState state, Row row) => - { - NativeRow native = state.WriteRow(row); - int status = context.Accumulate((void*)state.Native, &native, context.UserData); - if (status != NativeStatus.Ok) - { - CsvAggregateContext.Abort(status); - } - }, + Accumulate = accumulate, Combine = (accumulator, next) => { int status = context.Combine( @@ -136,6 +143,7 @@ private static RowAggregation Build(CsvAggregateContext conte { CsvAggregateContext.Abort(status); } + next.ReleaseScratch(); return accumulator; }, }; diff --git a/src/ExcelReader.Native/Csv/CsvAggregateState.cs b/src/ExcelReader.Native/Csv/CsvAggregateState.cs index d6c6567a..f577e294 100644 --- a/src/ExcelReader.Native/Csv/CsvAggregateState.cs +++ b/src/ExcelReader.Native/Csv/CsvAggregateState.cs @@ -11,6 +11,8 @@ internal sealed unsafe class CsvAggregateState : IDisposable internal nint Native { get; set; } + internal CellViewBuffer? Views { get; set; } + internal NativeRow WriteRow(in Row row) { int cellCount = 0; @@ -54,6 +56,8 @@ internal NativeRow WriteRow(in Row row) internal void ReleaseScratch() { + Views?.Dispose(); + Views = null; if (_scratch is not null) { NativeMemory.Free(_scratch); diff --git a/src/ExcelReader.Native/Exports.Csv.cs b/src/ExcelReader.Native/Exports.Csv.cs index 16729c93..9a501908 100644 --- a/src/ExcelReader.Native/Exports.Csv.cs +++ b/src/ExcelReader.Native/Exports.Csv.cs @@ -55,17 +55,42 @@ public static int CsvAggregateMemory( return status; } - private static bool IsValidAggregation(NativeCsvAggregationRaw* aggregation) + [UnmanagedCallersOnly(EntryPoint = "xl_aggregate")] + public static int Aggregate( + nint handle, int sheet, NativeCsvAggregationRaw* aggregation, + int headerRow, int degreeOfParallelism, void** outState) + { + NativeApi.ClearLastError(); + if (outState is null) + { + return NativeStatus.InvalidArgument; + } + *outState = null; + if (!IsValidAggregation(aggregation, "aggregation")) + { + return NativeStatus.InvalidArgument; + } + + int status = CsvAggregateApi.AggregateWorkbook( + Resolve(handle), sheet, *aggregation, headerRow, degreeOfParallelism, out nint result); + if (status == NativeStatus.Ok) + { + *outState = (void*)result; + } + return status; + } + + private static bool IsValidAggregation(NativeCsvAggregationRaw* aggregation, string name = "csv_aggregation") { if (aggregation is null) { - NativeApi.SetLastError("csv_aggregation must not be NULL."); + NativeApi.SetLastError($"{name} must not be NULL."); return false; } if (aggregation->StructSize != sizeof(NativeCsvAggregationRaw)) { NativeApi.SetLastError( - $"csv_aggregation.struct_size must be exactly {sizeof(NativeCsvAggregationRaw)}; got {aggregation->StructSize}."); + $"{name}.struct_size must be exactly {sizeof(NativeCsvAggregationRaw)}; got {aggregation->StructSize}."); return false; } if (aggregation->Seed == IntPtr.Zero || aggregation->Accumulate == IntPtr.Zero @@ -74,19 +99,19 @@ private static bool IsValidAggregation(NativeCsvAggregationRaw* aggregation) List missing = []; if (aggregation->Seed == IntPtr.Zero) { - missing.Add("csv_aggregation.seed"); + missing.Add($"{name}.seed"); } if (aggregation->Accumulate == IntPtr.Zero) { - missing.Add("csv_aggregation.accumulate"); + missing.Add($"{name}.accumulate"); } if (aggregation->Combine == IntPtr.Zero) { - missing.Add("csv_aggregation.combine"); + missing.Add($"{name}.combine"); } if (aggregation->FreeState == IntPtr.Zero) { - missing.Add("csv_aggregation.free_state"); + missing.Add($"{name}.free_state"); } NativeApi.SetLastError($"{string.Join(", ", missing)} must not be NULL."); return false; diff --git a/src/ExcelReader.Native/Exports.Reading.cs b/src/ExcelReader.Native/Exports.Reading.cs index 0373f9c7..389c5ef5 100644 --- a/src/ExcelReader.Native/Exports.Reading.cs +++ b/src/ExcelReader.Native/Exports.Reading.cs @@ -260,6 +260,19 @@ public static int RowsNextView(nint rows, NativeRow* outRow) return status; } + [UnmanagedCallersOnly(EntryPoint = "xl_rows_foreach")] + public static int RowsForEach(nint handle, int sheet, delegate* unmanaged callback, void* userData) + { + NativeApi.ClearLastError(); + + if (callback is null) + { + return NativeStatus.InvalidArgument; + } + + return ReadApi.ForEachRow(Resolve(handle), sheet, callback, userData); + } + [UnmanagedCallersOnly(EntryPoint = "xl_rows_read_all_blob")] public static int RowsReadAllBlob(nint rows, byte* buffer, int capacity, int* outWritten) { diff --git a/src/ExcelReader.Native/Exports.Writer.cs b/src/ExcelReader.Native/Exports.Writer.cs index 132d0bc6..8d692833 100644 --- a/src/ExcelReader.Native/Exports.Writer.cs +++ b/src/ExcelReader.Native/Exports.Writer.cs @@ -69,15 +69,11 @@ public static int WriteTypedToMemory(int format, NativeColumnSpecRaw* specs, Nat private static bool TryDecodeWriteOptions(NativeWriteOptionsRaw* options, out NativeWriteOptions decoded) { decoded = default; - int expectedSize = sizeof(NativeWriteOptionsRaw); - if (options is not null && options->StructSize != expectedSize) + if (!NativeWriteOptionsRaw.TryRead(options, out NativeWriteOptionsRaw raw, out string? readError)) { - NativeApi.SetLastError($"xl_write_options.struct_size is {options->StructSize}, but this library expects {expectedSize}."); + NativeApi.SetLastError(readError); return false; } - NativeWriteOptionsRaw raw = options is null - ? new NativeWriteOptionsRaw { StructSize = expectedSize } - : *options; string? sheetName = null; if (raw.SheetName is not null) { diff --git a/src/ExcelReader.Native/README.md b/src/ExcelReader.Native/README.md index dd0154f1..79b37b7a 100644 --- a/src/ExcelReader.Native/README.md +++ b/src/ExcelReader.Native/README.md @@ -55,7 +55,7 @@ may get worse, not only better. Reading exports: `xl_open_file`, `xl_open_memory`, `xl_open_source`, `xl_open_stream`, `xl_set_source_error`, `xl_close`, `xl_sheet_count`, `xl_sheet_name_at`, `xl_sheet_visibility_at`, `xl_sheet_index`, `xl_is_date1904`, `xl_rows_open`, `xl_rows_next`, -`xl_rows_next_view`, `xl_rows_read_all_blob`, `xl_rows_read_all_decoded`, `xl_rows_close`, +`xl_rows_next_view`, `xl_rows_foreach`, `xl_aggregate`, `xl_rows_read_all_blob`, `xl_rows_read_all_decoded`, `xl_rows_close`, `xl_free_rows`, `xl_parse_typed`, `xl_free_table`, `xl_typed_reader_open`, `xl_typed_reader_next`, `xl_typed_reader_close`, `xl_infer_schema`, `xl_free_schema`, `xl_last_error`, `xl_last_error_ptr`, `xl_parse_arrow`, `xl_parse_arrow_stream`. @@ -73,6 +73,14 @@ is rejected with `XL_INVALID_ARGUMENT` before anything else is inspected. It is whole-table-in-memory, no styling beyond the temporal number formats — see `include/excelreader.h` for the full contract. +For XLSX and XLSB, `degree_of_parallelism` and `batch_rows` format and compress row batches on +several threads: `degree_of_parallelism` `0` uses every processor and `1` writes serially; +`batch_rows` `0` keeps the default of 8,192. Negative values are rejected. A zero-initialized struct +is therefore parallel, while the 7.0 layout (`struct_size` 32) and a NULL options pointer stay serial. +The `xl_table` buffers are read from several threads, so they must not change during the call. With +`use_shared_strings` on, parallel runs give the same content in files that differ byte for byte. +XLS, CSV and the handle API below ignore both fields. + The streaming alternative is `xl_writer_handle`: one sheet and one row open at a time, written directly as each call arrives instead of building an `xl_table` up front. Call order is `xl_open_write_handle`/`xl_open_write_handle_to_memory`, then per sheet `xl_start_sheet`..`xl_end_sheet`, @@ -82,6 +90,33 @@ call per cell, then `xl_close_write_handle`. `xl_write_handle_bytes` reads back handle's bytes so far without releasing it. Both write paths release their buffers with `xl_free_buffer` (`xl_table`/`xl_inferred_schema` keep their own `xl_free_table`/`xl_free_schema`). +## Callback reads + +`xl_rows_foreach` walks a sheet and calls back once per row with an array of `xl_cell_view`; +`xl_aggregate` runs the same callback shape over several threads and folds per-chunk states. Both +skip the copy into a cursor-owned row. + +- `value` points into the reader's own row buffers. It is valid only during the callback and is not + NUL-terminated. `value` is NULL when `value_len` is 0. +- A cell carries text, a number, or both. When `has_number` is set, `number` is the cell's numeric + value (dates as Excel serials); when `value_len > 0`, `value` is its text. Which formats carry + which is not part of the contract. +- `cells` may be NULL when `cell_count` is 0. +- A callback returning anything other than `XL_OK` stops the run; that code is returned verbatim and + no last-error message is set. Library statuses are all `<= 0`, so use positive codes. +- Calling other `xl_` functions from inside a callback is allowed. No lock is held across the + callback; a nested call only overwrites the thread's last-error message. +- `xl_rows_foreach` calls back on the calling thread. `xl_aggregate` never runs seed, accumulate or + combine on the calling thread; `free_state` runs on the calling thread after the run. +- `xl_aggregate` seeds once per chunk (256 KiB of sheet data), not once per thread, and folds with + `combine(acc, next)` in source order. A chunk can be accumulated and then discarded without being + combined, in which case its state goes straight to `free_state`. `combine` must not free `next`; + the library does. +- XLS, table views and CSV opened via `xl_open_stream`/`xl_open_source` read sequentially, with the + same result. +- The CSV dialect, password and limits come from the `xl_open_options` the workbook was opened with; + `xl_aggregate` has no options struct of its own. + ## Consuming from C Include `include/excelreader.h` (add `include/excelreader_arrow.h` too if you want the Arrow C Data diff --git a/src/ExcelReader.Native/Reading/CellViewBuffer.cs b/src/ExcelReader.Native/Reading/CellViewBuffer.cs new file mode 100644 index 00000000..370097fa --- /dev/null +++ b/src/ExcelReader.Native/Reading/CellViewBuffer.cs @@ -0,0 +1,53 @@ +using System.Runtime.InteropServices; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Native.Reading +{ + /// + /// The xl_cell_view array handed to a row callback. The views borrow the row's own buffers, so the + /// caller pins , and + /// around and the callback it feeds. + /// + internal sealed unsafe class CellViewBuffer : IDisposable + { + private NativeCellView* _views; + private int _capacity; + + internal NativeCellView* Fill(ReadOnlySpan cells, byte* rowValues, byte* shared, byte* rowBuffer) + { + if (cells.Length > _capacity) + { + _capacity = Math.Max(Math.Max(64, _capacity * 2), cells.Length); + _views = (NativeCellView*)NativeMemory.Realloc(_views, (nuint)(_capacity * sizeof(NativeCellView))); + } + for (int index = 0; index < cells.Length; index++) + { + ref readonly CellDesc cell = ref cells[index]; + byte* source = cell.Source switch + { + CellValueSource.Shared => shared, + CellValueSource.RowBuffer => rowBuffer, + _ => rowValues, + }; + _views[index] = new NativeCellView + { + Column = cell.Column, + Type = (int)cell.Type, + ValueLength = cell.Length, + HasNumber = cell.HasNumber ? 1 : 0, + Value = cell.Length == 0 ? null : source + cell.Start, + Number = cell.Number, + }; + } + return _views; + } + + public void Dispose() + { + NativeMemory.Free(_views); + _views = null; + _capacity = 0; + } + } +} diff --git a/src/ExcelReader.Native/Reading/NativeRow.cs b/src/ExcelReader.Native/Reading/NativeRow.cs index d239a39b..103ca782 100644 --- a/src/ExcelReader.Native/Reading/NativeRow.cs +++ b/src/ExcelReader.Native/Reading/NativeRow.cs @@ -11,6 +11,17 @@ internal struct NativeRowCell public IntPtr Value; } + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeCellView + { + public int Column; + public int Type; + public int ValueLength; + public int HasNumber; + public byte* Value; + public double Number; + } + [StructLayout(LayoutKind.Sequential)] internal struct NativeRow { diff --git a/src/ExcelReader.Native/Reading/ReadApi.Rows.ForEach.cs b/src/ExcelReader.Native/Reading/ReadApi.Rows.ForEach.cs new file mode 100644 index 00000000..5f599c50 --- /dev/null +++ b/src/ExcelReader.Native/Reading/ReadApi.Rows.ForEach.cs @@ -0,0 +1,51 @@ +using ExcelReader.Core.Reader; + +namespace ExcelReader.Native.Reading +{ + internal static unsafe partial class ReadApi + { + internal static int ForEachRow(NativeHandle? handle, int sheet, + delegate* unmanaged callback, void* userData) + { + if (handle is null) + { + return NativeStatus.InvalidHandle; + } + + NativeApi.ClearLastError(); + int status = ResolveSheet(handle, sheet, out IExcelSheet? resolved); + if (status != NativeStatus.Ok) + { + return status; + } + + try + { + using CellViewBuffer views = new(); + using IExcelRowEnumerator rows = resolved!.GetEnumerator(); + while (rows.MoveNext()) + { + Row row = rows.Current; + fixed (byte* rowValues = row.RowValues) + fixed (byte* shared = row.Shared) + fixed (byte* rowBuffer = row.RowBuffer) + { + NativeCellView* cells = views.Fill(row.Descriptors, rowValues, shared, rowBuffer); + status = callback(cells, row.Descriptors.Length, userData); + } + + if (status != NativeStatus.Ok) + { + return status; + } + } + return NativeStatus.Ok; + } + catch (Exception exception) + { + NativeApi.SetLastError(exception.Message); + return NativeStatus.Error; + } + } + } +} diff --git a/src/ExcelReader.Native/Writer/NativeWriteOptions.cs b/src/ExcelReader.Native/Writer/NativeWriteOptions.cs index 556293fe..882125ed 100644 --- a/src/ExcelReader.Native/Writer/NativeWriteOptions.cs +++ b/src/ExcelReader.Native/Writer/NativeWriteOptions.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; +using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Csv; namespace ExcelReader.Native.Writer @@ -17,6 +18,37 @@ internal unsafe struct NativeWriteOptionsRaw public int CsvQuote; public int Date1904; public int UseSharedStrings; + + public int DegreeOfParallelism; + public int BatchRows; + + internal const int V7Size = 32; + + internal static unsafe bool TryRead(NativeWriteOptionsRaw* options, out NativeWriteOptionsRaw raw, [NotNullWhen(false)] out string? error) + { + raw = default; + error = null; + if (options is null) + { + raw = new NativeWriteOptionsRaw { StructSize = sizeof(NativeWriteOptionsRaw), DegreeOfParallelism = 1 }; + return true; + } + int size = options->StructSize; + if (size != sizeof(NativeWriteOptionsRaw) && size != V7Size) + { + error = $"xl_write_options.struct_size is {size}, but this library expects {sizeof(NativeWriteOptionsRaw)} (or {V7Size} for the 7.0 layout)."; + return false; + } + NativeWriteOptionsRaw copy = default; + Buffer.MemoryCopy(options, ©, sizeof(NativeWriteOptionsRaw), size); + copy.StructSize = sizeof(NativeWriteOptionsRaw); + if (size == V7Size) + { + copy.DegreeOfParallelism = 1; + } + raw = copy; + return true; + } } internal readonly struct NativeWriteOptions @@ -32,6 +64,21 @@ internal readonly struct NativeWriteOptions internal byte? CsvQuote { get; init; } internal bool? Date1904 { get; init; } internal bool? UseSharedStrings { get; init; } + internal int DegreeOfParallelism { get; init; } + internal int BatchRows { get; init; } + + internal ParallelWriteOptions? ToParallelWriteOptions(int format) + { + if (format is not (NativeFormat.Xlsx or NativeFormat.Xlsb) || DegreeOfParallelism == 1) + { + return null; + } + return new ParallelWriteOptions + { + DegreeOfParallelism = DegreeOfParallelism, + BatchRows = BatchRows == 0 ? ParallelWriteOptions.Default.BatchRows : BatchRows, + }; + } internal CsvWriterOptions ToCsvWriterOptions() { @@ -63,6 +110,16 @@ internal static bool TryDecode(NativeWriteOptionsRaw raw, string? sheetName, out { return false; } + if (raw.DegreeOfParallelism < 0) + { + error = $"{OptionsName}.degree_of_parallelism must be 0 (processor count) or positive; got {raw.DegreeOfParallelism}."; + return false; + } + if (raw.BatchRows < 0) + { + error = $"{OptionsName}.batch_rows must be 0 (library default) or positive; got {raw.BatchRows}."; + return false; + } options = new NativeWriteOptions { @@ -71,6 +128,8 @@ internal static bool TryDecode(NativeWriteOptionsRaw raw, string? sheetName, out CsvQuote = quote, Date1904 = date1904, UseSharedStrings = sharedStrings, + DegreeOfParallelism = raw.DegreeOfParallelism, + BatchRows = raw.BatchRows, }; return true; } diff --git a/src/ExcelReader.Native/Writer/WriteApi.cs b/src/ExcelReader.Native/Writer/WriteApi.cs index 1fa5a81a..27143a3e 100644 --- a/src/ExcelReader.Native/Writer/WriteApi.cs +++ b/src/ExcelReader.Native/Writer/WriteApi.cs @@ -94,15 +94,16 @@ internal static int OpenWriteHandle(ReadOnlySpan path, int format, NativeW } NativeApi.ClearLastError(); string filePath = Encoding.UTF8.GetString(path); - FileStream stream = new(filePath, FileMode.Create, FileAccess.Write, FileShare.None); + FileStream? stream = null; try { + stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None); handle = NativeWriterHandle.Create(stream, format, options); return NativeStatus.Ok; } catch (Exception exception) { - stream.Dispose(); + stream?.Dispose(); NativeApi.SetLastError(exception.Message); return NativeStatus.Error; } @@ -190,28 +191,32 @@ private static void WriteToStream(Stream stream, int format, NativeColumnSpec[] { bool date1904 = options.Date1904 ?? false; bool sharedStrings = options.UseSharedStrings ?? false; + ParallelWriteOptions? parallel = options.ToParallelWriteOptions(format); switch (format) { case NativeFormat.Xlsx: WriteWorkbook( - XlsxWorkbookWriter.Create(stream, options: new XlsxWriterOptions { Date1904 = date1904, UseSharedStrings = sharedStrings }), specs, table, sheetName, hasHeader); + XlsxWorkbookWriter.Create(stream, options: new XlsxWriterOptions { Date1904 = date1904, UseSharedStrings = sharedStrings }), specs, table, sheetName, hasHeader, + parallel, static (sheet, rows, writeRow, opts) => sheet.WriteRecordsParallelAsync(rows, writeRow, opts)); return; case NativeFormat.Xlsb: WriteWorkbook( - XlsbWorkbookWriter.Create(stream, options: new XlsbWriterOptions { Date1904 = date1904, UseSharedStrings = sharedStrings }), specs, table, sheetName, hasHeader); + XlsbWorkbookWriter.Create(stream, options: new XlsbWriterOptions { Date1904 = date1904, UseSharedStrings = sharedStrings }), specs, table, sheetName, hasHeader, + parallel, static (sheet, rows, writeRow, opts) => sheet.WriteRecordsParallelAsync(rows, writeRow, opts)); return; case NativeFormat.Xls: WriteWorkbook( - XlsWorkbookWriter.Create(stream, options: new XlsWriterOptions { Date1904 = date1904 }), specs, table, sheetName, hasHeader); + XlsWorkbookWriter.Create(stream, options: new XlsWriterOptions { Date1904 = date1904 }), specs, table, sheetName, hasHeader, null, null); return; default: WriteWorkbook( - CsvWorkbookWriter.Create(stream, options: options.ToCsvWriterOptions()), specs, table, sheetName, hasHeader); + CsvWorkbookWriter.Create(stream, options: options.ToCsvWriterOptions()), specs, table, sheetName, hasHeader, null, null); return; } } - private static void WriteWorkbook(IWorkbookWriter workbook, NativeColumnSpec[] specs, NativeTable table, string sheetName, bool hasHeader) + private static void WriteWorkbook(IWorkbookWriter workbook, NativeColumnSpec[] specs, NativeTable table, string sheetName, bool hasHeader, + ParallelWriteOptions? parallel, Func, Action, ParallelWriteOptions, ValueTask>? writeParallel) where TSheet : ISheetWriter where TRow : IRowWriter { @@ -224,9 +229,17 @@ private static void WriteWorkbook(IWorkbookWriter workbook { WriteHeaderRow(sheet, specs); } - for (long row = 0; row < table.RowCount; row++) + if (parallel is not null && writeParallel is not null) { - WriteDataRow(sheet, table, row); + writeParallel(sheet, RowIndices(table.RowCount), (row, index) => WriteDataCells(row, table, index), parallel) + .AsTask().GetAwaiter().GetResult(); + } + else + { + for (long row = 0; row < table.RowCount; row++) + { + WriteDataRow(sheet, table, row); + } } sheet.End(); @@ -282,12 +295,25 @@ private static void WriteDataRow(TSheet sheet, NativeTable table, where TRow : IRowWriter { using TRow row = sheet.StartRow(); + WriteDataCells(row, table, rowIndex); + } + + private static void WriteDataCells(IRowWriter row, NativeTable table, long rowIndex) + { for (int index = 0; index < table.ColumnCount; index++) { WriteCell(row, table.ColumnAt(index), rowIndex); } } + private static IEnumerable RowIndices(long count) + { + for (long index = 0; index < count; index++) + { + yield return index; + } + } + internal static void WriteCell(IRowWriter row, NativeColumn column, long rowIndex) { if (!IsValidAt(column, rowIndex)) diff --git a/src/ExcelReader.Native/include/excelreader.def b/src/ExcelReader.Native/include/excelreader.def index 6bd570d3..454ceb9e 100644 --- a/src/ExcelReader.Native/include/excelreader.def +++ b/src/ExcelReader.Native/include/excelreader.def @@ -15,6 +15,7 @@ EXPORTS xl_rows_close xl_rows_next xl_rows_next_view + xl_rows_foreach xl_rows_read_all_decoded xl_rows_read_all_blob xl_free_rows @@ -50,5 +51,6 @@ EXPORTS xl_write_handle_bytes xl_csv_aggregate_file xl_csv_aggregate_memory + xl_aggregate xl_last_error xl_last_error_ptr diff --git a/src/ExcelReader.Native/include/excelreader.h b/src/ExcelReader.Native/include/excelreader.h index b17f5a93..a556be64 100644 --- a/src/ExcelReader.Native/include/excelreader.h +++ b/src/ExcelReader.Native/include/excelreader.h @@ -136,6 +136,20 @@ int32_t xl_rows_next(xl_rows* rows, uint8_t* buffer, int32_t capacity, int32_t* int32_t xl_rows_next_view(xl_rows* rows, xl_row* out_row); +typedef struct xl_cell_view { + int32_t column; + int32_t type; + int32_t value_len; + int32_t has_number; /* number is valid; a cell carries text, a number, or both */ + const uint8_t* value; /* valid only until the callback returns, NOT NUL-terminated; NULL when value_len is 0 */ + double number; +} xl_cell_view; + +/* cells may be NULL when cell_count is 0; a non-XL_OK return stops the walk and is returned verbatim (use positive codes); other xl_ calls are allowed from inside it */ +typedef int32_t (*xl_row_fn)(const xl_cell_view* cells, int32_t cell_count, void* user_data); + +int32_t xl_rows_foreach(xl_workbook* handle, int32_t sheet, xl_row_fn callback, void* user_data); + int32_t xl_rows_read_all_blob(xl_rows* rows, uint8_t* buffer, int32_t capacity, int32_t* out_written); typedef struct xl_rows_decoded { @@ -202,7 +216,10 @@ typedef struct xl_write_options { int32_t date1904; - int32_t use_shared_strings; + int32_t use_shared_strings; + + int32_t degree_of_parallelism; + int32_t batch_rows; } xl_write_options; int32_t xl_write_typed(const uint8_t* path, int32_t path_len, @@ -305,6 +322,21 @@ int32_t xl_csv_aggregate_memory(const uint8_t* data, int32_t data_len, const xl_csv_parallel_options* options, void** out_state); +typedef int32_t (*xl_accumulate_fn)(void* state, const xl_cell_view* cells, int32_t cell_count, void* user_data); + +typedef struct xl_aggregation { + int32_t struct_size; + xl_csv_seed_fn seed; + xl_accumulate_fn accumulate; + xl_csv_combine_fn combine; + xl_csv_free_state_fn free_state; + void* user_data; +} xl_aggregation; + +/* seeds once per chunk; combine(acc, next) runs in source order and must not free next; free_state runs on the calling thread after the run */ +int32_t xl_aggregate(xl_workbook* handle, int32_t sheet, const xl_aggregation* agg, + int32_t header_row, int32_t degree_of_parallelism, void** out_state); + int32_t xl_last_error(uint8_t* buffer, int32_t capacity, int32_t* out_len); const uint8_t* xl_last_error_ptr(int32_t* out_len); diff --git a/src/ExcelReader.Native/include/excelreader.hpp b/src/ExcelReader.Native/include/excelreader.hpp index 8378445d..447da5a7 100644 --- a/src/ExcelReader.Native/include/excelreader.hpp +++ b/src/ExcelReader.Native/include/excelreader.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,10 @@ #include #include #include +#if !defined(__cpp_lib_to_chars) +#include +#include +#endif namespace xl { @@ -73,6 +78,11 @@ namespace xl struct Access; + /// Status a shim returns for "a callback threw". Positive, so it can never collide with a + /// library status (every one of those is <= 0). It CAN collide with a caller's own + /// `INT32_MAX`, so the entry points key on `threw` rather than on the status alone. + inline constexpr int32_t kCsvCallbackException = (std::numeric_limits::max)(); + } @@ -186,6 +196,9 @@ namespace xl int32_t date1904 = XL_OPT_DEFAULT; ///< XLS/XLSB only: use the 1904 date system. Off by default. int32_t use_shared_strings = XL_OPT_DEFAULT; ///< XLSX/XLSB only: store repeated text once in a shared-string table. Off by default. + + int32_t degree_of_parallelism = 1; ///< XLSX/XLSB only: batches formatted and compressed at once. 0 uses every processor, 1 (the default) writes serially. + int32_t batch_rows = 0; ///< XLSX/XLSB only: rows per parallel batch. 0 uses the library default (8192). }; namespace detail @@ -228,6 +241,8 @@ namespace xl opts.csv_quote = options.csv_quote; opts.date1904 = options.date1904; opts.use_shared_strings = options.use_shared_strings; + opts.degree_of_parallelism = options.degree_of_parallelism; + opts.batch_rows = options.batch_rows; return opts; } @@ -493,9 +508,112 @@ namespace xl /// @brief One cell of a row, borrowing the row's memory. struct CellView { - int32_t column{}; ///< 0-based column. - CellType type{}; ///< What the cell holds. - std::string_view value{}; ///< The text as stored, so a date is its serial number. + int32_t column{}; ///< 0-based column. + CellType type{}; ///< What the cell holds. + std::string_view value{}; ///< The text as stored, so a date is its serial number. Empty for a cell that carries only a number. + std::optional number{}; ///< The library's own double, set only by Sheet::for_each_row and Sheet::aggregate, and only when the cell carries one. + + /// @brief `number` when set, else `value` parsed as a double. + std::optional as_number() const + { + if (number.has_value()) + { + return number; + } + if (value.empty()) + { + return std::nullopt; + } + double parsed = 0; +#if defined(__cpp_lib_to_chars) + const char *last = value.data() + value.size(); + const auto result = std::from_chars(value.data(), last, parsed); + if (result.ec != std::errc{} || result.ptr != last) + { + return std::nullopt; + } +#else + std::istringstream in{std::string(value)}; + in.imbue(std::locale::classic()); + in >> std::noskipws >> parsed; + if (in.fail() || in.peek() != std::char_traits::eof()) + { + return std::nullopt; + } +#endif + return parsed; + } + + /// @brief `value` when non-empty, else `number` in shortest round-trip form, else empty. + /// + /// That formatting is this header's own, so it can differ from the text `xl_rows_next` writes. + std::string text() const + { + if (!value.empty() || !number.has_value()) + { + return std::string(value); + } +#if defined(__cpp_lib_to_chars) + char buffer[32]; + const auto result = std::to_chars(buffer, buffer + sizeof(buffer), *number); + return std::string(buffer, result.ptr); +#else + // Mirrors to_chars: the fewest digits that round-trip, fixed unless scientific is shorter. + const double v = *number; + std::string scientific; + for (int precision = 0; precision <= 16; ++precision) + { + std::ostringstream out; + out.imbue(std::locale::classic()); + out.precision(precision); + out << std::scientific << v; + scientific = out.str(); + std::istringstream back{scientific}; + back.imbue(std::locale::classic()); + double parsed = 0; + if (back >> parsed && parsed == v) + { + break; + } + } + const std::size_t e = scientific.find('e'); + if (e == std::string::npos) + { + return scientific; + } + const bool negative = scientific[0] == '-'; + std::string digits; + for (std::size_t i = negative ? 1 : 0; i < e; ++i) + { + if (scientific[i] != '.') + { + digits += scientific[i]; + } + } + int exponent = 0; + const char *exponent_start = scientific.data() + e + (scientific[e + 1] == '+' ? 2 : 1); + std::from_chars(exponent_start, scientific.data() + scientific.size(), exponent); + const std::string sign = negative ? "-" : ""; + std::string fixed; + if (exponent >= static_cast(digits.size()) - 1) + { + std::ostringstream out; + out.imbue(std::locale::classic()); + out.precision(0); + out << std::fixed << v; + fixed = out.str(); + } + else if (exponent >= 0) + { + fixed = sign + digits.substr(0, exponent + 1) + "." + digits.substr(exponent + 1); + } + else + { + fixed = sign + "0." + std::string(-exponent - 1, '0') + digits; + } + return fixed.size() <= scientific.size() ? fixed : scientific; +#endif + } }; /// @brief The cells of one row: a range of CellView. @@ -514,6 +632,16 @@ namespace xl /// @param index Position among the row's cells, below size(). CellView operator[](size_t index) const { + if (views_ != nullptr) + { + const xl_cell_view &view = views_[index]; + const size_t length = view.value_len > 0 ? static_cast(view.value_len) : 0; + return CellView{view.column, static_cast(view.type), + length == 0 || view.value == nullptr + ? std::string_view{} + : std::string_view(reinterpret_cast(view.value), length), + view.has_number != 0 ? std::optional(view.number) : std::nullopt}; + } const xl_row_cell &raw = cells_[index]; const size_t length = raw.value_len > 0 ? static_cast(raw.value_len) : 0; return CellView{raw.column, static_cast(raw.type), @@ -557,11 +685,14 @@ namespace xl private: friend class RowCursor; friend class DecodedRows; + friend class Sheet; friend struct detail::Access; RowView(const xl_row_cell *cells, size_t count) : cells_(cells), count_(count) {} + RowView(const xl_cell_view *views, size_t count) : views_(views), count_(views == nullptr ? 0 : count) {} const xl_row_cell *cells_{}; + const xl_cell_view *views_{}; size_t count_{}; }; @@ -714,6 +845,32 @@ namespace xl VeryHidden = XL_SHEET_VERY_HIDDEN, ///< Hidden and absent from the Unhide dialog; only code can show it. }; + /// @brief A fold over the rows of any sheet, for Sheet::aggregate. + /// + /// `Acc` must be move-constructible; the wrapper returns the surviving accumulator by moving out + /// of it. `combine` must fold `next` into `*this` without destroying it - the library owns + /// `next`'s lifetime and frees it afterwards. + /// + /// `accumulate` runs concurrently on worker threads, one `Acc` per chunk of sheet data (not per + /// thread), never on the calling thread. A chunk can be accumulated and then destroyed without + /// being combined. `combine` is effectively single-threaded but is not pinned to any particular + /// thread and also never runs on the calling thread. `~Acc()` does run on the calling thread, + /// once the run has finished, and must not throw. + /// + /// The `RowView` handed to `accumulate` borrows the library's buffers and is valid only for the + /// duration of that one call; copy anything you keep. A cell carries text, a number, or both. + /// + /// An exception escaping into the library is undefined behavior, so this wrapper catches + /// everything and reports it as an error. XLS, table views and CSV opened from a Source or + /// InputStream read sequentially. See `excelreader.h`'s `xl_aggregate` contract. + /// @tparam Acc The accumulator type. + template + concept Accumulator = + std::move_constructible && requires(Acc acc, Acc &next, RowView row) { + { acc.accumulate(row) } -> std::same_as; + { acc.combine(next) } -> std::same_as; + }; + /// @brief One sheet of a Workbook: the native workbook handle and an index. /// /// Holds no resource of its own, so it is cheap to copy and may be handed to another thread; each @@ -757,6 +914,71 @@ namespace xl return RowCursor(rows); } + /// @brief Calls `f` once per row on the calling thread, without copying rows into a cursor. + /// + /// Each RowView borrows the library's buffers for that one call; copy anything you keep. A + /// cell carries text, a number, or both: CellView::text() and CellView::as_number() cover all + /// three. Returning anything but `XL_OK` stops the walk and comes back as the error's code; + /// use positive codes, since every library status is <= 0. An exception from `f` stops the + /// walk and is reported as an error. Other calls on this workbook are allowed from inside `f`. + /// @param f Called as `int32_t f(RowView)`. + template + requires std::is_invocable_r_v + std::expected for_each_row(F &&f) const + { + if (const auto &abi = detail::check_abi_version(); !abi.has_value()) + { + return std::unexpected(abi.error()); + } + struct Walk + { + std::remove_reference_t *f; + bool threw; + } walk{std::addressof(f), false}; + const xl_row_fn shim = [](const xl_cell_view *cells, int32_t count, void *user_data) noexcept -> int32_t + { + auto *state = static_cast(user_data); + try + { + return (*state->f)(RowView(cells, count > 0 ? static_cast(count) : 0)); + } + catch (...) + { + state->threw = true; + return detail::kCsvCallbackException; + } + }; + const int32_t status = xl_rows_foreach(handle_, index_, shim, &walk); + if (walk.threw) + { + return std::unexpected(Error{detail::kCsvCallbackException, "a row callback threw"}); + } + if (status > 0) + { + return std::unexpected(Error{status, "a row callback returned status " + std::to_string(status)}); + } + if (status != XL_OK) + { + return std::unexpected(detail::make_error(status)); + } + return {}; + } + + /// @brief Folds this sheet into one `Acc` across several threads. + /// + /// Each chunk starts from a copy of `seed`, made on a worker thread; copies are made + /// concurrently from the one `seed`, which the call only reads. The CSV dialect, password and + /// limits come from the OpenOptions the workbook was opened with. + /// @tparam Acc The accumulator; see Accumulator. + /// @param seed The state every chunk starts from. + /// @param header_row 1-based header row, not aggregated; 0 means none. + /// @param degree_of_parallelism Worker threads; 0 uses every processor, 1 runs sequentially. + /// @return The combined accumulator. + template + requires std::copy_constructible + std::expected aggregate(const Acc &seed = Acc{}, int32_t header_row = 0, + int32_t degree_of_parallelism = 0) const; + /// @brief The whole sheet from its first row, decoded in one native call. std::expected read_all() const { @@ -2335,11 +2557,6 @@ namespace xl namespace detail { - /// Status a shim returns for "a callback threw". Positive, so it can never collide with a - /// library status (every one of those is <= 0). It CAN collide with a caller's own - /// `INT32_MAX`, so the entry points key on `threw` rather than on the status alone. - inline constexpr int32_t kCsvCallbackException = (std::numeric_limits::max)(); - /// What the four shims reach through `user_data`. The C ABI shares one `user_data` across /// every worker thread and requires it to be read-only or internally synchronized, so the /// seed is held by pointer-to-const and the "a callback threw" flag is atomic. @@ -2477,6 +2694,51 @@ namespace xl const int32_t status = call(&agg, &raw, &state); return csv_finish(context, status, state); } + + template + int32_t view_accumulate(void *state, const xl_cell_view *cells, int32_t count, void *user_data) noexcept + { + auto *context = static_cast *>(user_data); + try + { + const size_t length = count > 0 ? static_cast(count) : 0; + return static_cast(state)->accumulate(Access::make(cells, length)); + } + catch (...) + { + context->threw.store(true, std::memory_order_relaxed); + return kCsvCallbackException; + } + } + + template + xl_aggregation view_aggregation(CsvContext &context) noexcept + { + xl_aggregation agg{}; + agg.struct_size = static_cast(sizeof(xl_aggregation)); + agg.seed = &csv_seed; + agg.accumulate = &view_accumulate; + agg.combine = &csv_combine; + agg.free_state = &csv_free_state; + agg.user_data = &context; + return agg; + } + } + + template + requires std::copy_constructible + std::expected Sheet::aggregate(const Acc &seed, int32_t header_row, + int32_t degree_of_parallelism) const + { + if (const auto &abi = detail::check_abi_version(); !abi.has_value()) + { + return std::unexpected(abi.error()); + } + detail::CsvContext context{std::addressof(seed), {}}; + const xl_aggregation agg = detail::view_aggregation(context); + void *state = nullptr; + const int32_t status = xl_aggregate(handle_, index_, &agg, header_row, degree_of_parallelism, &state); + return detail::csv_finish(context, status, state); } /// @brief Folds the CSV file at `path` into one `Acc` across several threads. diff --git a/tests/ExcelReader.Benchmarks/Parse/CsvParallelParseBenchmark.cs b/tests/ExcelReader.Benchmarks/Parse/CsvParallelParseBenchmark.cs index e2430e70..eb7b33df 100644 --- a/tests/ExcelReader.Benchmarks/Parse/CsvParallelParseBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Parse/CsvParallelParseBenchmark.cs @@ -15,6 +15,17 @@ public sealed class WideRow public decimal TotalRevenue { get; set; } public int Units { get; set; } } + [StructLayout(LayoutKind.Auto)] + public struct WideRowStruct + { + public string? Region { get; set; } + public string? Country { get; set; } + public DateTime OrderDate { get; set; } + public decimal UnitPrice { get; set; } + public decimal TotalRevenue { get; set; } + public int Units { get; set; } + } + public ref struct WideRowRef { public ReadOnlySpan Region { get; set; } @@ -74,6 +85,22 @@ public async Task ConversionHeavy() return n; } + /// + /// The same parse into a struct with : no object per row, and the + /// corpus's few distinct region and country names are shared, so the GC has almost nothing to collect. + /// + [Benchmark] + public async Task ConversionHeavyStructInterned() + { + long n = 0; + using CsvReader csv = Excel.FromCsvFile(_wide, new ExcelReaderOptions { InternStrings = true }); + await foreach (WideRowStruct row in SheetParallel.ParseAsync(csv.FirstSheet, ExcelParser.FromAttributes(), new SheetParallelOptions { DegreeOfParallelism = Dop })) + { + n += row.Units; + } + return n; + } + [Benchmark] public async Task NarrowInt() { diff --git a/tests/ExcelReader.Benchmarks/Parse/CsvParallelVsSepBenchmark.cs b/tests/ExcelReader.Benchmarks/Parse/CsvParallelVsSepBenchmark.cs index 2174dce9..9006fe5f 100644 --- a/tests/ExcelReader.Benchmarks/Parse/CsvParallelVsSepBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Parse/CsvParallelVsSepBenchmark.cs @@ -67,6 +67,29 @@ private static WideRow ParseWide(SepReader.Row row) }; } + private static NarrowRowStruct ParseNarrowStruct(SepReader.Row row) + { + return new NarrowRowStruct + { + A = row["A"].Parse(), + B = row["B"].Parse(), + C = row["C"].Parse(), + }; + } + + private static WideRowStruct ParseWideStruct(SepReader.Row row) + { + return new WideRowStruct + { + Region = row["Region"].ToString(), + Country = row["Country"].ToString(), + OrderDate = row["OrderDate"].Parse(), + UnitPrice = row["UnitPrice"].Parse(), + TotalRevenue = row["TotalRevenue"].Parse(), + Units = row["Units"].Parse(), + }; + } + [Benchmark(Baseline = true)] public Task ExcelReader_Sequential() { @@ -119,6 +142,89 @@ public long Sep_Parallel() return acc; } + // The *_StructPooled rows give both libraries the same help: rows parsed into a struct, and repeated + // strings shared (InternStrings here, a per-column string pool in Sep). + [Benchmark] + public long ExcelReader_Sequential_StructPooled() + { + long acc = 0; + using CsvReader reader = Excel.FromCsvFile(Path, new ExcelReaderOptions { InternStrings = true }); + if (Corpus == ParallelCorpus.NarrowInt) + { + foreach (NarrowRowStruct row in ExcelParser.FromAttributes().Parse(reader.FirstSheet)) + { + acc += row.A; + } + return acc; + } + foreach (WideRowStruct row in ExcelParser.FromAttributes().Parse(reader.FirstSheet)) + { + acc += row.Units; + } + return acc; + } + + [Benchmark] + public async Task ExcelReader_Parallel_StructPooled() + { + long acc = 0; + using CsvReader csv = Excel.FromCsvFile(Path, new ExcelReaderOptions { InternStrings = true }); + var options = new SheetParallelOptions { DegreeOfParallelism = 0 }; + if (Corpus == ParallelCorpus.NarrowInt) + { + await foreach (NarrowRowStruct row in SheetParallel.ParseAsync(csv.FirstSheet, ExcelParser.FromAttributes(), options)) + { + acc += row.A; + } + return acc; + } + await foreach (WideRowStruct row in SheetParallel.ParseAsync(csv.FirstSheet, ExcelParser.FromAttributes(), options)) + { + acc += row.Units; + } + return acc; + } + + [Benchmark] + public long Sep_Sequential_StructPooled() + { + long acc = 0; + using SepReader reader = Sep.Reader(static o => o with { CreateToString = SepToString.PoolPerCol() }).FromFile(Path); + if (Corpus == ParallelCorpus.NarrowInt) + { + foreach (SepReader.Row row in reader) + { + acc += ParseNarrowStruct(row).A; + } + return acc; + } + foreach (SepReader.Row row in reader) + { + acc += ParseWideStruct(row).Units; + } + return acc; + } + + [Benchmark] + public long Sep_Parallel_StructPooled() + { + long acc = 0; + using SepReader reader = Sep.Reader(static o => o with { CreateToString = SepToString.PoolPerColThreadSafe() }).FromFile(Path); + if (Corpus == ParallelCorpus.NarrowInt) + { + foreach (NarrowRowStruct row in reader.ParallelEnumerate(ParseNarrowStruct)) + { + acc += row.A; + } + return acc; + } + foreach (WideRowStruct row in reader.ParallelEnumerate(ParseWideStruct)) + { + acc += row.Units; + } + return acc; + } + private async Task ReadExcelReaderAsync(int degreeOfParallelism) { if (degreeOfParallelism == 1) diff --git a/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs b/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs index cac206ca..e533240c 100644 --- a/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Parse/ParseBenchmark.cs @@ -54,6 +54,19 @@ public long ExcelParserSync() return acc; } + [Benchmark] + public long ExcelParserSyncInterned() + { + using var ms = new MemoryStream(_workbook, writable: false); + using var reader = Excel.FromXlsx(ms, options: new ExcelReaderOptions { InternStrings = true }); + long acc = 0; + foreach (Record rec in ExcelParser.FromAttributes().Parse(reader.FirstSheet)) + { + acc += Accumulate(rec); + } + return acc; + } + [Benchmark] public long ExcelParserSyncSharedStrings() { diff --git a/tests/ExcelReader.Benchmarks/Read/LargeSheetParallelBenchmark.cs b/tests/ExcelReader.Benchmarks/Read/LargeSheetParallelBenchmark.cs index 899fe36d..9865121e 100644 --- a/tests/ExcelReader.Benchmarks/Read/LargeSheetParallelBenchmark.cs +++ b/tests/ExcelReader.Benchmarks/Read/LargeSheetParallelBenchmark.cs @@ -49,6 +49,7 @@ public void Merge(CsvSum following) private readonly ExcelParser _parser = ExcelParser.FromAttributes(); private readonly ExcelParser _wideParser = ExcelParser.FromAttributes(); private readonly ExcelParser _wideRefParser = ExcelParser.FromAttributes(); + private readonly ExcelParser _wideStructParser = ExcelParser.FromAttributes(); private byte[] _xlsx = []; private byte[] _xlsb = []; private byte[] _csv = []; @@ -149,5 +150,17 @@ public async Task Csv_Parse() } return total; } + + [Benchmark] + public async Task Csv_ParseStructInterned() + { + long total = 0; + using CsvReader csv = Excel.FromCsv(_csv, new ExcelReaderOptions { InternStrings = true }); + await foreach (WideRowStruct row in SheetParallel.ParseAsync(csv.FirstSheet, _wideStructParser, new SheetParallelOptions { DegreeOfParallelism = Dop })) + { + total += row.Units; + } + return total; + } } } diff --git a/tests/ExcelReader.NativeSmoke/smoke.c b/tests/ExcelReader.NativeSmoke/smoke.c index 3e5a9639..5e45c991 100644 --- a/tests/ExcelReader.NativeSmoke/smoke.c +++ b/tests/ExcelReader.NativeSmoke/smoke.c @@ -29,6 +29,14 @@ XL_STATIC_ASSERT(offsetof(xl_row_cell, value_len) == 8, row_cell_value_len); XL_STATIC_ASSERT(offsetof(xl_row_cell, value) == 16, row_cell_value); XL_STATIC_ASSERT(sizeof(xl_row_cell) == 24, row_cell_size); +XL_STATIC_ASSERT(offsetof(xl_cell_view, column) == 0, cell_view_column); +XL_STATIC_ASSERT(offsetof(xl_cell_view, type) == 4, cell_view_type); +XL_STATIC_ASSERT(offsetof(xl_cell_view, value_len) == 8, cell_view_value_len); +XL_STATIC_ASSERT(offsetof(xl_cell_view, has_number) == 12, cell_view_has_number); +XL_STATIC_ASSERT(offsetof(xl_cell_view, value) == 16, cell_view_value); +XL_STATIC_ASSERT(offsetof(xl_cell_view, number) == 24, cell_view_number); +XL_STATIC_ASSERT(sizeof(xl_cell_view) == 32, cell_view_size); + XL_STATIC_ASSERT(offsetof(xl_row, cell_count) == 0, row_cell_count); XL_STATIC_ASSERT(offsetof(xl_row, cells) == 8, row_cells); XL_STATIC_ASSERT(sizeof(xl_row) == 16, row_size); @@ -52,7 +60,9 @@ XL_STATIC_ASSERT(offsetof(xl_write_options, csv_delimiter) == 16, write_options_ XL_STATIC_ASSERT(offsetof(xl_write_options, csv_quote) == 20, write_options_csv_quote); XL_STATIC_ASSERT(offsetof(xl_write_options, date1904) == 24, write_options_date1904); XL_STATIC_ASSERT(offsetof(xl_write_options, use_shared_strings) == 28, write_options_use_shared_strings); -XL_STATIC_ASSERT(sizeof(xl_write_options) == 32, write_options_size); +XL_STATIC_ASSERT(offsetof(xl_write_options, degree_of_parallelism) == 32, write_options_degree_of_parallelism); +XL_STATIC_ASSERT(offsetof(xl_write_options, batch_rows) == 36, write_options_batch_rows); +XL_STATIC_ASSERT(sizeof(xl_write_options) == 40, write_options_size); XL_STATIC_ASSERT(offsetof(xl_column, type) == 0, column_type); XL_STATIC_ASSERT(offsetof(xl_column, length) == 8, column_length); @@ -96,6 +106,8 @@ XL_STATIC_ASSERT(sizeof(struct ArrowArray) == 80, arrow_array_size); XL_STATIC_ASSERT(offsetof(xl_csv_aggregation, seed) == 8, csv_agg_seed); XL_STATIC_ASSERT(sizeof(xl_csv_aggregation) == 48, csv_agg_size); +XL_STATIC_ASSERT(offsetof(xl_aggregation, accumulate) == 16, agg_accumulate); +XL_STATIC_ASSERT(sizeof(xl_aggregation) == 48, agg_size); XL_STATIC_ASSERT(offsetof(xl_csv_parallel_options, max_cell_bytes) == 24, csv_opt_max_cell_bytes); XL_STATIC_ASSERT(sizeof(xl_csv_parallel_options) == 28, csv_opt_size); @@ -1350,6 +1362,90 @@ static int test_open_source(xl_lib_handle lib, const api_t* api, const char* fix return 0; } +typedef int32_t (*xl_rows_foreach_fn)(xl_workbook*, int32_t, xl_row_fn, void*); +typedef int32_t (*xl_aggregate_fn)(xl_workbook*, int32_t, const xl_aggregation*, int32_t, int32_t, void**); + +static int32_t smoke_count_row(const xl_cell_view* cells, int32_t cell_count, void* user_data) +{ + (void)cells; + (void)cell_count; + (*(int64_t*)user_data)++; + return XL_OK; +} + +static int32_t smoke_stop_row(const xl_cell_view* cells, int32_t cell_count, void* user_data) +{ + (void)cells; + (void)cell_count; + (*(int64_t*)user_data)++; + return 7; +} + +static int32_t smoke_view_seed(void** out_state, void* user_data) +{ + (void)user_data; + *out_state = calloc(1, sizeof(int64_t)); + return *out_state != NULL ? XL_OK : 1; +} + +static int32_t smoke_view_accumulate(void* state, const xl_cell_view* cells, int32_t cell_count, void* user_data) +{ + (void)cells; + (void)cell_count; + (void)user_data; + (*(int64_t*)state)++; + return XL_OK; +} + +static int32_t smoke_view_combine(void* accumulator, void* next, void* user_data) +{ + (void)user_data; + *(int64_t*)accumulator += *(int64_t*)next; + return XL_OK; +} + +static void smoke_view_free_state(void* state, void* user_data) +{ + (void)user_data; + free(state); +} + +static int test_callback_reads(xl_lib_handle lib, const api_t* api, const char* fixture) +{ + xl_rows_foreach_fn rows_foreach = (xl_rows_foreach_fn)load_symbol(lib, "xl_rows_foreach"); + CHECK(rows_foreach != NULL, "xl_rows_foreach must be exported"); + xl_aggregate_fn aggregate = (xl_aggregate_fn)load_symbol(lib, "xl_aggregate"); + CHECK(aggregate != NULL, "xl_aggregate must be exported"); + + xl_workbook* handle = NULL; + CHECK(open_fixture(api, fixture, &handle) == XL_OK, "xl_open_file must succeed"); + + int64_t rows = 0; + CHECK(rows_foreach(handle, 0, smoke_count_row, &rows) == XL_OK, "xl_rows_foreach must succeed"); + CHECK(rows == 101, "xl_rows_foreach must see all 101 rows"); + + int64_t calls = 0; + CHECK(rows_foreach(handle, 0, smoke_stop_row, &calls) == 7, "xl_rows_foreach must return the callback's code"); + CHECK(calls == 1, "a non-XL_OK callback code must stop the walk"); + + xl_aggregation agg; + memset(&agg, 0, sizeof(agg)); + agg.struct_size = (int32_t)sizeof(agg); + agg.seed = smoke_view_seed; + agg.accumulate = smoke_view_accumulate; + agg.combine = smoke_view_combine; + agg.free_state = smoke_view_free_state; + + void* state = NULL; + CHECK(aggregate(handle, 0, &agg, 0, 0, &state) == XL_OK && state != NULL, "xl_aggregate must succeed"); + int64_t aggregated = *(int64_t*)state; + free(state); + CHECK(aggregated == 101, "xl_aggregate must accumulate all 101 rows"); + + CHECK(api->close_(handle) == XL_OK, "xl_close must succeed"); + return 0; +} + int main(int argc, char** argv) { const char* library_path = argc > 1 ? argv[1] : EXCELREADER_LIB_PATH_DEFAULT; @@ -1391,6 +1487,7 @@ int main(int argc, char** argv) failures += test_csv_aggregate_file(lib); failures += test_csv_aggregate_quoted_records(lib); failures += test_open_source(lib, &api, fixture_path); + failures += test_callback_reads(lib, &api, fixture_path); if (failures == 0) { diff --git a/tests/ExcelReader.Tests/Native/AggregateNativeTests.cs b/tests/ExcelReader.Tests/Native/AggregateNativeTests.cs new file mode 100644 index 00000000..2d6de90c --- /dev/null +++ b/tests/ExcelReader.Tests/Native/AggregateNativeTests.cs @@ -0,0 +1,228 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Csv; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; +using ExcelReader.Native; +using ExcelReader.Native.Csv; +using ExcelReader.Native.Reading; + +namespace ExcelReader.Tests.Native +{ + public sealed unsafe class AggregateNativeTests + { + private const int Rows = 60_000; + + [StructLayout(LayoutKind.Sequential)] + private struct Partial + { + public long Sum; + public long Rows; + } + + [StructLayout(LayoutKind.Sequential)] + private struct Tally + { + public int Seeds; + public int Frees; + public int Combines; + public int FailAt; + public int Seen; + } + + private static void WriteRows(IWorkbookWriter workbook) + where TSheet : ISheetWriter + where TRow : IRowWriter + { + using TSheet sheet = workbook.AddSheet("S1"); + using (TRow header = sheet.StartRow()) + { + header.Write("id"); + header.Write("label"); + } + for (long index = 1; index <= Rows; index++) + { + using TRow row = sheet.StartRow(); + row.Write(index); + row.Write("filler-value-" + index.ToString(CultureInfo.InvariantCulture)); + } + sheet.End(); + workbook.End(); + } + + private static byte[] Build(int format) + { + using MemoryStream stream = new(); + switch (format) + { + case NativeFormat.Xlsx: + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(stream, leaveOpen: true)) + { + WriteRows(workbook); + } + break; + case NativeFormat.Xlsb: + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(stream, leaveOpen: true)) + { + WriteRows(workbook); + } + break; + case NativeFormat.Xls: + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(stream, leaveOpen: true)) + { + WriteRows(workbook); + } + break; + default: + using (CsvWorkbookWriter workbook = CsvWorkbookWriter.Create(stream, leaveOpen: true)) + { + WriteRows(workbook); + } + break; + } + return stream.ToArray(); + } + + [UnmanagedCallersOnly] + private static int Seed(void** outState, void* userData) + { + Interlocked.Increment(ref ((Tally*)userData)->Seeds); + *outState = NativeMemory.AllocZeroed((nuint)sizeof(Partial)); + return 0; + } + + [UnmanagedCallersOnly] + private static int SumFirstColumn(void* state, NativeCellView* cells, int count, void* userData) + { + Tally* tally = (Tally*)userData; + if (tally->FailAt != 0 && Interlocked.Increment(ref tally->Seen) >= tally->FailAt) + { + return 7; + } + Partial* partial = (Partial*)state; + partial->Rows++; + if (count > 0 && cells[0].Column == 0) + { + NativeCellView cell = cells[0]; + partial->Sum += cell.HasNumber != 0 && cell.ValueLength == 0 + ? (long)cell.Number + : long.Parse(Encoding.UTF8.GetString(cell.Value, cell.ValueLength), CultureInfo.InvariantCulture); + } + return 0; + } + + [UnmanagedCallersOnly] + private static int Combine(void* accumulator, void* next, void* userData) + { + Interlocked.Increment(ref ((Tally*)userData)->Combines); + ((Partial*)accumulator)->Sum += ((Partial*)next)->Sum; + ((Partial*)accumulator)->Rows += ((Partial*)next)->Rows; + return 0; + } + + [UnmanagedCallersOnly] + private static void Free(void* state, void* userData) + { + Interlocked.Increment(ref ((Tally*)userData)->Frees); + NativeMemory.Free(state); + } + + private static NativeCsvAggregationRaw Aggregation(Tally* tally) + { + return new NativeCsvAggregationRaw + { + StructSize = sizeof(NativeCsvAggregationRaw), + Seed = (IntPtr)(delegate* unmanaged)&Seed, + Accumulate = (IntPtr)(delegate* unmanaged)&SumFirstColumn, + Combine = (IntPtr)(delegate* unmanaged)&Combine, + FreeState = (IntPtr)(delegate* unmanaged)&Free, + UserData = (IntPtr)tally, + }; + } + + [Theory] + [InlineData(NativeFormat.Xlsx, true)] + [InlineData(NativeFormat.Xlsb, true)] + [InlineData(NativeFormat.Xls, false)] + [InlineData(NativeFormat.Csv, false)] + public void AggregateWorkbook_Should_Sum_Every_Data_Row_Once_And_Balance_Frees(int format, bool partitions) + { + Assert.Equal(NativeStatus.Ok, ReadApi.OpenMemory(Build(format), format, out NativeHandle? handle)); + try + { + Tally tally = default; + int status = CsvAggregateApi.AggregateWorkbook(handle, 0, Aggregation(&tally), 1, 4, out nint result); + + Assert.Equal(NativeStatus.Ok, status); + Partial* total = (Partial*)result; + Assert.Equal((long)Rows, total->Rows); + Assert.Equal((long)Rows * (Rows + 1) / 2, total->Sum); + Assert.Equal(tally.Seeds - 1, tally.Frees); + if (partitions) + { + Assert.True(tally.Combines > 0, "the fixture must partition for this format"); + } + NativeMemory.Free(total); + } + finally + { + ReadApi.Close(handle); + } + } + + [Fact] + public void AggregateWorkbook_Should_Return_The_Callers_Code_And_Free_Every_State() + { + Assert.Equal(NativeStatus.Ok, ReadApi.OpenMemory(Build(NativeFormat.Xlsx), NativeFormat.Xlsx, out NativeHandle? handle)); + try + { + Tally tally = new() { FailAt = 20_000 }; + int status = CsvAggregateApi.AggregateWorkbook(handle, 0, Aggregation(&tally), 1, 4, out nint result); + + Assert.Equal(7, status); + Assert.Equal(0, result); + Assert.Equal(tally.Seeds, tally.Frees); + } + finally + { + ReadApi.Close(handle); + } + } + + [Fact] + public void AggregateWorkbook_Should_Reject_A_Negative_Option() + { + Assert.Equal(NativeStatus.Ok, ReadApi.OpenMemory(Build(NativeFormat.Csv), NativeFormat.Csv, out NativeHandle? handle)); + try + { + Tally tally = default; + Assert.Equal(NativeStatus.InvalidArgument, CsvAggregateApi.AggregateWorkbook(handle, 0, Aggregation(&tally), -1, 0, out _)); + Assert.Equal(NativeStatus.InvalidArgument, CsvAggregateApi.AggregateWorkbook(handle, 0, Aggregation(&tally), 0, -1, out _)); + Assert.Equal(0, tally.Seeds); + } + finally + { + ReadApi.Close(handle); + } + } + + [Fact] + public void AggregateWorkbook_Should_Reject_A_Null_Handle() + { + Tally tally = default; + Assert.Equal(NativeStatus.InvalidHandle, CsvAggregateApi.AggregateWorkbook(null, 0, Aggregation(&tally), 0, 0, out _)); + } + + [Fact] + public void Aggregate_Export_Should_Reject_A_Null_Aggregation() + { + void* state; + int status = ((delegate* unmanaged)&Exports.Aggregate)(0, 0, null, 0, 0, &state); + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Contains("aggregation must not be NULL", NativeApi.LastErrorText(), StringComparison.Ordinal); + } + } +} diff --git a/tests/ExcelReader.Tests/Native/NativeApiConcurrencyTests.cs b/tests/ExcelReader.Tests/Native/NativeApiConcurrencyTests.cs index a8d987b2..52304d3a 100644 --- a/tests/ExcelReader.Tests/Native/NativeApiConcurrencyTests.cs +++ b/tests/ExcelReader.Tests/Native/NativeApiConcurrencyTests.cs @@ -67,7 +67,7 @@ public Task LastErrorIsPerThreadNotSharedAcrossHandles() return Task.WhenAll(tasks); } - private static List ReadSheet(NativeHandle handle, int sheet) + internal static List ReadSheet(NativeHandle handle, int sheet) { Assert.Equal(NativeStatus.Ok, ReadApi.OpenRows(handle, sheet, out NativeRowCursor? opened)); using NativeRowCursor cursor = opened!; diff --git a/tests/ExcelReader.Tests/Native/NativeApiTests.ForEach.cs b/tests/ExcelReader.Tests/Native/NativeApiTests.ForEach.cs new file mode 100644 index 00000000..c178c267 --- /dev/null +++ b/tests/ExcelReader.Tests/Native/NativeApiTests.ForEach.cs @@ -0,0 +1,126 @@ +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using ExcelReader.Native; +using ExcelReader.Native.Reading; + +namespace ExcelReader.Tests.Native +{ + public sealed partial class NativeApiTests + { + private static readonly string XlsFixture = Path.Combine(AppContext.BaseDirectory, "data", "excel-formula-strings.xls"); + + private sealed record BorrowedCell(int Column, int Type, string Text, bool HasNumber, double Number); + + private sealed class ForEachSink + { + internal readonly List> Rows = []; + internal int StopAfter = int.MaxValue; + } + + [UnmanagedCallersOnly] + private static unsafe int Collect(NativeCellView* cells, int count, void* userData) + { + ForEachSink sink = (ForEachSink)GCHandle.FromIntPtr((nint)userData).Target!; + List row = new(count); + for (int index = 0; index < count; index++) + { + NativeCellView cell = cells[index]; + string text = cell.ValueLength == 0 ? "" : Encoding.UTF8.GetString(cell.Value, cell.ValueLength); + row.Add(new BorrowedCell(cell.Column, cell.Type, text, cell.HasNumber != 0, cell.Number)); + } + sink.Rows.Add(row); + return sink.Rows.Count >= sink.StopAfter ? 42 : NativeStatus.Ok; + } + + private static unsafe int ForEach(NativeHandle? handle, ForEachSink sink) + { + GCHandle pinned = GCHandle.Alloc(sink); + try + { + return ReadApi.ForEachRow(handle, 0, &Collect, (void*)GCHandle.ToIntPtr(pinned)); + } + finally + { + pinned.Free(); + } + } + + [Theory] + [InlineData(NativeFormat.Xlsx)] + [InlineData(NativeFormat.Xlsb)] + [InlineData(NativeFormat.Xls)] + public void ForEachRow_Should_Match_NextRowView_On_Every_Row(int format) + { + string fixture = format switch + { + NativeFormat.Xlsb => XlsbFixture, + NativeFormat.Xls => XlsFixture, + _ => XlsxFixture, + }; + Assert.Equal(NativeStatus.Ok, OpenPath(fixture, format, out NativeHandle? handle)); + try + { + ForEachSink sink = new(); + Assert.Equal(NativeStatus.Ok, ForEach(handle, sink)); + + using NativeRowCursor cursor = OpenRows(handle); + foreach (List borrowed in sink.Rows) + { + Assert.Equal(NativeStatus.Ok, ReadApi.NextRowView(cursor, out NativeRow row)); + List expected = ReadView(row); + Assert.Equal(expected.Count, borrowed.Count); + for (int index = 0; index < expected.Count; index++) + { + BorrowedCell cell = borrowed[index]; + Assert.Equal(expected[index].Column, cell.Column); + Assert.Equal(expected[index].Type, cell.Type); + if (cell.Text.Length == 0 && cell.HasNumber) + { + Assert.Equal(double.Parse(expected[index].Value, CultureInfo.InvariantCulture), cell.Number); + } + else + { + Assert.Equal(expected[index].Value, cell.Text); + } + } + } + Assert.True(sink.Rows.Count > 1); + Assert.Equal(NativeStatus.Eof, ReadApi.NextRowView(cursor, out _)); + } + finally + { + ReadApi.Close(handle); + } + } + + [Fact] + public void ForEachRow_Should_Stop_And_Return_The_Callback_Status() + { + Assert.Equal(NativeStatus.Ok, OpenPath(XlsxFixture, NativeFormat.Xlsx, out NativeHandle? handle)); + try + { + ForEachSink sink = new() { StopAfter = 2 }; + Assert.Equal(42, ForEach(handle, sink)); + Assert.Equal(2, sink.Rows.Count); + } + finally + { + ReadApi.Close(handle); + } + } + + [Fact] + public void ForEachRow_Should_Reject_A_Null_Handle() + { + Assert.Equal(NativeStatus.InvalidHandle, ForEach(null, new ForEachSink())); + } + + [Fact] + public unsafe void ForEachRow_Export_Should_Reject_A_Null_Callback() + { + int status = ((delegate* unmanaged, void*, int>)&Exports.RowsForEach)(0, 0, null, null); + Assert.Equal(NativeStatus.InvalidArgument, status); + } + } +} diff --git a/tests/ExcelReader.Tests/Native/NativeApiTests.Writer.cs b/tests/ExcelReader.Tests/Native/NativeApiTests.Writer.cs index 5c4cd832..6e074684 100644 --- a/tests/ExcelReader.Tests/Native/NativeApiTests.Writer.cs +++ b/tests/ExcelReader.Tests/Native/NativeApiTests.Writer.cs @@ -4,6 +4,7 @@ using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Csv; using ExcelReader.Native; using ExcelReader.Native.Reading; @@ -450,6 +451,83 @@ public void WriteOptions_Should_Decode_An_All_Defaults_Struct() Assert.Null(options.CsvQuote); Assert.Null(options.Date1904); Assert.Null(options.UseSharedStrings); + Assert.Equal(0, options.DegreeOfParallelism); + Assert.Equal(0, options.BatchRows); + } + + [Fact] + public unsafe void WriteOptions_Should_Read_A_Null_Pointer_As_Serial() + { + Assert.True(NativeWriteOptionsRaw.TryRead(null, out NativeWriteOptionsRaw raw, out _)); + Assert.Equal(sizeof(NativeWriteOptionsRaw), raw.StructSize); + Assert.Equal(1, raw.DegreeOfParallelism); + } + + [Fact] + public unsafe void WriteOptions_Should_Read_The_7_0_Layout_As_Serial_Without_Reading_Past_It() + { + byte* buffer = stackalloc byte[sizeof(NativeWriteOptionsRaw)]; + new Span(buffer, sizeof(NativeWriteOptionsRaw)).Fill(0xFF); + new Span(buffer, NativeWriteOptionsRaw.V7Size).Clear(); + ((NativeWriteOptionsRaw*)buffer)->StructSize = NativeWriteOptionsRaw.V7Size; + + Assert.True(NativeWriteOptionsRaw.TryRead((NativeWriteOptionsRaw*)buffer, out NativeWriteOptionsRaw raw, out _)); + Assert.Equal(1, raw.DegreeOfParallelism); + Assert.Equal(0, raw.BatchRows); + Assert.Equal(sizeof(NativeWriteOptionsRaw), raw.StructSize); + } + + [Fact] + public unsafe void WriteOptions_Should_Reject_A_Size_That_Is_Neither_Layout() + { + NativeWriteOptionsRaw raw = DefaultWriteOptionsRaw(); + raw.StructSize = 36; + + Assert.False(NativeWriteOptionsRaw.TryRead(&raw, out _, out string? error)); + Assert.Contains("struct_size", error, StringComparison.Ordinal); + Assert.Contains("32", error, StringComparison.Ordinal); + } + + [Theory] + [InlineData(-1, 0, "degree_of_parallelism")] + [InlineData(0, -1, "batch_rows")] + public void WriteOptions_Should_Reject_Negative_Parallel_Fields(int degree, int batchRows, string field) + { + NativeWriteOptionsRaw raw = DefaultWriteOptionsRaw(); + raw.DegreeOfParallelism = degree; + raw.BatchRows = batchRows; + + Assert.False(NativeWriteOptions.TryDecode(raw, null, out _, out string? error)); + Assert.Contains(field, error, StringComparison.Ordinal); + } + + [Theory] + [InlineData(NativeFormat.Xlsx, 0, 0, 0, 8192)] + [InlineData(NativeFormat.Xlsb, 4, 100, 4, 100)] + public void WriteOptions_Should_Map_Parallel_Fields_For_Xlsx_And_Xlsb(int format, int degree, int batchRows, int expectedDegree, int expectedBatch) + { + NativeWriteOptionsRaw raw = DefaultWriteOptionsRaw(); + raw.DegreeOfParallelism = degree; + raw.BatchRows = batchRows; + + Assert.True(NativeWriteOptions.TryDecode(raw, null, out NativeWriteOptions options, out _)); + ParallelWriteOptions? parallel = options.ToParallelWriteOptions(format); + Assert.NotNull(parallel); + Assert.Equal(expectedDegree, parallel.DegreeOfParallelism); + Assert.Equal(expectedBatch, parallel.BatchRows); + } + + [Theory] + [InlineData(NativeFormat.Xlsx, 1)] + [InlineData(NativeFormat.Xls, 4)] + [InlineData(NativeFormat.Csv, 0)] + public void WriteOptions_Should_Stay_Serial_For_Degree_One_Or_Other_Formats(int format, int degree) + { + NativeWriteOptionsRaw raw = DefaultWriteOptionsRaw(); + raw.DegreeOfParallelism = degree; + + Assert.True(NativeWriteOptions.TryDecode(raw, null, out NativeWriteOptions options, out _)); + Assert.Null(options.ToParallelWriteOptions(format)); } [Fact] @@ -1176,5 +1254,179 @@ public void WriteTyped_Should_Use_The_Requested_Sheet_Name() File.Delete(path); } } + + private static NativeColumnSpec[] EveryTypeSpecs() + { + return + [ + new() { Names = ["text"], Type = NativeColumnType.String }, + new() { Names = ["int"], Type = NativeColumnType.Int64 }, + new() { Names = ["float"], Type = NativeColumnType.Float64 }, + new() { Names = ["bool"], Type = NativeColumnType.Bool }, + new() { Names = ["date"], Type = NativeColumnType.Date }, + new() { Names = ["time"], Type = NativeColumnType.Time }, + new() { Names = ["stamp"], Type = NativeColumnType.Timestamp }, + ]; + } + + private static NativeTable BuildEveryTypeTable(int rowCount, long[]? times = null) + { + string[] texts = [.. Enumerable.Range(0, rowCount).Select(i => i % 2 == 0 ? "repeated" : $"text {i}")]; + byte[] data = Encoding.UTF8.GetBytes(string.Concat(texts)); + int[] offsets = new int[rowCount + 1]; + byte[] validity = new byte[(rowCount + 7) / 8]; + for (int i = 0; i < rowCount; i++) + { + offsets[i + 1] = offsets[i] + Encoding.UTF8.GetByteCount(texts[i]); + if (i is not 0 and not 2 && i != rowCount - 1) + { + validity[i >> 3] |= (byte)(1 << (i & 7)); + } + } + IEnumerable rows = Enumerable.Range(0, rowCount); + NativeColumn[] columns = + [ + BuildColumn(NativeColumnType.String, offsets, validity, data), + BuildColumn(NativeColumnType.Int64, [.. rows.Select(i => i * 10L)], validity), + BuildColumn(NativeColumnType.Float64, [.. rows.Select(i => i + 0.5)], validity), + BuildColumn(NativeColumnType.Bool, [.. rows.Select(i => (byte)(i & 1))], validity), + BuildColumn(NativeColumnType.Date, [.. rows.Select(i => 19000 + i)], validity), + BuildColumn(NativeColumnType.Time, times ?? [.. rows.Select(i => i * 1_000_000L)], validity), + BuildColumn(NativeColumnType.Timestamp, [.. rows.Select(i => 1_700_000_000_000_000L + (i * 1_000_000L))], validity), + ]; + for (int index = 0; index < columns.Length; index++) + { + columns[index].Length = rowCount; + } + + int size = Marshal.SizeOf(); + IntPtr block = Marshal.AllocHGlobal(size * columns.Length); + for (int index = 0; index < columns.Length; index++) + { + Marshal.StructureToPtr(columns[index], IntPtr.Add(block, index * size), false); + } + return new NativeTable { ColumnCount = columns.Length, RowCount = rowCount, Columns = block }; + } + + private static NativeColumn BuildColumn(int type, T[] values, byte[] validity, byte[]? data = null) + where T : unmanaged + { + return new NativeColumn + { + Type = type, + Values = CopyToNative(values), + Validity = CopyToNative(validity), + Data = data is null ? IntPtr.Zero : CopyToNative(data), + DataLen = data?.Length ?? 0, + }; + } + + private static unsafe IntPtr CopyToNative(T[] values) + where T : unmanaged + { + if (values.Length == 0) + { + return IntPtr.Zero; + } + IntPtr block = Marshal.AllocHGlobal(values.Length * sizeof(T)); + values.AsSpan().CopyTo(new Span((void*)block, values.Length)); + return block; + } + + private static NativeWriteOptions ParallelWriteOptionsFor(int degree, int batchRows = 0, bool sharedStrings = false) + { + NativeWriteOptionsRaw raw = DefaultWriteOptionsRaw(); + raw.DegreeOfParallelism = degree; + raw.BatchRows = batchRows; + raw.UseSharedStrings = sharedStrings ? NativeOptionState.True : NativeOptionState.Default; + Assert.True(NativeWriteOptions.TryDecode(raw, null, out NativeWriteOptions options, out _)); + return options; + } + + private static List WriteAndReadBack(int format, NativeTable table, NativeWriteOptions options) + { + Assert.Equal(NativeStatus.Ok, WriteApi.WriteTypedToMemory(format, EveryTypeSpecs(), table, options, out byte[]? bytes)); + Assert.Equal(NativeStatus.Ok, ReadApi.OpenMemory(bytes!, format, out NativeHandle? handle)); + using NativeHandle live = handle!; + return NativeApiConcurrencyTests.ReadSheet(live, 0); + } + + [Theory] + [InlineData(NativeFormat.Xlsx, false)] + [InlineData(NativeFormat.Xlsx, true)] + [InlineData(NativeFormat.Xlsb, false)] + [InlineData(NativeFormat.Xlsb, true)] + public void WriteTyped_Parallel_Should_Read_Back_Like_Serial(int format, bool sharedStrings) + { + NativeTable table = BuildEveryTypeTable(7); + try + { + List serial = WriteAndReadBack(format, table, ParallelWriteOptionsFor(1, sharedStrings: sharedStrings)); + List parallel = WriteAndReadBack(format, table, ParallelWriteOptionsFor(4, 2, sharedStrings)); + + Assert.Equal(8, serial.Count); + Assert.Equal(serial, parallel); + } + finally + { + FreeBuiltTable(ref table); + } + } + + [Theory] + [InlineData(NativeFormat.Xls)] + [InlineData(NativeFormat.Csv)] + public void WriteTyped_Should_Ignore_Parallel_Fields_For_Xls_And_Csv(int format) + { + NativeTable table = BuildEveryTypeTable(7); + try + { + Assert.Equal(NativeStatus.Ok, WriteApi.WriteTypedToMemory(format, EveryTypeSpecs(), table, ParallelWriteOptionsFor(1), out byte[]? serial)); + Assert.Equal(NativeStatus.Ok, WriteApi.WriteTypedToMemory(format, EveryTypeSpecs(), table, ParallelWriteOptionsFor(4, 2), out byte[]? parallel)); + + Assert.Equal(serial, parallel); + } + finally + { + FreeBuiltTable(ref table); + } + } + + [Theory] + [InlineData(NativeFormat.Xlsx)] + [InlineData(NativeFormat.Xlsb)] + public void WriteTyped_Parallel_Should_Write_A_Header_Only_Sheet_For_Zero_Rows(int format) + { + NativeTable table = BuildEveryTypeTable(0); + try + { + List rows = WriteAndReadBack(format, table, ParallelWriteOptionsFor(4)); + + Assert.Equal(["text|int|float|bool|date|time|stamp|"], rows); + } + finally + { + FreeBuiltTable(ref table); + } + } + + [Theory] + [InlineData(NativeFormat.Xlsx)] + [InlineData(NativeFormat.Xlsb)] + public void WriteTyped_Parallel_Should_Report_A_Failing_Batch_As_An_Error(int format) + { + NativeTable table = BuildEveryTypeTable(7, [0, 0, 0, 0, 0, long.MaxValue, 0]); + try + { + Assert.Equal(NativeStatus.Error, WriteApi.WriteTypedToMemory(format, EveryTypeSpecs(), table, ParallelWriteOptionsFor(4, 2), out byte[]? bytes)); + + Assert.Null(bytes); + Assert.NotEmpty(NativeApi.LastErrorText()); + } + finally + { + FreeBuiltTable(ref table); + } + } } } diff --git a/tests/ExcelReader.Tests/Native/NativeWriterHandleTests.cs b/tests/ExcelReader.Tests/Native/NativeWriterHandleTests.cs index e7892075..ca327f11 100644 --- a/tests/ExcelReader.Tests/Native/NativeWriterHandleTests.cs +++ b/tests/ExcelReader.Tests/Native/NativeWriterHandleTests.cs @@ -27,6 +27,16 @@ private static int OpenWriteHandleToMemory(int format, out NativeWriterHandle? h return WriteApi.OpenWriteHandleToMemory(format, options, out handle); } + [Fact] + public void OpenWriteHandle_Should_Return_An_Error_When_The_File_Cannot_Be_Created() + { + string path = Path.Combine(Path.GetTempPath(), $"excelreader-missing-{Guid.NewGuid():N}", "out.xlsx"); + + Assert.Equal(NativeStatus.Error, OpenWriteHandle(path, NativeFormat.Xlsx, out NativeWriterHandle? handle)); + Assert.Null(handle); + Assert.NotEmpty(NativeApi.LastErrorText()); + } + [Theory] [InlineData(NativeFormat.Xlsx, "xlsx")] [InlineData(NativeFormat.Xlsb, "xlsb")] diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvChunkWorkerTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvChunkWorkerTests.cs index 99c9bde8..04a46369 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvChunkWorkerTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/CsvChunkWorkerTests.cs @@ -31,6 +31,7 @@ private static async Task> ParseAsync(byte[] csv, long start CsvReaderOptions.Default, new ExcelParserOptions(), [], + contentCache: null, CancellationToken.None); } @@ -134,6 +135,7 @@ public async Task ParsesTheSameRecordFromAFileBackedSourceAsFromMemory() CsvReaderOptions.Default, new ExcelParserOptions(), [], + contentCache: null, TestContext.Current.CancellationToken); Row row = Assert.Single(result.Models); diff --git a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs index 47f82a13..0a2aeec6 100644 --- a/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs +++ b/tests/ExcelReader.Tests/Parser/ParallelCsv/ParseCsvParallelTests.cs @@ -52,6 +52,41 @@ public async Task MemoryOverloadMatchesTheSequentialParser() } } + [Fact] + public async Task AFailureAfterBlankLinesReportsTheSameRowAsTheSequentialParser() + { + var sb = new StringBuilder("Name,Age\n"); + for (int i = 0; i < 30_000; i++) + { + sb.Append(i == 25_000 ? "bad,x\n" : $"name{i},{i}\n"); + if (i % 997 == 0) + { + sb.Append('\n'); + } + } + byte[] csv = Encoding.UTF8.GetBytes(sb.ToString()); + var options = new ExcelParserOptions { ThrowOnParseFailure = true }; + + using CsvReader sequentialSource = Excel.FromCsv(csv); + ExcelParseException sequential = Assert.Throws( + () => ExcelParser.FromAttributes(options).Parse(sequentialSource.FirstSheet).ToList()); + using CsvReader parallelSource = Excel.FromCsv(csv); + ExcelParseException parallel = await Assert.ThrowsAsync( + () => DrainAsync(SheetParallel.ParseAsync(parallelSource.FirstSheet, ExcelParser.FromAttributes(options), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken))); + + Assert.Equal(sequential.Row, parallel.Row); + } + + [Fact] + public async Task EnumeratingTheSameResultTwiceYieldsEveryRowBothTimes() + { + using CsvReader source = Excel.FromCsv(BuildCsv(20_000)); + IAsyncEnumerable rows = SheetParallel.ParseAsync(source.FirstSheet, ExcelParser.FromAttributes(), options: new SheetParallelOptions { DegreeOfParallelism = 4 }, ct: TestContext.Current.CancellationToken); + + Assert.Equal(20_000, (await DrainAsync(rows).WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)).Count); + Assert.Equal(20_000, (await DrainAsync(rows).WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken)).Count); + } + [Fact] public async Task FileOverloadMatchesTheSequentialParser() { diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs index f09a02ec..c395aac6 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs @@ -136,7 +136,7 @@ public void RecordsOfVaryingWidthKeepTheirFieldsAndStartOffsets() var rows = new List(); var starts = new List(); - using var e = new CsvReader.Enumerator(bytes, CsvReaderOptions.Default); + using var e = new CsvReader.Enumerator(bytes, CsvReaderOptions.Default, contentCache: null, TestContext.Current.CancellationToken); while (e.MoveNext()) { starts.Add(e.CurrentRecordStart); diff --git a/tests/ExcelReader.Tests/Reader/FastDoubleTests.cs b/tests/ExcelReader.Tests/Reader/FastDoubleTests.cs index c904cfa1..8a7336a6 100644 --- a/tests/ExcelReader.Tests/Reader/FastDoubleTests.cs +++ b/tests/ExcelReader.Tests/Reader/FastDoubleTests.cs @@ -44,6 +44,13 @@ private static void AssertMatchesDoubleTryParse(string text) [InlineData("123456789.123456")] [InlineData("000000000000000123")] [InlineData("0.00000000000000123")] + [InlineData("1.1234")] + [InlineData("1.1234567")] + [InlineData("1.12345678")] + [InlineData("1.123456789")] + [InlineData("-0.123456789012")] + [InlineData("0.1234567890123456")] + [InlineData("2.99999999")] public void AcceptsAndMatchesPlainDecimals(string text) { AssertMatchesDoubleTryParse(text); @@ -66,6 +73,8 @@ public void AcceptsAndMatchesPlainDecimals(string text) [InlineData("1e22")] [InlineData("1e-22")] [InlineData("-0e5")] + [InlineData("1.12345678e2")] + [InlineData("1.1234E-3")] public void AcceptsAndMatchesScientificNotation(string text) { AssertMatchesDoubleTryParse(text); @@ -101,6 +110,10 @@ public void AcceptsAndMatchesScientificNotation(string text) [InlineData("1e-400")] [InlineData("5E-324")] [InlineData("1.7976931348623157E+308")] + [InlineData("1.1234567x")] + [InlineData("1.12345678x")] + [InlineData("1.123x5678")] + [InlineData("1.12/4")] public void RejectsUnsupportedForms(string text) { byte[] utf8 = Encoding.ASCII.GetBytes(text); @@ -181,6 +194,31 @@ public void AcceptsShortDecimalsBitExactly() } } + [Theory] + [InlineData("999999999999999")] + [InlineData("000000000000000123")] + [InlineData("0.00000000000000123")] + [InlineData("-0.000000000000000001")] + [InlineData("1234567.12345678")] + public void LosslessAcceptsUpToFifteenSignificantDigits(string text) + { + byte[] utf8 = Encoding.ASCII.GetBytes(text); + Assert.True(FastDouble.TryParseLossless(utf8, out double actual), $"Expected TryParseLossless to accept \"{text}\"."); + Assert.Equal(BitConverter.DoubleToInt64Bits(double.Parse(text, CultureInfo.InvariantCulture)), BitConverter.DoubleToInt64Bits(actual)); + } + + [Theory] + [InlineData("1234567890123456")] + [InlineData("152.58000000000001")] + [InlineData("-12345678.12345678")] + [InlineData(".1234567890123456")] + [InlineData("0.1234567890123456")] + [InlineData("000.0001234567890123456")] + public void LosslessRejectsMoreThanFifteenSignificantDigits(string text) + { + Assert.False(FastDouble.TryParseLossless(Encoding.ASCII.GetBytes(text), out _), $"Expected TryParseLossless to reject \"{text}\"."); + } + [Fact] public void NegativeZeroMatchesSignBit() { diff --git a/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs b/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs index babf22dd..1107dc47 100644 --- a/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs +++ b/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs @@ -739,6 +739,17 @@ public void MiniStreamChainWithACycleThrowsInvalidDataInsteadOfRepeatingSectors( CfbContainer.ReadMiniStream(miniStream, miniFat, miniSectorSize: 64, startSector: 0, size: 256)); } + [Theory] + [InlineData(-2)] + [InlineData(-1)] + public void MiniStreamChainEndingBeforeItsSizeThrowsInvalidDataInsteadOfZeroFilling(int end) + { + byte[] miniStream = new byte[256]; + int[] miniFat = [end, -2, -2, -2]; + Assert.Throws(() => + CfbContainer.ReadMiniStream(miniStream, miniFat, miniSectorSize: 64, startSector: 0, size: 128)); + } + [Fact] public void SyntheticXlsbTableParses() { diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs index 04dd0027..3f0916b0 100644 --- a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs @@ -29,6 +29,17 @@ internal static class XlsWorkbookBuilder private const int FreeSector = unchecked((int)0xFFFFFFFF); internal static MemoryStream Build(bool date1904 = false, string? customDateFormat = null, byte[]? sheetStates = null, params (string Name, object?[][] Rows)[] sheets) + { + return BuildOle(BuildWorkbookStream(date1904, customDateFormat, sheetStates, sheets)); + } + + /// A version 4 compound file uses 4096-byte sectors, and its header takes a whole sector. + internal static MemoryStream BuildWithSectorSize(int sectorSize, params (string Name, object?[][] Rows)[] sheets) + { + return BuildOle(BuildWorkbookStream(false, null, null, sheets), sectorSize); + } + + private static byte[] BuildWorkbookStream(bool date1904, string? customDateFormat, byte[]? sheetStates, (string Name, object?[][] Rows)[] sheets) { List sharedStrings = []; Dictionary sharedIndexes = new(StringComparer.Ordinal); @@ -45,8 +56,7 @@ internal static MemoryStream Build(bool date1904 = false, string? customDateForm { workbook.Write(sheet); } - - return BuildOle(workbook.ToArray()); + return workbook.ToArray(); } internal static MemoryStream BuildNonBiff8() @@ -481,17 +491,17 @@ private static bool CanCompress(string value) return true; } - private static MemoryStream BuildOle(byte[] workbook) + private static MemoryStream BuildOle(byte[] workbook, int sectorSize = SectorSize) { - int workbookSize = Math.Max(RoundUp(workbook.Length, SectorSize), 4096 + SectorSize); - int workbookSectorCount = workbookSize / SectorSize; + int workbookSize = Math.Max(RoundUp(workbook.Length, sectorSize), 4096 + sectorSize); + int workbookSectorCount = workbookSize / sectorSize; int totalSectors = 2 + workbookSectorCount; - if (totalSectors > 128) + if (totalSectors > sectorSize / 4) { throw new NotSupportedException("Test workbook too large for the simple OLE builder."); } - byte[] fat = [.. Enumerable.Repeat(FreeSector, 128).SelectMany(I32)]; + byte[] fat = [.. Enumerable.Repeat(FreeSector, sectorSize / 4).SelectMany(I32)]; WriteI32(fat, 0, FatSector); WriteI32(fat, 4, EndOfChain); for (int i = 0; i < workbookSectorCount; i++) @@ -502,23 +512,23 @@ private static MemoryStream BuildOle(byte[] workbook) } MemoryStream ole = new(); - ole.Write(Header()); + ole.Write(Header(sectorSize)); ole.Write(fat); - ole.Write(Directory(workbookSize)); + ole.Write(Directory(workbookSize, sectorSize)); ole.Write(workbook); ole.Write(new byte[workbookSize - workbook.Length]); ole.Position = 0; return ole; } - private static byte[] Header() + private static byte[] Header(int sectorSize) { - byte[] header = new byte[SectorSize]; + byte[] header = new byte[sectorSize]; byte[] sig = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; sig.CopyTo(header, 0); WriteU16(header, 0x1A, 0x003E); WriteU16(header, 0x1C, 0x0003); - WriteU16(header, 0x1E, 9); + WriteU16(header, 0x1E, (ushort)System.Numerics.BitOperations.Log2((uint)sectorSize)); WriteU16(header, 0x20, 6); WriteI32(header, 0x2C, 1); WriteI32(header, 0x30, 1); @@ -535,9 +545,9 @@ private static byte[] Header() return header; } - private static byte[] Directory(int workbookSize) + private static byte[] Directory(int workbookSize, int sectorSize) { - byte[] dir = new byte[SectorSize]; + byte[] dir = new byte[sectorSize]; WriteDirectoryEntry(dir.AsSpan(0, 128), "Root Entry", 5, EndOfChain, 0, child: 1); WriteDirectoryEntry(dir.AsSpan(128, 128), "Workbook", 2, 2, workbookSize, child: EndOfChain); return dir; diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs index 514e1d66..55e4d184 100644 --- a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs @@ -1,3 +1,4 @@ +using System.Buffers.Binary; using System.Collections; using ExcelReader.Core.Parser; using ExcelReader.Core.Reader; @@ -43,6 +44,63 @@ public void ReadsBasicBiff8Cells() Assert.False(e.MoveNext()); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadsAVersion4CompoundFileWith4096ByteSectors(bool fromFile) + { + object?[][] rows = [.. Enumerable.Range(0, 600).Select(static i => new object?[] { $"row {i}", i })]; + using MemoryStream built = XlsWorkbookBuilder.BuildWithSectorSize(4096, ("S1", rows)); + string path = Path.Combine(Path.GetTempPath(), $"excelreader-v4-{Guid.NewGuid():N}.xls"); + File.WriteAllBytes(path, built.ToArray()); + try + { + using XlsWorkbook reader = fromFile ? Excel.FromXlsFile(path) : Excel.FromXls(built.ToArray()); + using var e = reader.FirstSheet.GetEnumerator(); + for (int i = 0; i < rows.Length; i++) + { + Assert.True(e.MoveNext()); + Assert.Equal($"row {i}", e.Current[0].GetString()); + Assert.True(e.Current[1].TryParse(null, out int value)); + Assert.Equal(i, value); + } + Assert.False(e.MoveNext()); + } + finally + { + File.Delete(path); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void ReadsAWorkbookStreamWhoseLastSectorIsNotPadded(bool fromFile) + { + using MemoryStream built = XlsWorkbookBuilder.Build(sheets: [("S1", [["a", 1], ["b", 2]])]); + const int Cut = 100; + const int WorkbookSizeOffset = 512 + 512 + 128 + 120; + byte[] bytes = built.ToArray()[..^Cut]; + long size = BinaryPrimitives.ReadInt64LittleEndian(bytes.AsSpan(WorkbookSizeOffset)); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(WorkbookSizeOffset), size - Cut); + string path = Path.Combine(Path.GetTempPath(), $"excelreader-unpadded-{Guid.NewGuid():N}.xls"); + File.WriteAllBytes(path, bytes); + try + { + using XlsWorkbook reader = fromFile ? Excel.FromXlsFile(path) : Excel.FromXls(bytes); + using var e = reader.FirstSheet.GetEnumerator(); + Assert.True(e.MoveNext()); + Assert.Equal("a", e.Current[0].GetString()); + Assert.True(e.MoveNext()); + Assert.Equal("b", e.Current[0].GetString()); + Assert.False(e.MoveNext()); + } + finally + { + File.Delete(path); + } + } + [Fact] public void AFormulaWithADateStyleReadsAsADate() { diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs index ea69a267..24765dad 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/NamespacePrefixAndIsoDateTests.cs @@ -178,6 +178,46 @@ public void IsoDateAtOaDateFloorIsStillParsedAsDate() Assert.Equal(CellType.Date, e.Current[0].Type); } + [Theory] + [InlineData("2020-01-01T00:00:00", false, 2020, 1, 1)] + [InlineData("2020-01-01T00:00:00", true, 2020, 1, 1)] + [InlineData("1904-01-01", true, 1904, 1, 1)] + [InlineData("1900-01-01", false, 1900, 1, 1)] + [InlineData("1900-02-28", false, 1900, 2, 28)] + [InlineData("1900-03-01", false, 1900, 3, 1)] + [InlineData("1899-12-31", false, 1899, 12, 31)] + [InlineData("1899-12-30", false, 1899, 12, 30)] + [InlineData("1850-06-15", false, 1850, 6, 15)] + [InlineData("1903-12-31", true, 1903, 12, 31)] + public void IsoDateReadsBackTheSameDayInEitherDateSystem(string text, bool date1904, int year, int month, int day) + { + using MemoryStream ms = WorkbookBuilder.Build( + $"""{text}""", date1904: date1904); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.True(e.MoveNext()); + Assert.Equal(CellType.Date, e.Current[0].Type); + Assert.True(e.Current[0].TryGetDateTime(reader.IsDate1904, out DateTime dt)); + Assert.Equal(new DateTime(year, month, day), dt); + } + + [Theory] + [InlineData("2020-01-01T10:00:00+05:00")] + [InlineData("2020-01-01T10:00:00-08:00")] + [InlineData("2020-01-01T10:00:00Z")] + public void IsoDateWithAnOffsetKeepsTheClockTimeAsWritten(string text) + { + using MemoryStream ms = WorkbookBuilder.Build( + $"""{text}"""); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.True(e.MoveNext()); + Assert.True(e.Current[0].TryGetDateTime(reader.IsDate1904, out DateTime dt)); + Assert.Equal(new DateTime(2020, 1, 1, 10, 0, 0), dt); + } + [Fact] public async Task IsoDateCellIsParsedAsDateAsync() { diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/PrefetchDecompressionTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/PrefetchDecompressionTests.cs index 7c9b1f2c..2e352c18 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/PrefetchDecompressionTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/PrefetchDecompressionTests.cs @@ -7,6 +7,7 @@ using ExcelReader.Core.Reader.Xlsx; using ExcelReader.Core.Reader.Zip; using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Zip; namespace ExcelReader.Tests.Reader.Xlsx { @@ -288,6 +289,32 @@ public void WrapUsesPrefetchForAnEntryAtOrAboveTheThresholdWhenEnabled() Assert.True(GetInnerStream(wrapped) is PrefetchStream); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PrefetchChecksTheDeclaredBytesOfAnEntryThatInflatesPastItsSize(bool crcMatches) + { + const int Declared = 99_000; + byte[] data = new byte[100_000]; + for (int i = 0; i < data.Length; i++) + { + data[i] = (byte)(i * 31); + } + uint crc = Crc32.Append(0, data.AsSpan(0, Declared)); + using var prefetch = new PrefetchStream(new MemoryStream(data), Declared, crcMatches ? crc : ~crc); + byte[] read = new byte[Declared]; + + if (crcMatches) + { + prefetch.ReadExactly(read); + Assert.Equal(data[..Declared], read); + } + else + { + Assert.Throws(() => prefetch.ReadExactly(read)); + } + } + [Fact] public void WrapNeverUsesPrefetchWhenDisabledRegardlessOfSize() { diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs new file mode 100644 index 00000000..0323414f --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxCanonicalCellTests.cs @@ -0,0 +1,85 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public class XlsxCanonicalCellTests + { + // An element prefix sends every cell through the general path, so the default-namespace sheet, + // whose cells in Excel's shape take the fast path, must read back the same. + [Fact] + public void CellsInExcelsShapeReadTheSameAsThroughTheGeneralPath() + { + const int Rows = 2000; + using var plain = WorkbookBuilder.Build(SheetRows("", Rows), SharedStrings(""), $"{StylesInner("")}"); + using var prefixed = WorkbookBuilder.BuildPrefixed("x", SheetRows("x:", Rows), SharedStrings("x:"), StylesInner("x:")); + + List expected = Snapshot(prefixed); + List actual = Snapshot(plain); + + Assert.Equal(Rows, expected.Count); + Assert.Equal(expected, actual); + } + + private static string SharedStrings(string p) + { + return $"<{p}si><{p}t>alpha<{p}si><{p}t>beta<{p}si><{p}t>gamma"; + } + + private static string StylesInner(string p) + { + return $"""<{p}cellXfs count="2"><{p}xf numFmtId="0"/><{p}xf numFmtId="14" applyNumberFormat="1"/>"""; + } + + private static string SheetRows(string p, int rows) + { + var sb = new StringBuilder(); + for (int r = 1; r <= rows; r++) + { + sb.Append(CultureInfo.InvariantCulture, $"""<{p}row r="{r}" spans="1:16">"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="A{r}" t="s"><{p}v>{r % 3}"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="B{r}" s="1"><{p}v>{45000 + r}"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="C{r}"><{p}v>152.58000000000001"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="D{r}" t="b"><{p}v>{r % 2}"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="E{r}" t="e"><{p}v>#N/A"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="F{r}" t="str"><{p}v>a&b_x0041_"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="G{r}" s="1"/>"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c t="n"><{p}v>-1.5E-3"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="J{r}" t="inlineStr"><{p}is><{p}t>inline {r}"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="K{r}" t="s"><{p}v>1234567890"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="L{r}" s="1" t="d"><{p}v>2024-01-02T03:04:05"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="M{r}" cm="1"><{p}v>7"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="N{r}"><{p}f>1+1<{p}v>2"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c r="O{r}" t="s"><{p}v> 1"""); + sb.Append(CultureInfo.InvariantCulture, $"""<{p}c s="1" r="P{r}"><{p}v>{r}.25"""); + sb.Append(CultureInfo.InvariantCulture, $""); + } + return sb.ToString(); + } + + private static List Snapshot(Stream workbook) + { + using XlsxWorkbook reader = Excel.FromXlsx(workbook); + var rows = new List(); + foreach (Row row in reader.FirstSheet) + { + var sb = new StringBuilder(); + foreach (RowCell rowCell in row.Cells) + { + Cell cell = rowCell.Value; + string number = "-"; + if (cell.TryGetDouble(out double value)) + { + number = value.ToString("R", CultureInfo.InvariantCulture); + } + sb.Append(CultureInfo.InvariantCulture, $"{rowCell.ColumnIndex}:{cell.Type}:{cell.StyleIndex}:{cell.GetString()}:{number}|"); + } + rows.Add(sb.ToString()); + } + return rows; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 3d9156bb..de083346 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -52,12 +52,18 @@ public void TryGetEntryReturnsFalseForAnAbsentName() Assert.False(index.TryGetEntry("xl/doesNotExist.xml"u8, out _)); } - [Fact] - public void CreateThrowsOnDuplicateCentralDirectoryEntryNames() + [Theory] + [InlineData(0)] + [InlineData(100)] + public void CreateThrowsOnDuplicateCentralDirectoryEntryNames(int otherEntries) { using var ms = new MemoryStream(); using (var zip = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true)) { + for (int i = 0; i < otherEntries; i++) + { + zip.CreateEntry(string.Create(CultureInfo.InvariantCulture, $"part{i}.txt"), CompressionLevel.NoCompression); + } using (var s1 = zip.CreateEntry("dup.txt", CompressionLevel.NoCompression).Open()) { s1.Write("first"u8); diff --git a/tests/ExcelReader.Tests/Writer/WriterEdgeValidationTests.cs b/tests/ExcelReader.Tests/Writer/WriterEdgeValidationTests.cs index a2a0ba7d..26ae82ca 100644 --- a/tests/ExcelReader.Tests/Writer/WriterEdgeValidationTests.cs +++ b/tests/ExcelReader.Tests/Writer/WriterEdgeValidationTests.cs @@ -108,6 +108,16 @@ public void NumberFormatsStopAtWhatExcelOpens(string format) }); } + [Theory] + [MemberData(nameof(Formats))] + public void NumberFormatsXmlCannotHoldAreRejected(string format) + { + foreach (string numberFormat in new[] { "0\u0001", "0\t", "0\n", "0\uFFFE", "0\uFFFF" }) + { + Assert.Throws(() => WithWorkbook(format, wb => wb.AddStyle(new CellStyle { NumberFormat = numberFormat }))); + } + } + [Fact] public void AValueWithNoNumericFormIsRefusedTheSameWayEverywhere() { diff --git a/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs b/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs index dfa1087d..af024c2d 100644 --- a/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs +++ b/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs @@ -56,6 +56,39 @@ public async Task RoundTripsAllCellTypes() Assert.False(e.MoveNext()); } + private static void WriteRows(XlsSheetWriter sheet, int rows) + { + for (int i = 0; i < rows; i++) + { + using var r = sheet.StartRow(); + r.Write(i); + } + sheet.End(); + } + + [Fact] + public async Task ContinuationSheetSkipsANameAlreadyTaken() + { + byte[] bytes = await WriteAsync(wb => + { + WriteRows(wb.AddSheet("Data (2)"), 1); + WriteRows(wb.AddSheet("Data"), 65_537); + }, ct: TestContext.Current.CancellationToken); + + using var reader = Excel.FromXls(new MemoryStream(bytes)); + Assert.Equal(["Data (2)", "Data", "Data (3)"], [.. reader.Sheets.Select(static s => s.Name)]); + } + + [Fact] + public async Task SheetCannotTakeTheNameOfAContinuationSheet() + { + await Assert.ThrowsAsync(() => WriteAsync(wb => + { + WriteRows(wb.AddSheet("Data"), 65_537); + wb.AddSheet("data (2)"); + }, ct: TestContext.Current.CancellationToken)); + } + [Fact] public async Task NumericOverloadsRoundTrip() { diff --git a/tests/ExcelReader.Tests/Writer/Zip/SegmentCompressorTests.cs b/tests/ExcelReader.Tests/Writer/Zip/SegmentCompressorTests.cs index a6baa300..1ab90a33 100644 --- a/tests/ExcelReader.Tests/Writer/Zip/SegmentCompressorTests.cs +++ b/tests/ExcelReader.Tests/Writer/Zip/SegmentCompressorTests.cs @@ -54,6 +54,23 @@ public void SegmentsConcatenateIntoOneStream(CompressionLevel level) AssertInflatesTo([.. parts.SelectMany(static p => p)], output.Span.ToArray()); } + [Fact] + public void FastestSegmentMatchesTheStreamingEncoder() + { + byte[] input = DeflateEncoderTests.SheetXml(40_000); + Assert.True(input.Length > 2 << 20); + using var streamed = new BiffBuffer(4096); + using (var encoder = new DeflateEncoder()) + { + encoder.Write(input, streamed); + encoder.FlushSync(streamed); + } + using var segment = new BiffBuffer(4096); + SegmentCompressor.Compress(input, CompressionLevel.Fastest, segment); + + Assert.Equal(streamed.Span.ToArray(), segment.Span.ToArray()); + } + [Fact] public void NoCompressionCopiesTheBytes() {