Parallel native writes, lighter parallel paths, faster first open, 11 bug fixes

.github/workflows/ci.yml

@@ -134,6 +134,7 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./TestResults/*.cobertura.xml
+          flags: csharp
           fail_ci_if_error: false
 
   isa-fallback:

.github/workflows/native-bindings.yml

@@ -105,7 +105,11 @@ jobs:
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
-          components: clippy
+          components: clippy, llvm-tools-preview
+
+      - name: Install cargo-llvm-cov (Linux)
+        if: runner.os == 'Linux'
+        uses: taiki-e/install-action@cargo-llvm-cov
 
       - name: Cache cargo
         uses: Swatinem/rust-cache@v2
@@ -143,11 +147,29 @@ jobs:
         run: cargo clippy --workspace --all-targets --all-features -- -D warnings
 
       - name: cargo test
+        if: runner.os != 'Linux'
         working-directory: rust
         env:
           EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }}
         run: cargo test --workspace --all-features --verbose
 
+      # The derive crate runs inside the compiler, where coverage is not recorded, so it is left out.
+      - name: cargo test with coverage (Linux)
+        if: runner.os == 'Linux'
+        working-directory: rust
+        env:
+          EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }}
+        run: cargo llvm-cov --workspace --all-features --ignore-filename-regex excelreader-derive --lcov --output-path lcov.info
+
+      - name: Upload Rust coverage to Codecov
+        if: runner.os == 'Linux'
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./rust/lcov.info
+          flags: rust
+          fail_ci_if_error: false
+
       # The next two compile the crate again (default features, then the bench profile); that
       # catches nothing platform-specific the test build above missed, so one OS is enough.
       - name: cargo build (default features)
@@ -197,10 +219,21 @@ jobs:
         run: chmod +x python/src/excelreader/_lib/*
 
       - name: Install the Python package
-        run: pip install -e "python[dev]"
+        run: pip install -e "python[dev]" ${{ matrix.os == 'ubuntu-latest' && 'pytest-cov' || '' }}
 
       - name: Run the Python tests
-        run: pytest python/tests -v
+        run: >-
+          pytest python/tests -v
+          ${{ matrix.os == 'ubuntu-latest' && '--cov=excelreader --cov-report=xml:coverage-python.xml' || '' }}
+
+      - name: Upload Python coverage to Codecov
+        if: matrix.os == 'ubuntu-latest'
+        uses: codecov/codecov-action@v7
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage-python.xml
+          flags: python
+          fail_ci_if_error: false
 
       - name: Build the wheel and check it installs here
         run: |

README.md

@@ -106,7 +106,7 @@ ExcelReader ships a NativeAOT shared library with a C ABI, so non-.NET languages
 XLSX, XLSB, XLS and CSV without a .NET runtime installed.
 
 - C ABI header: [`src/ExcelReader.Native/include/excelreader.h`](src/ExcelReader.Native/include/excelreader.h)
-- Python package: [`python/`](python/README.md)
+- Python package: `pip install excelreader-native` — see [`python/README.md`](python/README.md).
 - C++ package: header-only CMake wrapper, `xl::Workbook`/`xl::parse_sheet`/`xl::write_sheet` over the same ABI — see [`cpp/README.md`](cpp/README.md).
 - Rust crate: safe `Workbook`/`parse_sheet`/`write_sheet` bindings, downloadable via `cargo add excelreader` — see [`rust/excelreader/README.md`](rust/excelreader/README.md).
 

codecov.yml

@@ -0,0 +1,14 @@
+# Each language uploads under its own flag from a path-filtered job, so a flag a commit did not run keeps its last report.
+flag_management:
+  default_rules:
+    carryforward: true
+  individual_flags:
+    - name: csharp
+      paths:
+        - src/
+    - name: rust
+      paths:
+        - rust/
+    - name: python
+      paths:
+        - python/

cpp/README.md

@@ -350,6 +350,16 @@ toggles. Its `format` defaults to `XL_FORMAT_AUTO`, which takes the format from
 extension (.xlsx, .xlsb, .xls or .csv, in any case); an unrecognized extension, or AUTO with a vector
 destination, fails the write rather than silently choosing a format.
 
+For XLSX and XLSB, `degree_of_parallelism` formats and compresses batches of `batch_rows` rows on
+several threads: `0` uses every processor, `1` (the default) writes serially, and `batch_rows = 0`
+keeps the library default of 8,192. The columns must not change while the write runs. With
+`use_shared_strings` on, two parallel runs give the same content in files that differ byte for byte.
+XLS, CSV and `xl::WriterHandle` ignore both fields.
+
+```cpp
+xl::write_columns("big.xlsx", columns, {.degree_of_parallelism = 0});
+```
+
 `write_sheet` walks the range once and appends each field to its own column buffer, with the
 per-field dispatch resolved at compile time. That transpose is the only copy it makes, and it is
 what the ABI's columnar shape costs a row-shaped caller; `write_columns` pays nothing.
@@ -394,6 +404,40 @@ auto counted = xl::aggregate_csv("big.csv", RowCount{}, {.csv_delimiter = ';'},
 The dialect (`csv_delimiter`, `csv_quote`, `csv_detect_bom`) and `max_cell_bytes` come from
 `xl::OpenOptions`; the header row and the degree of parallelism (0 = every processor) follow it.
 
+## Callback reads
+
+`Sheet::for_each_row` calls a function once per row on the calling thread, with cells borrowed from
+the library's buffers for that call instead of copied into a cursor:
+
+```cpp
+double total = 0;
+auto walked = workbook->sheet(0).for_each_row([&](xl::RowView row)
+{
+    if (row.size() > 1)
+    {
+        total += row[1].as_number().value_or(0);
+    }
+    return XL_OK;
+});
+```
+
+`Sheet::aggregate` folds any sheet, in any format, across worker threads; the accumulator has the same
+shape as for `aggregate_csv`, so `RowCount` above works unchanged. New code should use it rather than
+`aggregate_csv`:
+
+```cpp
+auto counted = workbook->sheet(0).aggregate(RowCount{}, /*header_row*/ 1);
+```
+
+A cell in these callbacks carries text, a number, or both. `value` holds only the stored text, so it
+is empty for a cell that carries only a number; `text()` formats that number itself (shortest
+round-trip form) and `as_number()` returns it directly, or parses the text when there is no number.
+A return other than `XL_OK` stops the run and comes back as the error's code (use positive codes),
+and an exception is reported as an error. Calling other functions on the workbook from inside the
+callback is allowed. `Sheet::aggregate` seeds a copy per chunk of sheet data, not per thread, and may
+destroy a chunk's copy without combining it; XLS, table views and CSV opened from a `Source` or
+`InputStream` read sequentially.
+
 ## Bounds and ABI
 
 `TableView::operator[]` is unchecked, like `std::vector`'s. Use `TableView::at(row)`, which returns

cpp/benchmarks/CMakeLists.txt

@@ -58,6 +58,8 @@ if(EXCELREADER_BUILD_BENCHMARKS_COMPARE)
         GIT_TAG v1.5.0
     )
     FetchContent_MakeAvailable(xlnt)
+    target_compile_options(xlnt PUBLIC
+        "$<$<AND:$<COMPILE_LANGUAGE:CXX>,$<NOT:$<CXX_COMPILER_ID:MSVC>>>:SHELL:-include cstdint;SHELL:-include limits>")
 
     # xlsxio (https://github.com/brechtsanders/xlsxio) - its own CMakeLists.txt requires an
     # already-installed expat + Minizip (FIND_PACKAGE(... REQUIRED), no FetchContent-friendly

cpp/benchmarks/benchmark_csv_compare.cpp

@@ -290,6 +290,73 @@ static void BM_ExcelReader_Csv_Cells(benchmark::State &state)
 }
 BENCHMARK(BM_ExcelReader_Csv_Cells);
 
+static void BM_ExcelReader_Csv_ForEach(benchmark::State &state)
+{
+    const std::vector<std::uint8_t> buffer = read_fixture();
+    for (auto _ : state)
+    {
+        auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV});
+        if (!workbook.has_value())
+        {
+            state.SkipWithError(workbook.error().message);
+            return;
+        }
+        int64_t acc = 0;
+        auto opened = workbook->sheet(0).for_each_row([&acc](const xl::RowView &row) {
+            for (const xl::CellView cell : row)
+            {
+                acc += static_cast<int64_t>(cell.value.size());
+            }
+            return XL_OK;
+        });
+        benchmark::DoNotOptimize(acc);
+        state.counters["checksum"] = static_cast<double>(acc);
+    }
+}
+BENCHMARK(BM_ExcelReader_Csv_ForEach);
+
+static void BM_ExcelReader_Csv_Aggregate(benchmark::State &state)
+{
+    struct AggregateRun
+    {
+        std::int64_t cell_bytes = 0;
+        std::int32_t accumulate(xl::RowView row)
+        {
+            for (const xl::CellView cell : row)
+            {
+                cell_bytes += static_cast<int64_t>(cell.value.size());
+            }
+            return XL_OK;
+        }
+        std::int32_t combine(AggregateRun &next)
+        {
+            cell_bytes += next.cell_bytes;
+            return XL_OK;
+        }
+    };
+    const std::vector<std::uint8_t> buffer = read_fixture();
+    for (auto _ : state)
+    {
+        auto workbook = xl::Workbook::open(buffer, {.format = XL_FORMAT_CSV});
+        if (!workbook.has_value())
+        {
+            state.SkipWithError(workbook.error().message);
+            return;
+        }
+        int64_t acc = 0;
+        auto data = workbook->sheet(0).aggregate<AggregateRun>();
+        if (!data.has_value())
+        {
+            state.SkipWithError(data.error().message);
+            return;
+        }
+        auto &agg = data.value();
+        benchmark::DoNotOptimize(agg);
+        state.counters["checksum"] = static_cast<double>(agg.cell_bytes);
+    }
+}
+BENCHMARK(BM_ExcelReader_Csv_Aggregate);
+
 static void BM_Zsv_Csv_Cells(benchmark::State &state, unsigned char engine)
 {
     const std::vector<std::uint8_t> buffer = read_fixture();

cpp/tests/CMakeLists.txt

@@ -33,6 +33,7 @@ target_link_libraries(excelreader_cpp_rows PRIVATE xl::excelreader)
 set(EXCELREADER_XLSX_FIXTURE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../RealExcel.xlsx"
     CACHE FILEPATH "Path to the RealExcel.xlsx fixture the C++ row tests read")
 target_compile_definitions(excelreader_cpp_rows PRIVATE
+    EXCELREADER_FIXTURE_PATH="${EXCELREADER_FIXTURE_PATH}"
     EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}")
 
 add_test(NAME excelreader_cpp_rows COMMAND excelreader_cpp_rows)
@@ -68,4 +69,17 @@ add_executable(excelreader_cpp_aggregate aggregate.cpp)
 target_link_libraries(excelreader_cpp_aggregate PRIVATE xl::excelreader)
 
 add_test(NAME excelreader_cpp_aggregate COMMAND excelreader_cpp_aggregate)
-excelreader_copy_native_library(excelreader_cpp_aggregate)
+excelreader_copy_native_library(excelreader_cpp_aggregate)
+
+add_executable(excelreader_cpp_numbers numbers.cpp)
+target_link_libraries(excelreader_cpp_numbers PRIVATE xl::excelreader)
+
+add_test(NAME excelreader_cpp_numbers COMMAND excelreader_cpp_numbers)
+excelreader_copy_native_library(excelreader_cpp_numbers)
+
+add_executable(excelreader_cpp_numbers_fallback numbers.cpp)
+target_link_libraries(excelreader_cpp_numbers_fallback PRIVATE xl::excelreader)
+target_compile_definitions(excelreader_cpp_numbers_fallback PRIVATE EXCELREADER_TEST_NO_TO_CHARS)
+
+add_test(NAME excelreader_cpp_numbers_fallback COMMAND excelreader_cpp_numbers_fallback)
+excelreader_copy_native_library(excelreader_cpp_numbers_fallback)