diff --git a/CMakeLists.txt b/CMakeLists.txt index f63146a..102964d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -225,6 +225,10 @@ if(DTO_BUILD_TESTS) endif() endif() +include(GNUInstallDirs) + +install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + # Install and export the library install(TARGETS dto EXPORT DTOTargets diff --git a/Makefile b/Makefile index 06cc2e1..d0223b5 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ libdto_nostats: dto.c install: cp libdto.so.1.0 /usr/lib64/ + cp dto.h /usr/include/ ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so.1 ln -sf /usr/lib64/libdto.so.1.0 /usr/lib64/libdto.so diff --git a/dto.c b/dto.c index 6aa8a35..636c315 100644 --- a/dto.c +++ b/dto.c @@ -16,6 +16,8 @@ #include #include #include + +#include "dto.h" #include #include #include @@ -82,6 +84,7 @@ struct dto_wq { uint64_t dsa_gencap; int wq_size; uint32_t max_transfer_size; + uint32_t max_batch_size; int wq_fd; void *wq_portal; bool wq_mmapped; @@ -998,6 +1001,12 @@ static int dsa_init_from_wq_list(char *wq_list) goto fail_wq; } + wqs[num_wqs].max_batch_size = dto_get_param_ullong(dir_fd, "max_batch_size", &rc); + if (rc) { + close(dir_fd); + goto fail_wq; + } + dto_get_param_string(dir_fd, "mode", wq_mode); if (wq_mode[0] == '\0') { @@ -1161,6 +1170,7 @@ static int dsa_init_from_accfg(void) wqs[num_wqs].wq_size = accfg_wq_get_size(wq); wqs[num_wqs].max_transfer_size = accfg_wq_get_max_transfer_size(wq); + wqs[num_wqs].max_batch_size = accfg_wq_get_max_batch_size(wq); wqs[num_wqs].acc_wq = wq; wqs[num_wqs].dsa_gencap = accfg_device_get_gen_cap(device); @@ -1630,15 +1640,18 @@ static __always_inline struct dto_wq *get_wq(void* buf) return wq; } +/* 8-byte MEMFILL pattern for the byte value c (like memset) */ +static __always_inline uint64_t memfill_pattern(int c) +{ + return 0x0101010101010101ULL * (uint8_t)c; +} + static void dto_memset(void *s, int c, size_t n, int *result) { - uint64_t memset_pattern; + uint64_t memset_pattern = memfill_pattern(c); size_t cpu_size, dsa_size; struct dto_wq *wq = get_wq(s); - for (int i = 0; i < 8; ++i) - ((uint8_t *) &memset_pattern)[i] = (uint8_t) c; - thr_desc.opcode = DSA_OPCODE_MEMFILL; thr_desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; if (dto_dsa_cc && (wq->dsa_gencap & GENCAP_CC_MEMORY)) @@ -2135,3 +2148,260 @@ int memcmp(const void *s1, const void *s2, size_t n) } return ret; } + +/******************************************************************************* + * Explicit asynchronous submit/poll API (see dto.h). + ******************************************************************************/ + +/* With the default CRC flags the device inverts and bit-reflects both the + * seed and the result, so a seed of 0 yields the standard CRC32C + * (Castagnoli, init 0xFFFFFFFF, final inversion) and crc_val can be + * returned as is. */ +#define DSA_CRC32C_SEED 0u + +/* ENQCMD to a shared WQ can transiently fail when the queue is full; + * retry briefly before giving up so momentary bursts don't push work + * back onto the CPU. */ +#define ASYNC_SUBMIT_RETRIES 16 + +/* Failed async completions are logged at most this many times per process: + * poll may legitimately be called repeatedly on a failed op. */ +#define ASYNC_FAIL_LOG_LIMIT 3 + +struct dto_async_op_impl { + struct dsa_hw_desc desc; /* 64 bytes, 64-aligned via dto_async_op */ + struct dsa_completion_record comp __attribute__((aligned(32))); +}; +_Static_assert(sizeof(struct dto_async_op_impl) <= sizeof(dto_async_op), + "dto_async_op opaque storage too small"); +_Static_assert(sizeof(struct dsa_hw_desc) == 64, "unexpected descriptor size"); + +static __always_inline uint32_t async_desc_flags(const struct dto_wq *wq, + unsigned int flags, + bool has_dest) +{ + uint32_t f = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + + if (flags & DTO_SUBMIT_BOF) + f |= IDXD_OP_FLAG_BOF; + /* CC is only legal for operations with a destination; CRC Generation + * would be failed by the device with DSA_COMP_INVALID_FLAGS. */ + if ((flags & DTO_SUBMIT_CC) && has_dest && + (wq->dsa_gencap & GENCAP_CC_MEMORY)) + f |= IDXD_OP_FLAG_CC; + return f; +} + +static int async_submit(struct dto_wq *wq, struct dsa_hw_desc *desc) +{ + for (int attempt = 0; attempt < ASYNC_SUBMIT_RETRIES; attempt++) { + int rc = dsa_submit(wq, desc); + + if (rc == SUCCESS) + return DTO_ASYNC_SUBMITTED; + if (rc != RETRY) + break; + _mm_pause(); + } + return DTO_ASYNC_FALLBACK; +} + +/* Completion record status -> DTO_ASYNC_{PENDING,DONE,FAILED} */ +static int async_comp_result(const struct dsa_completion_record *comp, + uint32_t opcode, uint32_t size) +{ + static int fail_logged; + uint8_t status = __atomic_load_n((const uint8_t *)&comp->status, + __ATOMIC_ACQUIRE); + + if (status == 0) + return DTO_ASYNC_PENDING; + if (likely((status & DSA_COMP_STATUS_MASK) == DSA_COMP_SUCCESS)) + return DTO_ASYNC_DONE; + if (fail_logged < ASYNC_FAIL_LOG_LIMIT && + __atomic_fetch_add(&fail_logged, 1, __ATOMIC_RELAXED) < + ASYNC_FAIL_LOG_LIMIT) + LOG_ERROR("async op failed with status %x (opcode %x, size %x)\n", + status, opcode, size); + return DTO_ASYNC_FAILED; +} + +/* src_or_pattern is the source address, or the fill pattern for MEMFILL + * (both occupy the same descriptor slot). */ +static int dto_submit_async_common(dto_async_op *op, uint32_t opcode, + void *dest, uint64_t src_or_pattern, + size_t n, unsigned int flags) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + struct dto_wq *wq; + + if (!dto_initialized || use_std_lib_calls) + return DTO_ASYNC_FALLBACK; + /* The completion record must be 32-byte aligned. dto_async_op is + * declared 64-byte aligned, but a malloc'ed op need not be. */ + if (n == 0 || ((uintptr_t)op & 31) != 0) + return DTO_ASYNC_FALLBACK; + wq = get_wq(dest ? dest : (void *)(uintptr_t)src_or_pattern); + if (unlikely(wq == NULL) || n > wq->max_transfer_size) + return DTO_ASYNC_FALLBACK; + + orig_memset(&impl->desc, 0, sizeof(impl->desc)); + impl->desc.opcode = opcode; + impl->desc.flags = async_desc_flags(wq, flags, dest != NULL); + impl->desc.completion_addr = (uint64_t)&impl->comp; + impl->desc.xfer_size = (uint32_t)n; + impl->desc.src_addr = src_or_pattern; + impl->desc.dst_addr = (uint64_t)dest; + if (opcode == DSA_OPCODE_COPY_CRC || opcode == DSA_OPCODE_CRCGEN) + impl->desc.crc_seed = DSA_CRC32C_SEED; + impl->comp.status = 0; + + return async_submit(wq, &impl->desc); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_MEMMOVE, dest, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_MEMFILL, dest, + memfill_pattern(c), n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_COPY_CRC, dest, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags) +{ + return dto_submit_async_common(op, DSA_OPCODE_CRCGEN, NULL, + (uint64_t)src, n, flags); +} + +__attribute__((visibility("default"))) +int dto_async_poll(dto_async_op *op) +{ + struct dto_async_op_impl *impl = (struct dto_async_op_impl *)op; + + return async_comp_result(&impl->comp, impl->desc.opcode, + impl->desc.xfer_size); +} + +__attribute__((visibility("default"))) +uint32_t dto_async_crc_val(const dto_async_op *op) +{ + const struct dto_async_op_impl *impl = + (const struct dto_async_op_impl *)op; + return (uint32_t)impl->comp.crc_val; +} + +/* ---- Asynchronous batch copy (see dto.h) ---- */ + +struct dto_batch_op { + struct dsa_hw_desc desc __attribute__((aligned(64))); + struct dsa_completion_record comp __attribute__((aligned(32))); + struct dsa_hw_desc descs[DTO_BATCH_MAX] __attribute__((aligned(64))); + struct dsa_completion_record comps[DTO_BATCH_MAX] __attribute__((aligned(32))); +}; + +__attribute__((visibility("default"))) +dto_batch_op *dto_batch_op_new(void) +{ + void *p = NULL; + + if (posix_memalign(&p, 64, sizeof(struct dto_batch_op))) + return NULL; + /* usable before the constructor has run (orig_memset is not) */ + dto_internal_memset(p, 0, sizeof(struct dto_batch_op)); + return (dto_batch_op *)p; +} + +__attribute__((visibility("default"))) +void dto_batch_op_free(dto_batch_op *op) +{ + free(op); +} + +__attribute__((visibility("default"))) +int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, + size_t *sizes, int count, unsigned int flags) +{ + struct dto_wq *wq; + uint32_t desc_flags; + + /* a DSA batch needs at least two descriptors */ + if (unlikely(!dto_initialized || use_std_lib_calls || + count < 2 || count > DTO_BATCH_MAX)) + return DTO_ASYNC_FALLBACK; + wq = get_wq(dst[0]); + if (unlikely(wq == NULL) || (uint32_t)count > wq->max_batch_size) + return DTO_ASYNC_FALLBACK; + /* validate everything before touching the op so FALLBACK leaves it + * untouched */ + for (int i = 0; i < count; i++) { + if (sizes[i] == 0 || sizes[i] > wq->max_transfer_size) + return DTO_ASYNC_FALLBACK; + } + + desc_flags = async_desc_flags(wq, flags, true); + orig_memset(op->descs, 0, sizeof(op->descs[0]) * count); + for (int i = 0; i < count; i++) { + struct dsa_hw_desc *desc = &op->descs[i]; + + desc->opcode = DSA_OPCODE_MEMMOVE; + desc->flags = desc_flags; + desc->src_addr = (uint64_t)src[i]; + desc->dst_addr = (uint64_t)dst[i]; + desc->xfer_size = (uint32_t)sizes[i]; + desc->completion_addr = (uint64_t)&op->comps[i]; + op->comps[i].status = 0; + } + orig_memset(&op->desc, 0, sizeof(op->desc)); + op->desc.opcode = DSA_OPCODE_BATCH; + op->desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + op->desc.desc_list_addr = (uint64_t)op->descs; + op->desc.desc_count = count; + op->desc.completion_addr = (uint64_t)&op->comp; + op->comp.status = 0; + + return async_submit(wq, &op->desc); +} + +__attribute__((visibility("default"))) +int dto_batch_poll(dto_batch_op *op) +{ + int rc = async_comp_result(&op->comp, DSA_OPCODE_BATCH, + op->desc.desc_count); + + if (rc != DTO_ASYNC_FAILED) + return rc; + + /* redo the copies the device did not complete on the CPU */ + for (uint32_t i = 0; i < op->desc.desc_count; i++) { + const struct dsa_hw_desc *desc = &op->descs[i]; + + if ((op->comps[i].status & DSA_COMP_STATUS_MASK) != + DSA_COMP_SUCCESS) + orig_memcpy((void *)(uintptr_t)desc->dst_addr, + (const void *)(uintptr_t)desc->src_addr, + desc->xfer_size); + } + /* Mark the batch complete so a later poll (possibly from another + * thread) doesn't redo the copies from a source the caller may + * already have reused. */ + __atomic_store_n(&op->comp.status, DSA_COMP_SUCCESS, __ATOMIC_RELEASE); + return DTO_ASYNC_DONE; +} diff --git a/dto.h b/dto.h new file mode 100644 index 0000000..3f5176e --- /dev/null +++ b/dto.h @@ -0,0 +1,139 @@ +/******************************************************************************* + * Copyright (C) 2026 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * dto.h - Public asynchronous offload API for libdto. + * + * In addition to transparently accelerating memcpy/memmove/memset/memcmp via + * LD_PRELOAD, libdto exposes an explicit submit/poll API for applications + * that want to overlap accelerator data movement with CPU work. + * + * Lifecycle: + * + * dto_async_op op; + * if (dto_submit_memcpy(&op, dst, src, n, 1) == DTO_ASYNC_SUBMITTED) { + * ... other CPU work ... + * while (dto_async_poll(&op) == DTO_ASYNC_PENDING) + * ; // pause/yield + * if (dto_async_poll(&op) != DTO_ASYNC_DONE) + * memcpy(dst, src, n); // redo on the CPU + * } else { // DTO_ASYNC_FALLBACK: nothing submitted + * memcpy(dst, src, n); + * } + * + * The operation state is caller-allocated, so it can outlive the submitting + * call and be polled from a different thread; nothing is stored in + * thread-local state. dto_async_op is declared 64-byte aligned, which + * automatic and static storage honour; heap-allocated ops must come from + * aligned_alloc/posix_memalign (plain malloc does not guarantee it). An op + * that is not at least 32-byte aligned is rejected with DTO_ASYNC_FALLBACK. + * + * Submission returns DTO_ASYNC_FALLBACK when the accelerator is unavailable + * (not initialized, disabled via DTO_USESTDC_CALLS, no usable work queue) or + * the request is out of range (n == 0, or larger than the work queue's + * maximum transfer size). Nothing is copied in that case: the caller performs + * the operation on the CPU. Unlike the transparent path, explicit submits are + * not subject to the size heuristics (DTO_MIN_BYTES / auto-tuning): callers + * decide what to offload. + * + * On DTO_ASYNC_FAILED (e.g. a page fault the device could not resolve) the + * destination contents are unspecified; redo the whole operation on the CPU. + * + * @cache_control (DTO_SUBMIT_CC): directs the operation's output toward the + * CPU cache (IDXD_OP_FLAG_CC) when the device supports it, for destinations + * that will be read again soon. + * + * DTO_SUBMIT_BOF makes the device block on page faults and wait for the + * kernel to resolve them, instead of aborting the operation (which surfaces + * as DTO_ASYNC_FAILED from dto_async_poll). Use it when the buffers may not + * be fully faulted in. The work queue must be configured with + * block-on-fault enabled ("accel-config config-wq --block-on-fault=1"), + * otherwise the device rejects the descriptor and the operation fails. + ******************************************************************************/ + +#ifndef DTO_H +#define DTO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Caller-allocated operation state (opaque). */ +typedef struct dto_async_op { + unsigned char opaque[192] __attribute__((aligned(64))); +} dto_async_op; + +/* Submission flags */ +#define DTO_SUBMIT_CC (1u << 0) /* see @cache_control above */ +#define DTO_SUBMIT_BOF (1u << 1) /* block on page faults (see below) */ + +/* Submit results */ +#define DTO_ASYNC_SUBMITTED 0 +#define DTO_ASYNC_FALLBACK (-1) + +/* Poll results */ +#define DTO_ASYNC_PENDING 0 +#define DTO_ASYNC_DONE 1 +#define DTO_ASYNC_FAILED (-1) + +/* Copy n bytes from src to dest. */ +int dto_submit_memcpy(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Fill n bytes of dest with the byte value c (like memset). */ +int dto_submit_memset(dto_async_op *op, void *dest, int c, size_t n, + unsigned int flags); + +/* Copy n bytes from src to dest and compute the CRC32C of the data. + * The CRC value (read with dto_async_crc_val after DTO_ASYNC_DONE) is the + * standard CRC32C (Castagnoli, seed 0xFFFFFFFF, final inversion), i.e. it + * matches common software crc32c implementations. */ +int dto_submit_memcpy_crc(dto_async_op *op, void *dest, const void *src, + size_t n, unsigned int flags); + +/* Compute the CRC32C of n bytes at src without copying. */ +int dto_submit_crc(dto_async_op *op, const void *src, size_t n, + unsigned int flags); + +/* Poll a submitted operation. Returns DTO_ASYNC_PENDING, DTO_ASYNC_DONE or + * DTO_ASYNC_FAILED. May be called repeatedly, from any thread; once an op + * has left PENDING its result is stable. */ +int dto_async_poll(dto_async_op *op); + +/* CRC32C result of a completed dto_submit_memcpy_crc/dto_submit_crc. + * Valid only after dto_async_poll returned DTO_ASYNC_DONE. */ +uint32_t dto_async_crc_val(const dto_async_op *op); + +/* ---- Asynchronous batch copy ---- + * + * Submits up to DTO_BATCH_MAX copies as one DSA batch descriptor. The caller + * owns the op (dto_batch_op_new / dto_batch_op_free) and may reuse it for + * consecutive batches. A DSA batch requires at least two descriptors, so + * count < 2 returns DTO_ASYNC_FALLBACK, as does count > DTO_BATCH_MAX or + * count > the work queue's configured max_batch_size. + * + * dto_submit_batch_copy returns DTO_ASYNC_SUBMITTED or DTO_ASYNC_FALLBACK + * (nothing submitted, nothing copied). dto_batch_poll returns + * DTO_ASYNC_PENDING or DTO_ASYNC_DONE; copies the accelerator failed are + * redone on the CPU before DONE is returned, so DONE means every copy is + * complete. Once DONE, further polls return DONE without touching the + * buffers again. */ +#define DTO_BATCH_MAX 64 + +typedef struct dto_batch_op dto_batch_op; + +dto_batch_op *dto_batch_op_new(void); +void dto_batch_op_free(dto_batch_op *op); +int dto_submit_batch_copy(dto_batch_op *op, void **dst, void **src, + size_t *sizes, int count, unsigned int flags); +int dto_batch_poll(dto_batch_op *op); + +#ifdef __cplusplus +} +#endif + +#endif /* DTO_H */ diff --git a/tests/test_functional.c b/tests/test_functional.c index a2809dc..2c3f3a4 100644 --- a/tests/test_functional.c +++ b/tests/test_functional.c @@ -19,6 +19,8 @@ ******************************************************************************/ #include "dto_test_utils.h" +#include "../dto.h" +#include #include #include @@ -479,6 +481,229 @@ static int test_multithread(void) /* ---- Test runner ---- */ +/* ---- Explicit asynchronous API (dto.h) ---- */ + +/* Standard CRC32C (Castagnoli): init 0xFFFFFFFF, final inversion, the + * convention dto.h documents for dto_async_crc_val. */ +__attribute__((target("sse4.2"))) +static uint32_t ref_crc32c(const void *buf, size_t n) +{ + const uint8_t *p = buf; + uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < n; i++) + crc = _mm_crc32_u8(crc, p[i]); + return ~crc; +} + +/* Poll until poll_expr leaves PENDING and yield its result. A submitted op + * owns its (stack-allocated) state until it completes, so a stall past the + * 10s safety timeout aborts the process rather than returning into a test + * that would let the device write into a dead stack frame. */ +#define ASYNC_WAIT(poll_expr) ({ \ + struct timespec _start, _now; \ + int _rc; \ + clock_gettime(CLOCK_MONOTONIC, &_start); \ + while ((_rc = (poll_expr)) == DTO_ASYNC_PENDING) { \ + clock_gettime(CLOCK_MONOTONIC, &_now); \ + if (_now.tv_sec - _start.tv_sec > 10) { \ + fprintf(stderr, " FATAL: async op still pending after 10s at %s:%d\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ + } \ + _rc; }) + +/* Submit a memcpy with the given flags and verify the data afterwards, + * whichever of the three documented outcomes (DONE, FAILED + CPU redo, + * FALLBACK + CPU copy) occurs. */ +static int check_async_memcpy(unsigned int flags) +{ + const size_t n = 1 << 20; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + + rc = dto_submit_memcpy(&op, dst, src, n, flags); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc != DTO_ASYNC_DONE) + memcpy(dst, src, n); /* documented caller fallback */ + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memcpy(dst, src, n); + } + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +static int test_async_memcpy(void) +{ + return check_async_memcpy(DTO_SUBMIT_CC | DTO_SUBMIT_BOF); +} + +static int test_async_memcpy_no_bof(void) +{ + /* Without DTO_SUBMIT_BOF a page fault aborts the operation and poll + * reports DTO_ASYNC_FAILED; the caller redoes the copy on the CPU. + * Either outcome is valid here, the data must be right afterwards. */ + return check_async_memcpy(0); +} + +static int test_async_memset(void) +{ + const size_t n = 1 << 20; + uint8_t *dst = malloc(n); + dto_async_op op; + int rc; + + ASSERT_TRUE(dst != NULL); + clear_buf(dst, n); + + rc = dto_submit_memset(&op, dst, 0x5A, n, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc != DTO_ASYNC_DONE) + memset(dst, 0x5A, n); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memset(dst, 0x5A, n); + } + ASSERT_TRUE(verify_set(dst, 0x5A, n)); + free(dst); + return 0; +} + +/* BOF is rejected by the device on a WQ configured without block-on-fault + * (dto.h), so like the other tests the CRC tests accept DTO_ASYNC_FAILED; + * the CRC value is only checked when the device completed the op. */ +static int test_async_crc(void) +{ + const size_t n = 256 * 1024; + uint8_t *src = malloc(n); + dto_async_op op; + uint32_t expect; + int rc; + + ASSERT_TRUE(src != NULL); + fill_pattern(src, n); + expect = ref_crc32c(src, n); + + rc = dto_submit_crc(&op, src, n, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), expect); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + } + free(src); + return 0; +} + +static int test_async_memcpy_crc(void) +{ + const size_t n = 256 * 1024; + uint8_t *src = malloc(n), *dst = malloc(n); + dto_async_op op; + uint32_t expect; + int rc; + + ASSERT_TRUE(src && dst); + fill_pattern(src, n); + clear_buf(dst, n); + expect = ref_crc32c(src, n); + + rc = dto_submit_memcpy_crc(&op, dst, src, n, DTO_SUBMIT_CC | DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_async_poll(&op)); + ASSERT_TRUE(rc == DTO_ASYNC_DONE || rc == DTO_ASYNC_FAILED); + if (rc == DTO_ASYNC_DONE) + ASSERT_EQ(dto_async_crc_val(&op), expect); + else + memcpy(dst, src, n); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + memcpy(dst, src, n); + } + ASSERT_TRUE(verify_equal(dst, src, n)); + free(src); + free(dst); + return 0; +} + +static int test_async_bad_args(void) +{ + uint8_t a[64], b[64]; + void *dstv[1] = { a }, *srcv[1] = { b }; + size_t sizes[1] = { 64 }; + dto_async_op op; + dto_batch_op *bop; + + /* zero length never reaches the device */ + ASSERT_EQ(dto_submit_memcpy(&op, a, b, 0, 0), DTO_ASYNC_FALLBACK); + ASSERT_EQ(dto_submit_crc(&op, b, 0, 0), DTO_ASYNC_FALLBACK); + + /* a DSA batch needs at least two descriptors */ + bop = dto_batch_op_new(); + ASSERT_TRUE(bop != NULL); + ASSERT_EQ(dto_submit_batch_copy(bop, dstv, srcv, sizes, 1, 0), + DTO_ASYNC_FALLBACK); + dto_batch_op_free(bop); + return 0; +} + +static int test_async_batch_copy(void) +{ + enum { COUNT = 8 }; + const size_t base = 16 * 1024; + uint8_t *src[COUNT], *dst[COUNT]; + void *srcv[COUNT], *dstv[COUNT]; + size_t sizes[COUNT]; + dto_batch_op *bop; + int rc; + + for (int i = 0; i < COUNT; i++) { + sizes[i] = base * (i + 1) + i * 64; /* varied, some unaligned */ + src[i] = malloc(sizes[i]); + dst[i] = malloc(sizes[i]); + ASSERT_TRUE(src[i] && dst[i]); + fill_pattern(src[i], sizes[i]); + clear_buf(dst[i], sizes[i]); + srcv[i] = src[i]; + dstv[i] = dst[i]; + } + + bop = dto_batch_op_new(); + ASSERT_TRUE(bop != NULL); + rc = dto_submit_batch_copy(bop, dstv, srcv, sizes, COUNT, DTO_SUBMIT_BOF); + if (rc == DTO_ASYNC_SUBMITTED) { + rc = ASYNC_WAIT(dto_batch_poll(bop)); + ASSERT_EQ(rc, DTO_ASYNC_DONE); + /* a completed batch stays DONE */ + ASSERT_EQ(dto_batch_poll(bop), DTO_ASYNC_DONE); + } else { + ASSERT_EQ(rc, DTO_ASYNC_FALLBACK); + for (int i = 0; i < COUNT; i++) + memcpy(dst[i], src[i], sizes[i]); + } + for (int i = 0; i < COUNT; i++) { + ASSERT_TRUE(verify_equal(dst[i], src[i], sizes[i])); + free(src[i]); + free(dst[i]); + } + dto_batch_op_free(bop); + return 0; +} + int main(void) { printf("DTO Functional Tests\n"); @@ -495,6 +720,13 @@ int main(void) TEST_ENTRY(test_zero_length), TEST_ENTRY(test_memcpy_unaligned), TEST_ENTRY(test_multithread), + TEST_ENTRY(test_async_memcpy), + TEST_ENTRY(test_async_memset), + TEST_ENTRY(test_async_crc), + TEST_ENTRY(test_async_memcpy_crc), + TEST_ENTRY(test_async_memcpy_no_bof), + TEST_ENTRY(test_async_bad_args), + TEST_ENTRY(test_async_batch_copy), {NULL, NULL} };