diff --git a/llama-cpp-bindings-tests/tests/multimodal_vision.rs b/llama-cpp-bindings-tests/tests/multimodal_vision.rs index 0ee90825..874868a4 100644 --- a/llama-cpp-bindings-tests/tests/multimodal_vision.rs +++ b/llama-cpp-bindings-tests/tests/multimodal_vision.rs @@ -7,6 +7,8 @@ use llama_cpp_bindings::context::LlamaContext; use llama_cpp_bindings::error::EvalMultimodalChunksError; use llama_cpp_bindings::ingest_prompt_chunk::ingest_prompt_chunk; use llama_cpp_bindings::llama_batch::LlamaBatch; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunk; +use llama_cpp_bindings::mtmd::FittedMtmdInputChunks; use llama_cpp_bindings::mtmd::MtmdBitmap; use llama_cpp_bindings::mtmd::MtmdContext; use llama_cpp_bindings::mtmd::MtmdContextParams; @@ -697,6 +699,42 @@ fn eval_chunks_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn fitting_no_chunks_leaves_nothing_to_evaluate(fixture: &LlamaFixture<'_>) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_chunks = + FittedMtmdInputChunks::fit(MtmdInputChunks::new()?, mtmd_ctx, llama_ctx.n_ubatch())?; + + assert!(fitted_chunks.is_empty()); + assert!(FittedMtmdInputChunk::at(&fitted_chunks, 0).is_none()); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -719,13 +757,18 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> let mtmd_ctx = fixture .mtmd_context .expect("mmproj_file declared in attribute"); - let chunks = tokenize_synthetic(fixture, "Describe: <__media__>")?; - let first_chunk = chunks.get(0).context("tokenization produced no chunks")?; let llama_ctx = LlamaContext::from_model( fixture.model, fixture.backend, (*fixture.context_params).into_llama_context_params(), )?; + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + llama_ctx.n_ubatch(), + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; assert_eq!( first_chunk.eval_single(mtmd_ctx, &llama_ctx, 0, 0, 0, true), @@ -736,6 +779,64 @@ fn eval_single_rejects_a_zero_batch_before_evaluating(fixture: &LlamaFixture<'_> Ok(()) } +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "mmproj-F16.gguf"), +)] +#[llama_test( + model_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf"), + n_gpu_layers = 999, + load_mode = Mmap, + n_ctx = 512, + n_batch = 128, + n_ubatch = 64, + mmproj_source = HuggingFace("unsloth/Qwen3.6-35B-A3B-GGUF", "mmproj-F16.gguf"), +)] +fn eval_single_rejects_a_micro_batch_smaller_than_the_fitted_one_before_evaluating( + fixture: &LlamaFixture<'_>, +) -> Result<()> { + let mtmd_ctx = fixture + .mtmd_context + .expect("mmproj_file declared in attribute"); + let llama_ctx = LlamaContext::from_model( + fixture.model, + fixture.backend, + (*fixture.context_params).into_llama_context_params(), + )?; + let fitted_micro_batch_tokens = llama_ctx.n_ubatch(); + let fitted_chunks = FittedMtmdInputChunks::fit( + tokenize_synthetic(fixture, "Describe: <__media__>")?, + mtmd_ctx, + fitted_micro_batch_tokens, + )?; + let first_chunk = + FittedMtmdInputChunk::at(&fitted_chunks, 0).context("tokenization produced no chunks")?; + let smaller_micro_batch_tokens = fitted_micro_batch_tokens - 1; + + assert_eq!( + first_chunk.eval_single( + mtmd_ctx, + &llama_ctx, + 0, + 0, + i32::try_from(smaller_micro_batch_tokens)?, + true + ), + Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { + evaluated_micro_batch_tokens: smaller_micro_batch_tokens, + fitted_micro_batch_tokens, + }) + ); + assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); + + Ok(()) +} + #[llama_test( model_source = HuggingFace("unsloth/Qwen3.5-0.8B-GGUF", "Qwen3.5-0.8B-Q4_K_M.gguf"), n_gpu_layers = 999, @@ -1913,7 +2014,7 @@ fn gemma3_eval_chunks_rejects_a_non_causal_image_larger_than_the_micro_batch_bef n_ubatch = 128, mmproj_source = HuggingFace("unsloth/gemma-3-4b-it-GGUF", "mmproj-F16.gguf"), )] -fn gemma3_eval_single_rejects_a_non_causal_image_larger_than_the_micro_batch_before_evaluating( +fn gemma3_fitting_rejects_a_non_causal_image_larger_than_the_micro_batch( fixture: &LlamaFixture<'_>, ) -> Result<()> { let mtmd_ctx = fixture @@ -1931,14 +2032,7 @@ fn gemma3_eval_single_rejects_a_non_causal_image_larger_than_the_micro_batch_bef )?; assert_eq!( - image_chunk.eval_single( - mtmd_ctx, - &llama_ctx, - 0, - 0, - i32::try_from(llama_ctx.n_batch())?, - true - ), + image_chunk.fit_to_micro_batch(mtmd_ctx, llama_ctx.n_ubatch()), Err(MtmdEvalError::NonCausalChunkExceedsMicroBatch( NonCausalChunkMicroBatchMismatch { chunk_tokens: image_chunk.n_tokens(), @@ -1946,7 +2040,6 @@ fn gemma3_eval_single_rejects_a_non_causal_image_larger_than_the_micro_batch_bef } )) ); - assert_eq!(llama_ctx.kv_cache_seq_pos_max(0)?, -1); Ok(()) } diff --git a/llama-cpp-bindings/src/mtmd.rs b/llama-cpp-bindings/src/mtmd.rs index 36f29a2d..49da06dd 100644 --- a/llama-cpp-bindings/src/mtmd.rs +++ b/llama-cpp-bindings/src/mtmd.rs @@ -1,3 +1,5 @@ +pub mod fitted_mtmd_input_chunk; +pub mod fitted_mtmd_input_chunks; pub mod micro_batch_tokens; pub mod mtmd_bitmap; pub mod mtmd_bitmap_error; @@ -19,6 +21,8 @@ pub mod mtmd_tokenize_error; pub mod non_causal_chunk_micro_batch_mismatch; pub mod positive_batch_tokens; +pub use fitted_mtmd_input_chunk::FittedMtmdInputChunk; +pub use fitted_mtmd_input_chunks::FittedMtmdInputChunks; pub use micro_batch_tokens::micro_batch_tokens; pub use mtmd_bitmap::MtmdBitmap; pub use mtmd_bitmap_error::MtmdBitmapError; diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs new file mode 100644 index 00000000..2023fbb1 --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunk.rs @@ -0,0 +1,343 @@ +use std::borrow::Borrow; +use std::ffi::c_char; +use std::marker::PhantomData; + +use crate::context::LlamaContext; +use llama_cpp_ffi_status::read_and_free_cpp_string; + +use super::fitted_mtmd_input_chunks::FittedMtmdInputChunks; +use super::micro_batch_tokens::micro_batch_tokens; +use super::mtmd_context::MtmdContext; +use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_input_chunk::MtmdInputChunk; +use super::mtmd_input_chunks::MtmdInputChunks; +use super::positive_batch_tokens::positive_batch_tokens; + +fn eval_chunk_single_status_to_result( + status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, + final_position: llama_cpp_bindings_sys::llama_pos, + out_llama_cpp_return_code: i32, + out_error: *mut c_char, +) -> Result { + match status { + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { + Err(MtmdEvalError::EvalFailed { + code: out_llama_cpp_return_code, + }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { + Err(MtmdEvalError::NotEnoughMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { + Err(MtmdEvalError::LlamaCppOutOfMemory) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { + let message = unsafe { + read_and_free_cpp_string( + out_error, + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }?; + Err(MtmdEvalError::Reported { message }) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into()) + } + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into()) + } + other => Err(crate::FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: i64::from(other), + } + .into()), + } +} + +#[derive(Debug)] +pub struct FittedMtmdInputChunk<'fitted> { + chunk: MtmdInputChunk, + fitted_chunks: PhantomData<&'fitted MtmdInputChunks>, + micro_batch_tokens: u32, +} + +impl<'fitted> FittedMtmdInputChunk<'fitted> { + #[must_use] + pub fn at( + fitted_chunks: &'fitted FittedMtmdInputChunks, + chunk_index: usize, + ) -> Option + where + TChunks: Borrow, + { + fitted_chunks.get(chunk_index).map(|chunk| Self { + chunk, + fitted_chunks: PhantomData, + micro_batch_tokens: fitted_chunks.micro_batch_tokens(), + }) + } + + #[must_use] + pub const fn chunk(&self) -> &MtmdInputChunk { + &self.chunk + } + + /// # Errors + /// + /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, + /// [`MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted`] when decodes of `n_batch` in + /// `llama_ctx` are smaller than the micro batch the chunks were fitted to, or + /// [`MtmdEvalError::EvalFailed`] if the underlying encode or decode step fails. + pub fn eval_single( + &self, + mtmd_ctx: &MtmdContext, + llama_ctx: &LlamaContext, + start_position: llama_cpp_bindings_sys::llama_pos, + seq_id: llama_cpp_bindings_sys::llama_seq_id, + n_batch: i32, + logits_last: bool, + ) -> Result { + let evaluated_micro_batch_tokens = + micro_batch_tokens(llama_ctx, positive_batch_tokens(n_batch)?); + + if evaluated_micro_batch_tokens < self.micro_batch_tokens { + return Err(MtmdEvalError::EvaluatedMicroBatchSmallerThanFitted { + evaluated_micro_batch_tokens, + fitted_micro_batch_tokens: self.micro_batch_tokens, + }); + } + + let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; + let mut out_llama_cpp_return_code: i32 = 0; + let mut out_error: *mut c_char = std::ptr::null_mut(); + + let status = unsafe { + llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( + mtmd_ctx.context.as_ptr(), + llama_ctx.context.as_ptr(), + self.chunk.chunk.as_ptr(), + start_position, + seq_id, + n_batch, + logits_last, + &raw mut final_position, + &raw mut out_llama_cpp_return_code, + &raw mut out_error, + ) + }; + + eval_chunk_single_status_to_result( + status, + final_position, + out_llama_cpp_return_code, + out_error, + ) + } +} + +#[cfg(test)] +mod unit_tests { + use llama_cpp_ffi_status::read_and_free_cpp_string; + use llama_cpp_wrapper_error_fixture::wrapper_allocated_error_message_pointer; + + use super::eval_chunk_single_status_to_result; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + + #[test] + fn eval_chunk_single_status_cxx_exception_surfaces_the_reported_message() { + let expected_message = unsafe { + read_and_free_cpp_string( + wrapper_allocated_error_message_pointer(), + "llama_rs_mtmd_eval_chunk_single", + "reported a thrown C++ exception without an error message", + ) + }; + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + wrapper_allocated_error_message_pointer(), + ); + + assert_eq!( + result.err(), + expected_message + .ok() + .map(|message| MtmdEvalError::Reported { message }) + ); + } + + #[test] + fn eval_chunk_single_status_ok_returns_final_position() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, + 7, + 0, + std::ptr::null_mut(), + ); + + assert_eq!(result, Ok(7)); + } + + #[test] + fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, + 0, + -3, + std::ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); + } + + #[test] + fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, + 0, + 0, + std::ptr::null_mut(), + ); + + assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); + } + + #[test] + fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { + let result = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, + 0, + 0, + std::ptr::null_mut(), + ); + + assert_eq!( + result, + Err(crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "reported a thrown C++ exception without an error message", + } + .into()) + ); + } + + #[test] + fn eval_chunk_single_unknown_status_is_preserved() { + let result = eval_chunk_single_status_to_result(255, 0, 0, std::ptr::null_mut()); + + assert_eq!( + result, + Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { + operation: "llama_rs_mtmd_eval_chunk_single", + code: 255, + })) + ); + } +} + +#[cfg(test)] +mod ffi_contract_status_tests { + use super::eval_chunk_single_status_to_result; + use crate::mtmd::mtmd_eval_error::MtmdEvalError; + use std::ptr; + + #[test] + fn eval_chunk_single_status_to_result_maps_every_contract_status() { + let outcome_0 = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome_0.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null mtmd_ctx argument", + } + .into() + ) + ); + let outcome_1 = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome_1.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null llama_ctx argument", + } + .into() + ) + ); + let outcome_2 = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome_2.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null chunk argument", + } + .into() + ) + ); + let outcome_3 = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!( + outcome_3.err(), + Some( + crate::FfiContractError { + operation: "llama_rs_mtmd_eval_chunk_single", + detail: "was given a null out_new_n_past argument", + } + .into() + ) + ); + let outcome_4 = eval_chunk_single_status_to_result( + llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, + 0, + 0, + ptr::null_mut(), + ); + assert_eq!(outcome_4.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); + } +} diff --git a/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunks.rs b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunks.rs new file mode 100644 index 00000000..fbc6ad5e --- /dev/null +++ b/llama-cpp-bindings/src/mtmd/fitted_mtmd_input_chunks.rs @@ -0,0 +1,61 @@ +use std::borrow::Borrow; + +use super::mtmd_context::MtmdContext; +use super::mtmd_eval_error::MtmdEvalError; +use super::mtmd_input_chunk::MtmdInputChunk; +use super::mtmd_input_chunks::MtmdInputChunks; + +#[derive(Debug)] +pub struct FittedMtmdInputChunks +where + TChunks: Borrow, +{ + chunks: TChunks, + micro_batch_tokens: u32, +} + +impl FittedMtmdInputChunks +where + TChunks: Borrow, +{ + /// Checks every chunk once with [`MtmdInputChunks::fit_to_micro_batch`], so the chunks can + /// be evaluated without checking them again. + /// + /// # Errors + /// + /// Returns any error of [`MtmdInputChunks::fit_to_micro_batch`]. + pub fn fit( + chunks: TChunks, + mtmd_ctx: &MtmdContext, + micro_batch_tokens: u32, + ) -> Result { + chunks + .borrow() + .fit_to_micro_batch(mtmd_ctx, micro_batch_tokens)?; + + Ok(Self { + chunks, + micro_batch_tokens, + }) + } + + #[must_use] + pub fn get(&self, chunk_index: usize) -> Option { + self.chunks.borrow().get(chunk_index) + } + + #[must_use] + pub fn len(&self) -> usize { + self.chunks.borrow().len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.chunks.borrow().is_empty() + } + + #[must_use] + pub const fn micro_batch_tokens(&self) -> u32 { + self.micro_batch_tokens + } +} diff --git a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs index 9027b742..5f566306 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_eval_error.rs @@ -17,6 +17,13 @@ pub enum MtmdEvalError { .0.micro_batch_tokens, )] NonCausalChunkExceedsMicroBatch(NonCausalChunkMicroBatchMismatch), + #[error( + "a chunk fitted to decodes of {fitted_micro_batch_tokens} tokens cannot be evaluated in decodes of {evaluated_micro_batch_tokens}" + )] + EvaluatedMicroBatchSmallerThanFitted { + evaluated_micro_batch_tokens: u32, + fitted_micro_batch_tokens: u32, + }, #[error("multimodal chunk eval failed with code: {code}")] EvalFailed { code: i32 }, #[error("the chunk type could not be classified before evaluating it: {0}")] diff --git a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs index 35285060..4011d418 100644 --- a/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs +++ b/llama-cpp-bindings/src/mtmd/mtmd_input_chunk.rs @@ -1,20 +1,15 @@ use std::ffi::CStr; -use std::ffi::c_char; use std::ptr::NonNull; use std::slice; -use crate::context::LlamaContext; use crate::token::LlamaToken; -use llama_cpp_ffi_status::read_and_free_cpp_string; -use super::micro_batch_tokens::micro_batch_tokens; use super::mtmd_context::MtmdContext; use super::mtmd_eval_error::MtmdEvalError; use super::mtmd_input_chunk_error::MtmdInputChunkError; use super::mtmd_input_chunk_type::MtmdInputChunkType; use super::mtmd_input_chunk_type_error::MtmdInputChunkTypeError; use super::non_causal_chunk_micro_batch_mismatch::NonCausalChunkMicroBatchMismatch; -use super::positive_batch_tokens::positive_batch_tokens; /// # Safety /// @@ -36,71 +31,6 @@ const unsafe fn tokens_from_raw_ptr<'chunk>( } } -fn eval_chunk_single_status_to_result( - status: llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single_status, - final_position: llama_cpp_bindings_sys::llama_pos, - out_llama_cpp_return_code: i32, - out_error: *mut c_char, -) -> Result { - match status { - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK => Ok(final_position), - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE => { - Err(MtmdEvalError::EvalFailed { - code: out_llama_cpp_return_code, - }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED => { - Err(MtmdEvalError::NotEnoughMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY => { - Err(MtmdEvalError::LlamaCppOutOfMemory) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION => { - let message = unsafe { - read_and_free_cpp_string( - out_error, - "llama_rs_mtmd_eval_chunk_single", - "reported a thrown C++ exception without an error message", - ) - }?; - Err(MtmdEvalError::Reported { message }) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into()) - } - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG => { - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into()) - } - other => Err(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: i64::from(other), - } - .into()), - } -} - fn non_causal_chunk_micro_batch_error( decodes_non_causally: bool, chunk_tokens: usize, @@ -213,53 +143,6 @@ impl MtmdInputChunk { ) .map_or(Ok(()), Err) } - - /// # Errors - /// - /// Returns [`MtmdEvalError::NonPositiveBatchSize`] when `n_batch` is not positive, - /// [`MtmdEvalError::NonCausalChunkExceedsMicroBatch`] when this chunk has to fit a single - /// decode but does not, or [`MtmdEvalError::EvalFailed`] if the underlying encode or decode - /// step fails. - pub fn eval_single( - &self, - mtmd_ctx: &MtmdContext, - llama_ctx: &LlamaContext, - start_position: llama_cpp_bindings_sys::llama_pos, - seq_id: llama_cpp_bindings_sys::llama_seq_id, - n_batch: i32, - logits_last: bool, - ) -> Result { - self.fit_to_micro_batch( - mtmd_ctx, - micro_batch_tokens(llama_ctx, positive_batch_tokens(n_batch)?), - )?; - - let mut final_position: llama_cpp_bindings_sys::llama_pos = start_position; - let mut out_llama_cpp_return_code: i32 = 0; - let mut out_error: *mut c_char = std::ptr::null_mut(); - - let status = unsafe { - llama_cpp_bindings_sys::llama_rs_mtmd_eval_chunk_single( - mtmd_ctx.context.as_ptr(), - llama_ctx.context.as_ptr(), - self.chunk.as_ptr(), - start_position, - seq_id, - n_batch, - logits_last, - &raw mut final_position, - &raw mut out_llama_cpp_return_code, - &raw mut out_error, - ) - }; - - eval_chunk_single_status_to_result( - status, - final_position, - out_llama_cpp_return_code, - out_error, - ) - } } impl Drop for MtmdInputChunk { @@ -272,7 +155,6 @@ impl Drop for MtmdInputChunk { #[cfg(test)] mod unit_tests { - use super::eval_chunk_single_status_to_result; use super::non_causal_chunk_micro_batch_error; use super::tokens_from_raw_ptr; use crate::mtmd::mtmd_eval_error::MtmdEvalError; @@ -298,74 +180,6 @@ mod unit_tests { assert_eq!(result.unwrap().len(), 2); } - #[test] - fn eval_chunk_single_status_ok_returns_final_position() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_OK, - 7, - 0, - std::ptr::null_mut(), - ); - - assert_eq!(result, Ok(7)); - } - - #[test] - fn eval_chunk_single_status_nonzero_code_maps_to_eval_failed() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_RETURNED_NONZERO_CODE, - 0, - -3, - std::ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::EvalFailed { code: -3 })); - } - - #[test] - fn eval_chunk_single_status_allocation_failed_maps_to_not_enough_memory() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_ERROR_STRING_ALLOCATION_FAILED, - 0, - 0, - std::ptr::null_mut(), - ); - - assert_eq!(result, Err(MtmdEvalError::NotEnoughMemory)); - } - - #[test] - fn eval_chunk_single_status_cxx_exception_reports_unknown_error_for_null() { - let result = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_THREW_CXX_EXCEPTION, - 0, - 0, - std::ptr::null_mut(), - ); - - assert_eq!( - result, - Err(crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "reported a thrown C++ exception without an error message", - } - .into()) - ); - } - - #[test] - fn eval_chunk_single_unknown_status_is_preserved() { - let result = eval_chunk_single_status_to_result(255, 0, 0, std::ptr::null_mut()); - - assert_eq!( - result, - Err(MtmdEvalError::FfiStatus(crate::FfiStatusError { - operation: "llama_rs_mtmd_eval_chunk_single", - code: 255, - })) - ); - } - #[test] fn a_non_causal_chunk_larger_than_the_micro_batch_reports_the_mismatch() { assert_eq!( @@ -389,85 +203,3 @@ mod unit_tests { assert!(non_causal_chunk_micro_batch_error(true, 4, 4).is_none()); } } - -#[cfg(test)] -mod ffi_contract_status_tests { - use super::eval_chunk_single_status_to_result; - use crate::mtmd::mtmd_eval_error::MtmdEvalError; - use std::ptr; - - #[test] - fn eval_chunk_single_status_to_result_maps_every_contract_status() { - let outcome_0 = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_MTMD_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_0.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null mtmd_ctx argument", - } - .into() - ) - ); - let outcome_1 = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_LLAMA_CTX_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_1.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null llama_ctx argument", - } - .into() - ) - ); - let outcome_2 = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_CHUNK_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_2.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null chunk argument", - } - .into() - ) - ); - let outcome_3 = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_NULL_OUT_NEW_N_PAST_ARG, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!( - outcome_3.err(), - Some( - crate::FfiContractError { - operation: "llama_rs_mtmd_eval_chunk_single", - detail: "was given a null out_new_n_past argument", - } - .into() - ) - ); - let outcome_4 = eval_chunk_single_status_to_result( - llama_cpp_bindings_sys::LLAMA_RS_MTMD_EVAL_CHUNK_SINGLE_LLAMA_CPP_OUT_OF_MEMORY, - 0, - 0, - ptr::null_mut(), - ); - assert_eq!(outcome_4.err(), Some(MtmdEvalError::LlamaCppOutOfMemory)); - } -} diff --git a/llama-cpp-bindings/src/sampled_token_classifier.rs b/llama-cpp-bindings/src/sampled_token_classifier.rs index 7db1e2f2..e95ce8fc 100644 --- a/llama-cpp-bindings/src/sampled_token_classifier.rs +++ b/llama-cpp-bindings/src/sampled_token_classifier.rs @@ -18,6 +18,8 @@ use crate::generation_progress::GenerationProgress; use crate::json_probe_outcome::JsonProbeOutcome; use crate::llama_batch::LlamaBatch; use crate::model::LlamaModel; +use crate::mtmd::FittedMtmdInputChunk; +use crate::mtmd::FittedMtmdInputChunks; use crate::mtmd::MtmdContext; use crate::mtmd::MtmdInputChunks; use crate::mtmd::micro_batch_tokens; @@ -467,21 +469,20 @@ impl<'model> SampledTokenClassifier<'model> { llama_ctx: &LlamaContext, params: EvalMultimodalChunksParams, ) -> Result { - chunks.fit_to_micro_batch( + let fitted_chunks = FittedMtmdInputChunks::fit( + chunks, mtmd_ctx, micro_batch_tokens(llama_ctx, positive_batch_tokens(params.n_batch)?), )?; - - let chunk_count = chunks.len(); + let chunk_count = fitted_chunks.len(); let mut next_position = params.start_position; for index in 0..chunk_count { - let chunk = chunks - .get(index) + let fitted_chunk = FittedMtmdInputChunk::at(&fitted_chunks, index) .ok_or(EvalMultimodalChunksError::ChunkOutOfBounds(index))?; let logits_for_this_chunk = params.logits_last && index + 1 == chunk_count; - next_position = chunk.eval_single( + next_position = fitted_chunk.eval_single( mtmd_ctx, llama_ctx, next_position, @@ -489,7 +490,7 @@ impl<'model> SampledTokenClassifier<'model> { params.n_batch, logits_for_this_chunk, )?; - crate::ingest_prompt_chunk::ingest_prompt_chunk(self, &chunk)?; + crate::ingest_prompt_chunk::ingest_prompt_chunk(self, fitted_chunk.chunk())?; } Ok(next_position)