From aa87cbfc4be640ab4ce3bb6c39e3d71f1315de57 Mon Sep 17 00:00:00 2001 From: Pierre-Antoine Bannier Date: Fri, 11 Sep 2026 13:46:57 +0200 Subject: [PATCH] chore: remove test/debug API and tests/ tests/ was the only consumer of the sam3_test_* functions, the state/model tensor dump and info helpers, sam3_encode_image_from_preprocessed, sam3_encode_vit_from_preprocessed_selective, struct sam3_tensor_info and the sam3_vit_block_stage / sam3_vit_prefix_stage enums. Remove them along with the static helpers only they used, the Windows mkdir macro and its includes, and the tests/ directory itself. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01M2qLmQ9ag49hT61V6i9qAq --- sam3.cpp | 11583 ++++++---------- sam3.h | 219 - tests/CMakeLists.txt | 129 - tests/analyze_divergence.py | 114 - tests/analyze_f16_divergence.py | 96 - tests/cat.jpg | Bin 215264 -> 0 bytes tests/check_f16_weight_error.py | 180 - tests/check_preprocessing.py | 73 - tests/check_rounding.py | 31 - tests/compare_actual_resize.py | 51 - tests/compare_cpp_dump_vs_finterp.py | 62 - tests/compare_ddec_tensors.py | 218 - tests/compare_f32_vs_f16.py | 105 - tests/compare_mask_decoder.py | 165 - tests/compare_preprocessed_float.py | 45 - tests/compare_preprocessing.py | 118 - tests/compare_resize.py | 100 - tests/compare_sam2_video.py | 122 - tests/compare_tensors.py | 57 - tests/compare_vit_output.py | 96 - tests/dump_cpp_preprocessed.py | 62 - tests/dump_ddec_from_package.py | 411 - tests/dump_fenc_from_package.py | 469 - tests/dump_geom_encoder_reference.py | 482 - tests/dump_phase3_reference.py | 427 - tests/dump_phase4_reference.py | 257 - tests/dump_phase5_reference.py | 778 -- tests/dump_phase6_reference.py | 493 - tests/dump_phase7_reference.py | 836 -- tests/dump_reference.py | 169 - tests/dump_sam2_pvs_reference.py | 132 - tests/dump_sam2_reference.py | 231 - tests/dump_vit_reference.py | 307 - tests/llama.jpg | Bin 79236 -> 0 bytes tests/phase4_prompts.tsv | 5 - tests/phase6_cases.tsv | 3 - tests/phase7_cases.tsv | 3 - tests/test_debug_encoder.cpp | 242 - tests/test_debug_fenc.cpp | 174 - tests/test_e2e_pcs.cpp | 468 - tests/test_e2e_pvs.cpp | 333 - tests/test_full_pipeline.py | 57 - tests/test_geom_enc.cpp | 142 - tests/test_jpeg_decode.cpp | 45 - tests/test_llama_pcs.cpp | 76 - tests/test_load.cpp | 38 - tests/test_mem_enc.cpp | 110 - tests/test_metal_block0_stage_compare.cpp | 398 - tests/test_metal_block_stage_compare.cpp | 535 - tests/test_metal_conv_dw.cpp | 214 - tests/test_metal_conv_transpose_2d.cpp | 338 - tests/test_metal_conv_transpose_2d_stress.cpp | 219 - tests/test_metal_encoder_checkpoints.cpp | 203 - ...t_metal_flash_attn_ext_block15_compare.cpp | 715 - tests/test_metal_overhead.cpp | 377 - tests/test_metal_phase6.cpp | 195 - tests/test_metal_profile.cpp | 215 - tests/test_metal_python_ref_compare.cpp | 503 - tests/test_metal_real_profile.cpp | 108 - tests/test_metal_vit.cpp | 166 - tests/test_metal_vit_prefix_compare.cpp | 187 - tests/test_metal_vit_upstream_compare.cpp | 383 - tests/test_metal_win.cpp | 252 - tests/test_phase3.cpp | 632 - tests/test_phase4.cpp | 334 - tests/test_phase5.cpp | 256 - tests/test_phase6.cpp | 287 - tests/test_phase7.cpp | 244 - tests/test_preprocess_compare.py | 70 - tests/test_preprocess_match.cpp | 57 - tests/test_pvs_point.cpp | 72 - tests/test_random.jpg | Bin 185354 -> 0 bytes tests/test_resize_from_pil.py | 70 - tests/test_sam2_backbone_compare.cpp | 47 - tests/test_sam2_e2e_all_variants.py | 345 - tests/test_sam2_pe_cache_size.cpp | 116 - tests/test_sam2_pvs_compare.cpp | 74 - tests/test_sam2_video_debug.cpp | 176 - tests/test_text_enc_cat.cpp | 75 - tests/test_text_enc_dump.cpp | 58 - tests/test_tokenizer.cpp | 87 - tests/test_utils.h | 194 - tests/test_v_cont_removal.cpp | 183 - tests/test_video_2frame.cpp | 107 - tests/test_video_text.cpp | 89 - tests/test_visual_only.cpp | 102 - tests/test_visual_only_compare.cpp | 260 - tests/test_vit.cpp | 465 - tests/trace_single_pixel.py | 130 - tests/trace_torchvision_resize.py | 124 - tests/visualize_cpp_masks.py | 65 - 91 files changed, 4111 insertions(+), 25930 deletions(-) delete mode 100644 tests/CMakeLists.txt delete mode 100644 tests/analyze_divergence.py delete mode 100644 tests/analyze_f16_divergence.py delete mode 100644 tests/cat.jpg delete mode 100644 tests/check_f16_weight_error.py delete mode 100644 tests/check_preprocessing.py delete mode 100644 tests/check_rounding.py delete mode 100644 tests/compare_actual_resize.py delete mode 100644 tests/compare_cpp_dump_vs_finterp.py delete mode 100644 tests/compare_ddec_tensors.py delete mode 100644 tests/compare_f32_vs_f16.py delete mode 100644 tests/compare_mask_decoder.py delete mode 100644 tests/compare_preprocessed_float.py delete mode 100644 tests/compare_preprocessing.py delete mode 100644 tests/compare_resize.py delete mode 100644 tests/compare_sam2_video.py delete mode 100644 tests/compare_tensors.py delete mode 100644 tests/compare_vit_output.py delete mode 100644 tests/dump_cpp_preprocessed.py delete mode 100644 tests/dump_ddec_from_package.py delete mode 100644 tests/dump_fenc_from_package.py delete mode 100644 tests/dump_geom_encoder_reference.py delete mode 100644 tests/dump_phase3_reference.py delete mode 100644 tests/dump_phase4_reference.py delete mode 100644 tests/dump_phase5_reference.py delete mode 100644 tests/dump_phase6_reference.py delete mode 100644 tests/dump_phase7_reference.py delete mode 100644 tests/dump_reference.py delete mode 100644 tests/dump_sam2_pvs_reference.py delete mode 100644 tests/dump_sam2_reference.py delete mode 100644 tests/dump_vit_reference.py delete mode 100644 tests/llama.jpg delete mode 100644 tests/phase4_prompts.tsv delete mode 100644 tests/phase6_cases.tsv delete mode 100644 tests/phase7_cases.tsv delete mode 100644 tests/test_debug_encoder.cpp delete mode 100644 tests/test_debug_fenc.cpp delete mode 100644 tests/test_e2e_pcs.cpp delete mode 100644 tests/test_e2e_pvs.cpp delete mode 100644 tests/test_full_pipeline.py delete mode 100644 tests/test_geom_enc.cpp delete mode 100644 tests/test_jpeg_decode.cpp delete mode 100644 tests/test_llama_pcs.cpp delete mode 100644 tests/test_load.cpp delete mode 100644 tests/test_mem_enc.cpp delete mode 100644 tests/test_metal_block0_stage_compare.cpp delete mode 100644 tests/test_metal_block_stage_compare.cpp delete mode 100644 tests/test_metal_conv_dw.cpp delete mode 100644 tests/test_metal_conv_transpose_2d.cpp delete mode 100644 tests/test_metal_conv_transpose_2d_stress.cpp delete mode 100644 tests/test_metal_encoder_checkpoints.cpp delete mode 100644 tests/test_metal_flash_attn_ext_block15_compare.cpp delete mode 100644 tests/test_metal_overhead.cpp delete mode 100644 tests/test_metal_phase6.cpp delete mode 100644 tests/test_metal_profile.cpp delete mode 100644 tests/test_metal_python_ref_compare.cpp delete mode 100644 tests/test_metal_real_profile.cpp delete mode 100644 tests/test_metal_vit.cpp delete mode 100644 tests/test_metal_vit_prefix_compare.cpp delete mode 100644 tests/test_metal_vit_upstream_compare.cpp delete mode 100644 tests/test_metal_win.cpp delete mode 100644 tests/test_phase3.cpp delete mode 100644 tests/test_phase4.cpp delete mode 100644 tests/test_phase5.cpp delete mode 100644 tests/test_phase6.cpp delete mode 100644 tests/test_phase7.cpp delete mode 100644 tests/test_preprocess_compare.py delete mode 100644 tests/test_preprocess_match.cpp delete mode 100644 tests/test_pvs_point.cpp delete mode 100644 tests/test_random.jpg delete mode 100644 tests/test_resize_from_pil.py delete mode 100644 tests/test_sam2_backbone_compare.cpp delete mode 100644 tests/test_sam2_e2e_all_variants.py delete mode 100644 tests/test_sam2_pe_cache_size.cpp delete mode 100644 tests/test_sam2_pvs_compare.cpp delete mode 100644 tests/test_sam2_video_debug.cpp delete mode 100644 tests/test_text_enc_cat.cpp delete mode 100644 tests/test_text_enc_dump.cpp delete mode 100644 tests/test_tokenizer.cpp delete mode 100644 tests/test_utils.h delete mode 100644 tests/test_v_cont_removal.cpp delete mode 100644 tests/test_video_2frame.cpp delete mode 100644 tests/test_video_text.cpp delete mode 100644 tests/test_visual_only.cpp delete mode 100644 tests/test_visual_only_compare.cpp delete mode 100644 tests/test_vit.cpp delete mode 100644 tests/trace_single_pixel.py delete mode 100644 tests/trace_torchvision_resize.py delete mode 100644 tests/visualize_cpp_masks.py diff --git a/sam3.cpp b/sam3.cpp index 43d88a8..c7148a6 100644 --- a/sam3.cpp +++ b/sam3.cpp @@ -18,15 +18,12 @@ #define STB_IMAGE_WRITE_IMPLEMENTATION #ifdef _WIN32 -#include #include #define popen _popen #define pclose _pclose -#define mkdir(path, mode) _mkdir(path) #define SAM3_NULL_DEV "NUL" #define SAM3_POPEN_READ "rb" #else -#include #define SAM3_NULL_DEV "/dev/null" #define SAM3_POPEN_READ "r" #endif @@ -1009,9 +1006,6 @@ static struct ggml_tensor* sam3_layer_norm_2d(struct ggml_context* ctx, struct ggml_tensor* w, struct ggml_tensor* b); -static bool sam3_copy_tensor_to_f32(struct ggml_tensor * t, - std::vector & output); - /***************************************************************************** ** Internal Helper Implementations *****************************************************************************/ @@ -6215,6449 +6209,3311 @@ bool sam3_encode_image(sam3_state& state, return true; } -static void sam3_clear_encoder_state(sam3_state & state) { - state.vit_output = nullptr; - for (int i = 0; i < 4; ++i) { - state.neck_det[i] = nullptr; - state.neck_trk[i] = nullptr; - state.neck_det_pe[i] = nullptr; - state.neck_trk_pe[i] = nullptr; - } -} - -static bool sam3_mark_named_outputs(struct ggml_context * ctx, - const std::vector & output_tensors) { - for (const auto & name : output_tensors) { - struct ggml_tensor * t = ggml_get_tensor(ctx, name.c_str()); - if (!t) { - fprintf(stderr, "%s: requested tensor '%s' was not found\n", __func__, name.c_str()); - return false; - } - ggml_set_output(t); - } - return true; -} - -bool sam3_encode_vit_from_preprocessed_selective(sam3_state & state, - const sam3_model & model, - const float * chw_data, - int img_size, - const std::vector & output_tensors) { - const auto & hp = model.hparams; - - if (img_size != hp.img_size) { - fprintf(stderr, "%s: img_size mismatch: got %d, expected %d\n", - __func__, img_size, hp.img_size); - return false; - } - - state.orig_width = img_size; - state.orig_height = img_size; - - const size_t buf_size = ggml_tensor_overhead() * 4096 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/ buf_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - struct ggml_context * ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; - } +/***************************************************************************** +** Multi-head attention helper (used by fusion encoder, DETR decoder, seg head) +*****************************************************************************/ - auto * inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img_size, img_size, 3, 1); - ggml_set_name(inp, "input_image"); - ggml_set_input(inp); +// Standard multi-head attention with fused in_proj. +// q_in, k_in, v_in: [D, N, B] (if fused_qkv, only q_in is used and contains QKV stacked) +// in_proj_w: [D, 3*D] (fused Q/K/V projection) +// in_proj_b: [3*D] +// out_proj_w: [D, D], out_proj_b: [D] +// n_heads: number of attention heads +// Returns: [D, N_q, B] +// +// If separate_kv is true, q_in/k_in/v_in are already separate (no fused proj needed). +// The in_proj is applied to form Q from q_in, and K/V from the concatenated k/v source. +static struct ggml_tensor* sam3_multihead_attn_fused( + struct ggml_context* ctx, + struct ggml_tensor* q_in, // [D, N_q, B] + struct ggml_tensor* kv_in, // [D, N_kv, B] (can be same as q_in for self-attn) + struct ggml_tensor* in_proj_w, // [D, 3*D] — fused QKV weights + struct ggml_tensor* in_proj_b, // [3*D] + struct ggml_tensor* out_proj_w, // [D, D] + struct ggml_tensor* out_proj_b, // [D] + int n_heads, + struct ggml_tensor* attn_mask = nullptr) // [N_kv, N_q] or nullptr +{ + const int64_t D = q_in->ne[0]; // 256 + const int64_t N_q = q_in->ne[1]; + const int64_t B = q_in->ne[2]; + const int64_t N_kv = kv_in->ne[1]; + const int64_t HD = D / n_heads; - auto * vit_out = sam3_build_vit_graph(ctx0, inp, model); - ggml_set_name(vit_out, "vit_output"); - ggml_set_output(vit_out); + auto* q_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], D * in_proj_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], 2 * D * in_proj_w->nb[1]); - if (!sam3_mark_named_outputs(ctx0, output_tensors)) { - ggml_free(ctx0); - return false; - } + auto* q_b = ggml_view_1d(ctx, in_proj_b, D, 0); + auto* k_b = ggml_view_1d(ctx, in_proj_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, in_proj_b, D, 2 * D * sizeof(float)); - struct ggml_cgraph * graph = ggml_new_graph_custom(ctx0, 8192, false); - ggml_build_forward_expand(graph, vit_out); + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, kv_in), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, kv_in), v_b); - auto * galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph)) { - fprintf(stderr, "%s: failed to reserve graph memory\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); // [HD, N_q, NH, B] - if (!ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); // [HD, N_kv, NH, B] - const size_t data_bytes = (size_t) 3 * img_size * img_size * sizeof(float); - ggml_backend_tensor_set(inp, chw_data, 0, data_bytes); - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); // [HD, N_kv, NH, B] non-contiguous; flash_attn uses strides - if (state.galloc) { - ggml_gallocr_free(state.galloc); - } - if (state.ctx) { - ggml_free(state.ctx); - } - if (state.pe_buf) { - ggml_backend_buffer_free(state.pe_buf); - state.pe_buf = nullptr; - } - if (state.pe_ctx) { - ggml_free(state.pe_ctx); - state.pe_ctx = nullptr; - } + float scale = 1.0f / sqrtf((float)HD); + auto* attn_out = ggml_flash_attn_ext(ctx, Q, K, V, attn_mask, scale, 0.0f, 0.0f); - state.ctx = ctx0; - state.galloc = galloc; - state.backend = model.backend; - sam3_clear_encoder_state(state); - state.vit_output = vit_out; - state.pe_cache_valid = false; + auto* merged = ggml_reshape_3d(ctx, attn_out, D, N_q, B); + merged = ggml_mul_mat(ctx, out_proj_w, merged); + merged = ggml_add(ctx, merged, out_proj_b); - return true; + return merged; } -static void sam3_normalize_ne4(const int64_t input_ne[4], int64_t ne[4]); -static struct ggml_tensor * sam3_new_f32_tensor_4d_from_ne(struct ggml_context * ctx, - const int64_t ne[4]); -static bool sam3_copy_tensor_to_f32(struct ggml_tensor * t, std::vector & output); -static struct ggml_tensor * sam3_build_vit_prefix_stage_from_input(struct ggml_context * ctx, - struct ggml_tensor * input, - const sam3_model & model, - sam3_vit_prefix_stage stage); - -bool sam3_test_run_vit_block0_input(const sam3_model & model, - const float * chw_data, - int img_size, - std::vector & output_data, - int64_t output_ne[4], - int n_threads) { - if (!chw_data) { - fprintf(stderr, "%s: chw_data is null\n", __func__); - return false; - } - if (img_size != model.hparams.img_size) { - fprintf(stderr, "%s: img_size mismatch: got %d, expected %d\n", - __func__, img_size, model.hparams.img_size); - return false; +static struct ggml_tensor* sam3_expand_token_attn_bias( + struct ggml_context* ctx, + struct ggml_tensor* token_bias, // [T, 1, B] or nullptr + int64_t n_q, + int n_heads, + int64_t batch) { + if (!token_bias) { + return nullptr; } - const size_t ctx_size = ggml_tensor_overhead() * 256 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - - struct ggml_context * ctx = ggml_init(params); - if (!ctx) { - fprintf(stderr, "%s: failed to create ggml context\n", __func__); - return false; - } + const int64_t n_kv = token_bias->ne[0]; + auto* bias_4d = ggml_reshape_4d(ctx, token_bias, n_kv, 1, 1, batch); + auto* full_bias = ggml_repeat( + ctx, + bias_4d, + ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_kv, n_q, n_heads, batch)); - struct ggml_tensor * input = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, img_size, img_size, 3, 1); - ggml_set_name(input, "vit_prefix_input"); - ggml_set_input(input); + // ggml_flash_attn_ext expects mask storage in fp16. + return ggml_cont(ctx, ggml_cast(ctx, full_bias, GGML_TYPE_F16)); +} - struct ggml_tensor * output = sam3_build_vit_prefix_graph(ctx, input, model); - ggml_set_name(output, "vit_block0_input"); - ggml_set_output(output); +/***************************************************************************** +** Geometry / exemplar encoder — graph building +*****************************************************************************/ - struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, 1024, false); - ggml_build_forward_expand(graph, output); +// Sinusoidal positional encoding for box coordinates. +// Matches Python PositionEmbeddingSine.encode_boxes(cx, cy, w, h). +// Output: [258] = [pos_y(128), pos_x(128), h, w] +static void sam3_sine_encode_box(float* out, float cx, float cy, float w, float h, + int num_pos_feats, int temperature) { + const float scale = 2.0f * (float)M_PI; + float x_embed = cx * scale; + float y_embed = cy * scale; - auto * galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return false; + // dim_t[i] = temperature^(2*(i//2)/num_pos_feats) + for (int i = 0; i < num_pos_feats; ++i) { + int div_idx = 2 * (i / 2); + float dim_t = powf((float)temperature, (float)div_idx / (float)num_pos_feats); + float px = x_embed / dim_t; + float py = y_embed / dim_t; + // Interleaved sin/cos: even indices get sin, odd get cos + if (i % 2 == 0) { + out[i] = sinf(py); // pos_y first + out[num_pos_feats + i] = sinf(px); // pos_x second + } else { + out[i] = cosf(py); + out[num_pos_feats + i] = cosf(px); + } } + out[2 * num_pos_feats] = h; // h + out[2 * num_pos_feats + 1] = w; // w +} - const size_t input_bytes = (size_t) 3 * img_size * img_size * sizeof(float); - ggml_backend_tensor_set(input, chw_data, 0, input_bytes); - sam3_graph_compute(model.backend, graph, n_threads); +// CPU-side ROI Align matching torchvision.ops.roi_align behavior. +// Features in ggml [C, W, H] layout. Box in XYXY format scaled to feature grid coords. +// Uses sub-sampling matching torchvision's sampling_ratio=0 (auto). +// Output: [C * roi_size * roi_size] in ggml layout. +static void sam3_roi_align_single( + const float* feats, // [C, W, H] ggml layout (C innermost, then W, then H) + int C, int W_feat, int H_feat, + float x0, float y0, float x1, float y1, + int roi_size, + float* out) // [C, roi_size, roi_size] +{ + float roi_w = std::max(x1 - x0, 1e-6f); + float roi_h = std::max(y1 - y0, 1e-6f); + float bin_w = roi_w / (float)roi_size; + float bin_h = roi_h / (float)roi_size; - for (int i = 0; i < 4; ++i) { - output_ne[i] = output->ne[i]; - } + // Match torchvision sampling_ratio=0: use ceil(bin_size) sub-samples + int sample_y_count = std::max(1, (int)ceilf(bin_h)); + int sample_x_count = std::max(1, (int)ceilf(bin_w)); + float inv_count = 1.0f / (float)(sample_y_count * sample_x_count); - const bool ok = sam3_copy_tensor_to_f32(output, output_data); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return ok; -} - -bool sam3_test_run_vit_prefix_stage(const sam3_model & model, - sam3_vit_prefix_stage stage, - const float * input_data, - const int64_t input_ne[4], - std::vector & output_data, - int64_t output_ne[4], - int n_threads) { - if (!input_data) { - fprintf(stderr, "%s: input_data is null\n", __func__); - return false; - } + auto bilinear_sample = [&](float sx, float sy, int c) -> float { + // Clamp to [-0.5, W-0.5] then adjust — matching torchvision + if (sx < -1.0f || sx > (float)W_feat || sy < -1.0f || sy > (float)H_feat) + return 0.0f; - int64_t ne[4]; - sam3_normalize_ne4(input_ne, ne); + sy = std::max(0.0f, sy); + sx = std::max(0.0f, sx); - switch (stage) { - case SAM3_VIT_PREFIX_STAGE_PATCH_IM2COL: - case SAM3_VIT_PREFIX_STAGE_PATCH_EMBED: - if (ne[0] != model.hparams.img_size || ne[1] != model.hparams.img_size || ne[2] != 3) { - fprintf(stderr, "%s: patch_embed input shape mismatch [%lld,%lld,%lld,%lld]\n", - __func__, - (long long) ne[0], (long long) ne[1], (long long) ne[2], (long long) ne[3]); - return false; - } - break; + int y_lo = (int)sy; + int x_lo = (int)sx; + int y_hi = std::min(y_lo + 1, H_feat - 1); + int x_hi = std::min(x_lo + 1, W_feat - 1); + y_lo = std::min(y_lo, H_feat - 1); + x_lo = std::min(x_lo, W_feat - 1); - case SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW: - case SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT: - if (ne[0] != model.vit.patch_embed_w->ne[0] * model.vit.patch_embed_w->ne[1] * model.vit.patch_embed_w->ne[2] || - ne[1] != model.hparams.n_img_embd() || - ne[2] != model.hparams.n_img_embd()) { - fprintf(stderr, "%s: patch_mulmat input shape mismatch [%lld,%lld,%lld,%lld]\n", - __func__, - (long long) ne[0], (long long) ne[1], (long long) ne[2], (long long) ne[3]); - return false; - } - break; + float ly = sy - (float)y_lo; + float lx = sx - (float)x_lo; - default: - if (ne[0] != model.hparams.vit_embed_dim || - ne[1] != model.hparams.n_img_embd() || - ne[2] != model.hparams.n_img_embd()) { - fprintf(stderr, "%s: prefix feature input shape mismatch [%lld,%lld,%lld,%lld]\n", - __func__, - (long long) ne[0], (long long) ne[1], (long long) ne[2], (long long) ne[3]); - return false; - } - break; - } + // ggml layout: feats[c + x * C + y * C * W_feat] + float v00 = feats[c + x_lo * C + y_lo * C * W_feat]; + float v10 = feats[c + x_hi * C + y_lo * C * W_feat]; + float v01 = feats[c + x_lo * C + y_hi * C * W_feat]; + float v11 = feats[c + x_hi * C + y_hi * C * W_feat]; - const size_t ctx_size = ggml_tensor_overhead() * 128 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, + return v00 * (1 - ly) * (1 - lx) + v10 * (1 - ly) * lx + v01 * ly * (1 - lx) + v11 * ly * lx; }; - struct ggml_context * ctx = ggml_init(params); - if (!ctx) { - fprintf(stderr, "%s: failed to create ggml context\n", __func__); - return false; + for (int ph = 0; ph < roi_size; ++ph) { + for (int pw = 0; pw < roi_size; ++pw) { + for (int c = 0; c < C; ++c) { + float sum = 0.0f; + for (int iy = 0; iy < sample_y_count; ++iy) { + float sy = y0 + bin_h * ((float)ph + ((float)iy + 0.5f) / (float)sample_y_count); + for (int ix = 0; ix < sample_x_count; ++ix) { + float sx = x0 + bin_w * ((float)pw + ((float)ix + 0.5f) / (float)sample_x_count); + sum += bilinear_sample(sx, sy, c); + } + } + out[c + pw * C + ph * C * roi_size] = sum * inv_count; + } + } } +} - struct ggml_tensor * input = nullptr; - if (stage == SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW || stage == SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT) { - input = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, ne[0], ne[1], ne[2], ne[3]); - } else { - input = sam3_new_f32_tensor_4d_from_ne(ctx, ne); - } - ggml_set_name(input, "vit_prefix_stage_input"); - ggml_set_input(input); +// Build geometry encoder graph and pre-compute box embeddings on CPU. +// Returns the geometry features as a pre-computed input tensor [D, N_geo, 1] +// where N_geo = n_exemplar_boxes + 1 (CLS). +// For dummy prompts (no boxes), N_geo = 1 (just CLS token). +struct sam3_geom_result { + struct ggml_tensor* geo_feats; // [D, N_geo, 1] + int n_tokens; // N_geo +}; - struct ggml_tensor * output = sam3_build_vit_prefix_stage_from_input(ctx, input, model, stage); - if (!output) { - fprintf(stderr, "%s: failed to build prefix stage %d\n", __func__, (int) stage); - ggml_free(ctx); - return false; - } - ggml_set_name(output, "vit_prefix_stage_output"); - ggml_set_output(output); - - struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, 512, false); - ggml_build_forward_expand(graph, output); - - auto * galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return false; - } - - if (input->type == GGML_TYPE_F16) { - const int64_t numel = ggml_nelements(input); - std::vector tmp((size_t) numel); - ggml_fp32_to_fp16_row(input_data, tmp.data(), numel); - ggml_backend_tensor_set(input, tmp.data(), 0, (size_t) numel * sizeof(ggml_fp16_t)); - } else { - ggml_backend_tensor_set(input, input_data, 0, (size_t) ggml_nbytes(input)); - } - sam3_graph_compute(model.backend, graph, n_threads); - - for (int i = 0; i < 4; ++i) { - output_ne[i] = output->ne[i]; - } - - const bool ok = sam3_copy_tensor_to_f32(output, output_data); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return ok; -} - -bool sam3_test_run_patch_mulmat_host_ref(const sam3_model & model, - const float * input_data, - const int64_t input_ne[4], - bool use_double_accum, - std::vector & output_data, - int64_t output_ne[4]) { - if (!input_data) { - return false; - } - - int64_t ne[4]; - sam3_normalize_ne4(input_ne, ne); +static sam3_geom_result sam3_build_geom_enc_graph( + struct ggml_context* ctx, + const sam3_model& model, + const sam3_pcs_params& params, + struct ggml_tensor* img_feats, // [D, N_img, 1] where N_img = H*H + struct ggml_tensor* img_pe) // [D, N_img, 1] sinusoidal PE +{ + const auto& ge = model.geom_enc; + const int D = model.hparams.neck_dim; // 256 + const int n_heads = 8; + const int n_boxes = (int)(params.pos_exemplars.size() + params.neg_exemplars.size()); + const int N_geo = n_boxes + 1; // +1 for CLS - const int64_t patch_k = model.vit.patch_embed_w->ne[0] * model.vit.patch_embed_w->ne[1] * model.vit.patch_embed_w->ne[2]; - const int64_t n_img = model.hparams.n_img_embd(); + // Input tensor: pre-computed geometry embeddings after final_proj + norm [D, N_geo, 1] + // The caller pre-computes: final_proj(box_embeds + CLS) → LayerNorm + auto* x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_geo, 1); + ggml_set_name(x, "geom_post_final_proj"); // also used as upload target + ggml_set_input(x); - if (ne[0] != patch_k || ne[1] != n_img || ne[2] != n_img) { - fprintf(stderr, "%s: patch_mulmat input shape mismatch [%lld,%lld,%lld,%lld]\n", - __func__, - (long long) ne[0], (long long) ne[1], (long long) ne[2], (long long) ne[3]); - return false; - } + // Transformer layers (3 layers: self-attn + cross-attn + FFN, pre-norm) + for (int i = 0; i < (int)ge.layers.size(); ++i) { + const auto& ly = ge.layers[i]; - const int64_t n_patch = ne[1] * ne[2] * ne[3]; - const int64_t n_out = model.vit.patch_embed_w->ne[3]; + // 1. Self-attention (pre-norm, pos_enc_at_attn=False) + { + auto* shortcut = x; + auto* xn = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); - std::vector input_f16((size_t) (patch_k * n_patch)); - ggml_fp32_to_fp16_row(input_data, input_f16.data(), patch_k * n_patch); + // Q = K = V = norm(x) — no positional encoding at self-attention + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0), xn), + ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0)); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]), xn), + ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float))); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]), xn), + ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float))); - std::vector weight_f16((size_t) (patch_k * n_out)); - ggml_backend_tensor_get(model.vit.patch_embed_w, weight_f16.data(), 0, weight_f16.size() * sizeof(ggml_fp16_t)); + const int64_t S = N_geo; + const int64_t HD = D / n_heads; - output_data.resize((size_t) (n_patch * n_out)); - output_ne[0] = n_patch; - output_ne[1] = n_out; - output_ne[2] = 1; - output_ne[3] = 1; + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, S, 1); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, S, 1); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, S, 1); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - for (int64_t oc = 0; oc < n_out; ++oc) { - const ggml_fp16_t * w_row = weight_f16.data() + oc * patch_k; - float * dst_row = output_data.data() + oc * n_patch; + float scale = 1.0f / sqrtf((float)HD); + auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + sa_out = ggml_reshape_3d(ctx, sa_out, D, S, 1); - for (int64_t p = 0; p < n_patch; ++p) { - const ggml_fp16_t * x_row = input_f16.data() + p * patch_k; + sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); + sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); - if (use_double_accum) { - double acc = 0.0; - for (int64_t k = 0; k < patch_k; ++k) { - acc += (double) ggml_fp16_to_fp32(x_row[k]) * (double) ggml_fp16_to_fp32(w_row[k]); - } - dst_row[p] = (float) acc; - } else { - float acc = 0.0f; - for (int64_t k = 0; k < patch_k; ++k) { - acc += ggml_fp16_to_fp32(x_row[k]) * ggml_fp16_to_fp32(w_row[k]); - } - dst_row[p] = acc; - } + x = ggml_add(ctx, shortcut, sa_out); } - } - - return true; -} - -bool sam3_test_run_vit_block_linear_host_ref(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const float * input_data, - const int64_t input_ne[4], - bool use_double_accum, - std::vector & output_data, - int64_t output_ne[4]) { - if (!input_data || block_idx < 0 || block_idx >= (int) model.vit.blocks.size()) { - return false; - } - const sam3_vit_block & blk = model.vit.blocks[(size_t) block_idx]; - - const ggml_tensor * w = nullptr; - const ggml_tensor * b = nullptr; - - switch (stage) { - case SAM3_VIT_BLOCK_STAGE_QKV_PROJ: - w = blk.qkv_w; - b = blk.qkv_b; - break; - case SAM3_VIT_BLOCK_STAGE_ATTN_PROJ: - w = blk.proj_w; - b = blk.proj_b; - break; - case SAM3_VIT_BLOCK_STAGE_MLP_FC1: - w = blk.mlp_fc1_w; - b = blk.mlp_fc1_b; - break; - case SAM3_VIT_BLOCK_STAGE_MLP_FC2: - w = blk.mlp_fc2_w; - b = blk.mlp_fc2_b; - break; - default: - fprintf(stderr, "%s: unsupported stage %d\n", __func__, (int) stage); - return false; - } - - if (!w || !b || w->type != GGML_TYPE_F16 || b->type != GGML_TYPE_F32) { - fprintf(stderr, "%s: unsupported tensor types for stage %d\n", __func__, (int) stage); - return false; - } - - int64_t ne[4]; - sam3_normalize_ne4(input_ne, ne); + // 2. Cross-attention (pre-norm, Q from x, K from img+PE, V from img) + { + auto* shortcut = x; + auto* xn = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); - const int64_t k = w->ne[0]; - const int64_t out_dim = w->ne[1]; - const int64_t n_col = ne[1] * ne[2] * ne[3]; + // Q from normalized geometry tokens (no pos) + // K from image features + PE + // V from image features + auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); + auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); - if (ne[0] != k) { - fprintf(stderr, "%s: linear input shape mismatch [%lld,%lld,%lld,%lld] for K=%lld\n", - __func__, - (long long) ne[0], (long long) ne[1], (long long) ne[2], (long long) ne[3], - (long long) k); - return false; - } + auto* k_input = ggml_add(ctx, img_feats, img_pe); // pos_enc_at_cross_attn_keys - std::vector input_f16((size_t) (k * n_col)); - ggml_fp32_to_fp16_row(input_data, input_f16.data(), k * n_col); + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, xn), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_input), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, img_feats), v_b); - std::vector weight_f16((size_t) (k * out_dim)); - ggml_backend_tensor_get(w, weight_f16.data(), 0, weight_f16.size() * sizeof(ggml_fp16_t)); + const int64_t S_q = N_geo; + const int64_t S_kv = img_feats->ne[1]; + const int64_t HD = D / n_heads; - std::vector bias_f32((size_t) out_dim); - ggml_backend_tensor_get(b, bias_f32.data(), 0, bias_f32.size() * sizeof(float)); + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, S_q, 1); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, S_kv, 1); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, S_kv, 1); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - output_data.resize((size_t) (out_dim * n_col)); - output_ne[0] = out_dim; - output_ne[1] = ne[1]; - output_ne[2] = ne[2]; - output_ne[3] = ne[3]; + float scale = 1.0f / sqrtf((float)HD); + auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + ca_out = ggml_reshape_3d(ctx, ca_out, D, S_q, 1); - for (int64_t col = 0; col < n_col; ++col) { - const ggml_fp16_t * x_col = input_f16.data() + col * k; - float * dst_col = output_data.data() + col * out_dim; + ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); + ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); - for (int64_t oc = 0; oc < out_dim; ++oc) { - const ggml_fp16_t * w_row = weight_f16.data() + oc * k; + x = ggml_add(ctx, shortcut, ca_out); + } - if (use_double_accum) { - double acc = (double) bias_f32[oc]; - for (int64_t i = 0; i < k; ++i) { - acc += (double) ggml_fp16_to_fp32(w_row[i]) * (double) ggml_fp16_to_fp32(x_col[i]); - } - dst_col[oc] = (float) acc; - } else { - float acc = bias_f32[oc]; - for (int64_t i = 0; i < k; ++i) { - acc += ggml_fp16_to_fp32(w_row[i]) * ggml_fp16_to_fp32(x_col[i]); - } - dst_col[oc] = acc; - } + // 3. FFN (pre-norm, ReLU) + { + auto* shortcut = x; + auto* xn = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); + auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, xn); + ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); + ffn = ggml_relu(ctx, ffn); + ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); + ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); + x = ggml_add(ctx, shortcut, ffn); } + sam3_name_tensorf(x, "geom_layer%d_out", i); } - return true; -} + // Final encode norm + x = sam3_layer_norm(ctx, x, ge.encode_norm_w, ge.encode_norm_b); + ggml_set_name(x, "geom_output"); + ggml_set_output(x); -static void sam3_normalize_ne4(const int64_t input_ne[4], int64_t ne[4]) { - for (int i = 0; i < 4; ++i) { - ne[i] = (input_ne && input_ne[i] > 0) ? input_ne[i] : 1; - } + sam3_geom_result result; + result.geo_feats = x; + result.n_tokens = N_geo; + return result; } -static struct ggml_tensor * sam3_new_f32_tensor_4d_from_ne(struct ggml_context * ctx, - const int64_t ne[4]) { - return ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); -} +// Pre-compute geometry encoder input on CPU: box embeddings + CLS token. +// Reads model weights from GPU, computes embeddings, returns float vector. +// Layout: [D, N_geo] row-major where N_geo = n_boxes + 1 (CLS last). +static std::vector sam3_precompute_geom_input( + const sam3_model& model, + const sam3_pcs_params& params, + const float* img_feats_data, // [D, W, H] ggml-layout backbone features (nullable if no boxes) + int W_feat, int H_feat) { + const auto& ge = model.geom_enc; + const int D = model.hparams.neck_dim; // 256 + const int roi_size = 7; + const int num_pos_feats = D / 2; // 128 -static bool sam3_copy_tensor_to_f32(struct ggml_tensor * t, std::vector & output) { - if (!t) { - return false; + // Collect all exemplar boxes + struct box_info { + float cx, cy, w, h; + int label; + }; + std::vector boxes; + for (const auto& b : params.pos_exemplars) { + // API provides XYXY in original image space — convert to normalized CxCyWH [0,1] + float cx = (b.x0 + b.x1) * 0.5f; + float cy = (b.y0 + b.y1) * 0.5f; + float bw = b.x1 - b.x0; + float bh = b.y1 - b.y0; + boxes.push_back({cx, cy, bw, bh, 0}); // label 0 = positive } - if (!ggml_is_contiguous(t)) { - fprintf(stderr, "%s: tensor '%s' is not contiguous\n", __func__, ggml_get_name(t)); - return false; + for (const auto& b : params.neg_exemplars) { + float cx = (b.x0 + b.x1) * 0.5f; + float cy = (b.y0 + b.y1) * 0.5f; + float bw = b.x1 - b.x0; + float bh = b.y1 - b.y0; + boxes.push_back({cx, cy, bw, bh, 1}); // label 1 = negative } - const int64_t numel = ggml_nelements(t); - output.resize((size_t) numel); + const int n_boxes = (int)boxes.size(); + const int N_geo = n_boxes + 1; - if (t->type == GGML_TYPE_F32) { - ggml_backend_tensor_get(t, output.data(), 0, (size_t) numel * sizeof(float)); - return true; - } + // Read needed weights from GPU + std::vector box_proj_w_data(4 * D), box_proj_b_data(D); + std::vector type_embed_data(D * 2); + std::vector cls_data(D); + std::vector box_pos_proj_w_data(258 * D), box_pos_proj_b_data(D); + std::vector box_pool_proj_w_data(7 * 7 * D * D), box_pool_proj_b_data(D); + std::vector img_pre_norm_w_data(D), img_pre_norm_b_data(D); - if (t->type == GGML_TYPE_F16) { - std::vector tmp((size_t) numel); - ggml_backend_tensor_get(t, tmp.data(), 0, (size_t) numel * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), output.data(), numel); - return true; - } + auto read_f32 = [](struct ggml_tensor* t, float* dst, size_t n) { + sam3_read_f32(t, dst, n); + }; - fprintf(stderr, "%s: unsupported tensor type %d for '%s'\n", - __func__, (int) t->type, ggml_get_name(t)); - return false; -} + read_f32(ge.box_proj_w, box_proj_w_data.data(), 4 * D); + read_f32(ge.box_proj_b, box_proj_b_data.data(), D); + read_f32(ge.type_embed, type_embed_data.data(), D * 2); + read_f32(ge.cls_token, cls_data.data(), D); + read_f32(ge.box_pos_proj_w, box_pos_proj_w_data.data(), 258 * D); + read_f32(ge.box_pos_proj_b, box_pos_proj_b_data.data(), D); -static struct ggml_tensor * sam3_build_vit_prefix_stage_from_input(struct ggml_context * ctx, - struct ggml_tensor * input, - const sam3_model & model, - sam3_vit_prefix_stage stage) { - switch (stage) { - case SAM3_VIT_PREFIX_STAGE_PATCH_IM2COL: - return ggml_im2col(ctx, - model.vit.patch_embed_w, - input, - model.vit.patch_embed_w->ne[0], - model.vit.patch_embed_w->ne[1], - 0, 0, 1, 1, true, - model.vit.patch_embed_w->type); - - case SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW: - case SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT: { - struct ggml_tensor * result = ggml_mul_mat( - ctx, - ggml_reshape_2d(ctx, input, input->ne[0], input->ne[3] * input->ne[2] * input->ne[1]), - ggml_reshape_2d(ctx, - model.vit.patch_embed_w, - model.vit.patch_embed_w->ne[0] * model.vit.patch_embed_w->ne[1] * model.vit.patch_embed_w->ne[2], - model.vit.patch_embed_w->ne[3])); - - if (stage == SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW) { - return result; - } + if (n_boxes > 0) { + read_f32(ge.box_pool_proj_w, box_pool_proj_w_data.data(), 7 * 7 * D * D); + read_f32(ge.box_pool_proj_b, box_pool_proj_b_data.data(), D); + read_f32(ge.img_pre_norm_w, img_pre_norm_w_data.data(), D); + read_f32(ge.img_pre_norm_b, img_pre_norm_b_data.data(), D); + } - result = ggml_reshape_4d(ctx, result, input->ne[1], input->ne[2], input->ne[3], model.vit.patch_embed_w->ne[3]); - return ggml_cont(ctx, ggml_permute(ctx, result, 0, 1, 3, 2)); - } + // Output: [D * N_geo] in row-major [D, N_geo] ggml order + std::vector out(D * N_geo, 0.0f); - case SAM3_VIT_PREFIX_STAGE_PATCH_EMBED: { - struct ggml_tensor * x = ggml_conv_2d_sk_p0(ctx, model.vit.patch_embed_w, input); - return ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3)); - } + // Encode each box + for (int bi = 0; bi < n_boxes; ++bi) { + const auto& box = boxes[bi]; + float embed[256] = {}; - case SAM3_VIT_PREFIX_STAGE_POS_ADD: { - struct ggml_tensor * pos_target = ggml_new_tensor_4d( - ctx, GGML_TYPE_F32, - model.hparams.vit_embed_dim, - model.hparams.n_img_embd(), - model.hparams.n_img_embd(), - 1); - struct ggml_tensor * pos_tiled = ggml_repeat(ctx, model.vit.pos_embed, pos_target); - return ggml_add(ctx, input, pos_tiled); + // 1. Direct projection: Linear(4, D) + // box_proj_w: ggml [4, D] = PyTorch [D, 4] + // out = x @ W^T + b where x=[cx,cy,w,h] + { + float coords[4] = {box.cx, box.cy, box.w, box.h}; + for (int d = 0; d < D; ++d) { + float sum = box_proj_b_data[d]; + for (int j = 0; j < 4; ++j) + sum += coords[j] * box_proj_w_data[j + d * 4]; // ggml: [4, D] stride + embed[d] = sum; + } } - case SAM3_VIT_PREFIX_STAGE_LN_PRE_NORM: - return ggml_norm(ctx, input, 1e-5f); + // 2. ROI Align + Conv2d(D, D, 7) pool projection + if (img_feats_data) { + // Apply img_pre_norm (LayerNorm) to image features before pooling + // For simplicity, we compute LayerNorm per spatial position + // Actually, LayerNorm is applied to the [D] dimension at each spatial location + // But we need the full feature map, so let's normalize in-place copy + const int N_spatial = W_feat * H_feat; + std::vector normed_feats(D * N_spatial); - case SAM3_VIT_PREFIX_STAGE_LN_PRE: { - struct ggml_tensor * x = ggml_norm(ctx, input, 1e-5f); - x = ggml_mul_inplace(ctx, x, model.vit.ln_pre_w); - x = ggml_add_inplace(ctx, x, model.vit.ln_pre_b); - return x; - } - } + for (int s = 0; s < N_spatial; ++s) { + // Compute mean and variance over D dimension + float mean = 0.0f; + for (int d = 0; d < D; ++d) + mean += img_feats_data[d + s * D]; + mean /= D; - return nullptr; -} + float var = 0.0f; + for (int d = 0; d < D; ++d) { + float diff = img_feats_data[d + s * D] - mean; + var += diff * diff; + } + var /= D; -static struct ggml_tensor * sam3_build_vit_attn_core_from_qkv(struct ggml_context * ctx, - struct ggml_tensor * qkv, - const sam3_vit_block & blk, - const sam3_hparams & hp) { - const int E = hp.vit_embed_dim; - const int NH = hp.vit_num_heads; - const int HD = hp.vit_head_dim(); + float inv_std = 1.0f / sqrtf(var + 1e-5f); + for (int d = 0; d < D; ++d) { + normed_feats[d + s * D] = + (img_feats_data[d + s * D] - mean) * inv_std * img_pre_norm_w_data[d] + img_pre_norm_b_data[d]; + } + } - const int64_t W_cur = qkv->ne[1]; - const int64_t H_cur = qkv->ne[2]; - const int64_t B_cur = qkv->ne[3]; + // Convert CxCyWH [0,1] → XYXY in feature grid coordinates + float fx0 = (box.cx - box.w * 0.5f) * (float)W_feat; + float fy0 = (box.cy - box.h * 0.5f) * (float)H_feat; + float fx1 = (box.cx + box.w * 0.5f) * (float)W_feat; + float fy1 = (box.cy + box.h * 0.5f) * (float)H_feat; - struct ggml_tensor * cur = ggml_reshape_4d(ctx, qkv, E, 3, W_cur * H_cur, B_cur); - cur = ggml_cont(ctx, ggml_permute(ctx, cur, 0, 3, 1, 2)); + // ROI Align + std::vector roi_data(D * roi_size * roi_size); + sam3_roi_align_single(normed_feats.data(), D, W_feat, H_feat, + fx0, fy0, fx1, fy1, roi_size, roi_data.data()); - struct ggml_tensor * Q = ggml_view_3d(ctx, cur, E, W_cur * H_cur, B_cur, - cur->nb[1], cur->nb[2], 0); - struct ggml_tensor * K = ggml_view_3d(ctx, cur, E, W_cur * H_cur, B_cur, - cur->nb[1], cur->nb[2], 1 * cur->nb[3]); - struct ggml_tensor * V = ggml_view_3d(ctx, cur, E, W_cur * H_cur, B_cur, - cur->nb[1], cur->nb[2], 2 * cur->nb[3]); + // Conv2d(D, D, 7): kernel [7, 7, D, D] in ggml = [D_out, D_in, kH, kW] in PyTorch + // Since roi_size = 7 = kernel_size, output is [D, 1, 1] + // This is effectively a matrix multiply: out[d_out] = sum over (d_in, kh, kw) + for (int d_out = 0; d_out < D; ++d_out) { + float sum = box_pool_proj_b_data[d_out]; + for (int kh = 0; kh < roi_size; ++kh) { + for (int kw = 0; kw < roi_size; ++kw) { + for (int d_in = 0; d_in < D; ++d_in) { + // ggml weight layout: [kW=7, kH=7, D_in=256, D_out=256] + int w_idx = kw + kh * 7 + d_in * 7 * 7 + d_out * 7 * 7 * D; + // roi_data layout: [C, W, H] = [D_in, kW, kH] + int r_idx = d_in + kw * D + kh * D * roi_size; + sum += box_pool_proj_w_data[w_idx] * roi_data[r_idx]; + } + } + } + embed[d_out] += sum; + } + } - Q = ggml_reshape_4d(ctx, Q, HD, NH, W_cur * H_cur, B_cur); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - Q = ggml_reshape_3d(ctx, Q, HD, W_cur * H_cur, NH * B_cur); + // 3. Sinusoidal positional encoding + Linear(258, D) + { + float pos_enc[258]; + sam3_sine_encode_box(pos_enc, box.cx, box.cy, box.w, box.h, + num_pos_feats, 10000); - K = ggml_reshape_4d(ctx, K, HD, NH, W_cur * H_cur, B_cur); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - K = ggml_reshape_3d(ctx, K, HD, W_cur * H_cur, NH * B_cur); + for (int d = 0; d < D; ++d) { + float sum = box_pos_proj_b_data[d]; + for (int j = 0; j < 258; ++j) + sum += pos_enc[j] * box_pos_proj_w_data[j + d * 258]; + embed[d] += sum; + } + } - V = ggml_reshape_4d(ctx, V, HD, NH, W_cur * H_cur, B_cur); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + // 4. Label embedding + { + int label = box.label; + for (int d = 0; d < D; ++d) + embed[d] += type_embed_data[d + label * D]; + } - if (blk.freqs_cis) { - Q = sam3_apply_rope(ctx, Q, blk.freqs_cis); - K = sam3_apply_rope(ctx, K, blk.freqs_cis); + // Store in output: ggml layout [D, N_geo] — embed goes at column bi + for (int d = 0; d < D; ++d) + out[d + bi * D] = embed[d]; } - Q = ggml_reshape_4d(ctx, Q, HD, W_cur * H_cur, NH, B_cur); - K = ggml_reshape_4d(ctx, K, HD, W_cur * H_cur, NH, B_cur); + // CLS token goes at position n_boxes (last) + for (int d = 0; d < D; ++d) + out[d + n_boxes * D] = cls_data[d]; - const float scale = 1.0f / sqrtf((float) HD); - struct ggml_tensor * attn_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + // Apply final_proj (Linear(D,D)) + LayerNorm on CPU + std::vector proj_w(D * D), proj_b(D); + std::vector ln_w(D), ln_b(D); + read_f32(ge.post_proj_w, proj_w.data(), D * D); + read_f32(ge.post_proj_b, proj_b.data(), D); + read_f32(ge.norm_w, ln_w.data(), D); + read_f32(ge.norm_b, ln_b.data(), D); - return ggml_cont(ctx, ggml_reshape_4d(ctx, attn_out, E, W_cur, H_cur, B_cur)); -} + std::vector projected(D * N_geo); + for (int t = 0; t < N_geo; ++t) { + // Linear: y = W @ x + b + // W is stored in PyTorch row-major [D_out, D_in] → ggml [D_in, D_out] + // proj_w[d_in + d_out * D] = W_py[d_out, d_in] + for (int d_out = 0; d_out < D; ++d_out) { + float sum = proj_b[d_out]; + for (int d_in = 0; d_in < D; ++d_in) + sum += out[d_in + t * D] * proj_w[d_in + d_out * D]; + projected[d_out + t * D] = sum; + } + } + if (n_boxes > 0) { + fprintf(stderr, "%s: %d exemplar boxes encoded (%d pos, %d neg)\n", + __func__, n_boxes, + (int)params.pos_exemplars.size(), + (int)params.neg_exemplars.size()); + } + + // LayerNorm over D dimension for each token + for (int t = 0; t < N_geo; ++t) { + float mean = 0.0f; + for (int d = 0; d < D; ++d) + mean += projected[d + t * D]; + mean /= D; -static struct ggml_tensor * sam3_build_vit_block_stage_from_input(struct ggml_context * ctx, - struct ggml_tensor * input, - const sam3_vit_block & blk, - const sam3_hparams & hp, - sam3_vit_block_stage stage) { - switch (stage) { - case SAM3_VIT_BLOCK_STAGE_NORM1: - return sam3_layer_norm(ctx, input, blk.norm1_w, blk.norm1_b); + float var = 0.0f; + for (int d = 0; d < D; ++d) { + float diff = projected[d + t * D] - mean; + var += diff * diff; + } + var /= D; - case SAM3_VIT_BLOCK_STAGE_WINDOW_PART: - return ggml_win_part(ctx, input, hp.vit_window_size); + float inv_std = 1.0f / sqrtf(var + 1e-5f); + for (int d = 0; d < D; ++d) { + float normalized = (projected[d + t * D] - mean) * inv_std; + out[d + t * D] = normalized * ln_w[d] + ln_b[d]; + } + } + return out; +} - case SAM3_VIT_BLOCK_STAGE_QKV_PROJ: - return ggml_add(ctx, ggml_mul_mat(ctx, blk.qkv_w, input), blk.qkv_b); +/***************************************************************************** +** Fusion encoder — graph building (6 layers) +*****************************************************************************/ - case SAM3_VIT_BLOCK_STAGE_ATTN_CORE: - return sam3_build_vit_attn_core_from_qkv(ctx, input, blk, hp); +// Single fusion encoder layer. +// x: [D, N, B] image features (N=5184), prompt: [D, T, B] text/exemplar tokens, pos: [D, N, B] +// Returns: updated x [D, N, B] +static struct ggml_tensor* sam3_fenc_layer_forward( + struct ggml_context* ctx, + const sam3_fenc_layer& ly, + struct ggml_tensor* x, + struct ggml_tensor* prompt, + struct ggml_tensor* pos, + struct ggml_tensor* prompt_attn_bias, + int n_heads) { + // Self-attention: Q/K get positional encoding, V does not + { + auto* shortcut = x; + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); + auto* q_in = ggml_add(ctx, x_norm, pos); + auto* k_in = ggml_add(ctx, x_norm, pos); - case SAM3_VIT_BLOCK_STAGE_ATTN_PROJ: - return ggml_add(ctx, ggml_mul_mat(ctx, blk.proj_w, input), blk.proj_b); + const int64_t D = x->ne[0]; - case SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART: - return ggml_win_unpart(ctx, input, hp.n_img_embd(), hp.n_img_embd(), hp.vit_window_size); + auto* q_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]); - case SAM3_VIT_BLOCK_STAGE_NORM2: - return sam3_layer_norm(ctx, input, blk.norm2_w, blk.norm2_b); + auto* q_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float)); - case SAM3_VIT_BLOCK_STAGE_MLP_FC1: - return ggml_add(ctx, ggml_mul_mat(ctx, blk.mlp_fc1_w, input), blk.mlp_fc1_b); + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_in), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, x_norm), v_b); - case SAM3_VIT_BLOCK_STAGE_MLP_GELU: - return ggml_gelu_erf(ctx, input); + const int64_t N = x->ne[1]; + const int64_t B = x->ne[2]; + const int64_t HD = D / n_heads; - case SAM3_VIT_BLOCK_STAGE_MLP_FC2: - return ggml_add(ctx, ggml_mul_mat(ctx, blk.mlp_fc2_w, input), blk.mlp_fc2_b); + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, N, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - case SAM3_VIT_BLOCK_STAGE_MLP: { - struct ggml_tensor * x = ggml_mul_mat(ctx, blk.mlp_fc1_w, input); - x = ggml_add(ctx, x, blk.mlp_fc1_b); - x = ggml_gelu_erf(ctx, x); - x = ggml_mul_mat(ctx, blk.mlp_fc2_w, x); - x = ggml_add(ctx, x, blk.mlp_fc2_b); - return x; - } - } + float scale = 1.0f / sqrtf((float)HD); + auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + sa_out = ggml_reshape_3d(ctx, sa_out, D, N, B); - return nullptr; -} + sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); + sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); -bool sam3_test_run_vit_block_stage(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const float * input_data, - const int64_t input_ne[4], - std::vector & output_data, - int64_t output_ne[4], - int n_threads) { - if (!input_data) { - fprintf(stderr, "%s: input_data is null\n", __func__); - return false; - } - if (block_idx < 0 || block_idx >= (int) model.vit.blocks.size()) { - fprintf(stderr, "%s: invalid block_idx=%d\n", __func__, block_idx); - return false; + x = ggml_add(ctx, shortcut, sa_out); } - const auto & hp = model.hparams; - const auto & blk = model.vit.blocks[block_idx]; + // Cross-attention: Q from image features, K/V from prompt tokens. + // ca_q_w stores fused [D, 3*D] weights split as Q-proj, K-proj, V-proj. + { + auto* shortcut = x; + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); + const int64_t D = x->ne[0]; - int64_t ne[4]; - sam3_normalize_ne4(input_ne, ne); + auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); - switch (stage) { - case SAM3_VIT_BLOCK_STAGE_WINDOW_PART: - case SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART: - if (ne[0] != hp.vit_embed_dim) { - fprintf(stderr, "%s: window stage input ne0=%lld expected %d\n", - __func__, (long long) ne[0], hp.vit_embed_dim); - return false; - } - break; - case SAM3_VIT_BLOCK_STAGE_QKV_PROJ: - if (ne[0] != hp.vit_embed_dim) { - fprintf(stderr, "%s: qkv input ne0=%lld expected %d\n", - __func__, (long long) ne[0], hp.vit_embed_dim); - return false; - } - break; - case SAM3_VIT_BLOCK_STAGE_ATTN_CORE: - if (ne[0] != 3 * hp.vit_embed_dim) { - fprintf(stderr, "%s: attn input ne0=%lld expected %d\n", - __func__, (long long) ne[0], 3 * hp.vit_embed_dim); - return false; - } - break; - case SAM3_VIT_BLOCK_STAGE_MLP_FC1: - if (ne[0] != hp.vit_embed_dim) { - fprintf(stderr, "%s: mlp_fc1 input ne0=%lld expected %d\n", - __func__, (long long) ne[0], hp.vit_embed_dim); - return false; - } - break; - case SAM3_VIT_BLOCK_STAGE_MLP_GELU: - case SAM3_VIT_BLOCK_STAGE_MLP_FC2: - if (ne[0] != hp.vit_mlp_dim) { - fprintf(stderr, "%s: mlp stage input ne0=%lld expected %d\n", - __func__, (long long) ne[0], hp.vit_mlp_dim); - return false; - } - break; - default: - if (ne[0] != hp.vit_embed_dim) { - fprintf(stderr, "%s: stage input ne0=%lld expected %d\n", - __func__, (long long) ne[0], hp.vit_embed_dim); - return false; - } - break; - } + auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); - const size_t ctx_size = ggml_tensor_overhead() * 256 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, x_norm), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, prompt), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, prompt), v_b); - struct ggml_context * ctx = ggml_init(params); - if (!ctx) { - fprintf(stderr, "%s: failed to create ggml context\n", __func__); - return false; - } + const int64_t N_q = x->ne[1]; + const int64_t N_kv = prompt->ne[1]; + const int64_t B = x->ne[2]; + const int64_t HD = D / n_heads; - struct ggml_tensor * input = sam3_new_f32_tensor_4d_from_ne(ctx, ne); - ggml_set_name(input, "vit_block_stage_input"); - ggml_set_input(input); + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - struct ggml_tensor * output = sam3_build_vit_block_stage_from_input(ctx, input, blk, hp, stage); - if (!output) { - fprintf(stderr, "%s: failed to build stage %d\n", __func__, (int) stage); - ggml_free(ctx); - return false; - } - ggml_set_name(output, "vit_block_stage_output"); - ggml_set_output(output); + auto* ca_mask = sam3_expand_token_attn_bias(ctx, prompt_attn_bias, N_q, n_heads, B); + float scale = 1.0f / sqrtf((float)HD); + auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, ca_mask, scale, 0.0f, 0.0f); + ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); - struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, 1024, false); - ggml_build_forward_expand(graph, output); + ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); + ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); - auto * galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph for stage %d\n", __func__, (int) stage); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return false; + x = ggml_add(ctx, shortcut, ca_out); } - const size_t input_bytes = (size_t) ne[0] * (size_t) ne[1] * (size_t) ne[2] * (size_t) ne[3] * sizeof(float); - ggml_backend_tensor_set(input, input_data, 0, input_bytes); - sam3_graph_compute(model.backend, graph, n_threads); + { + auto* shortcut = x; + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); - for (int i = 0; i < 4; ++i) { - output_ne[i] = output->ne[i]; + auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, x_norm); + ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); + ffn = ggml_relu(ctx, ffn); + ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); + ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); + + x = ggml_add(ctx, shortcut, ffn); } - const bool ok = sam3_copy_tensor_to_f32(output, output_data); - ggml_gallocr_free(galloc); - ggml_free(ctx); - return ok; + return x; } -// Test-only: encode from pre-preprocessed float data (bypasses C++ resize/normalize). -// chw_data is in [C, H, W] layout, already normalized, with C=3, H=W=img_size. -bool sam3_encode_image_from_preprocessed(sam3_state& state, - const sam3_model& model, - const float* chw_data, - int img_size) { - auto t_start = std::chrono::high_resolution_clock::now(); +// Build full fusion encoder graph (6 layers). +// image_feats: [D, N, B] where N=5184 (72*72), D=256 +// prompt_tokens: [D, T, B] text/exemplar features +// pos_enc: [D, N, B] sinusoidal positional encoding for image features +// Returns: conditioned_features [D, N, B] +static struct ggml_tensor* sam3_build_fenc_graph( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* image_feats, + struct ggml_tensor* prompt_tokens, + struct ggml_tensor* pos_enc, + struct ggml_tensor* prompt_attn_bias = nullptr) { const auto& hp = model.hparams; + auto* x = image_feats; - if (img_size != hp.img_size) { - fprintf(stderr, "%s: img_size mismatch: got %d, expected %d\n", - __func__, img_size, hp.img_size); - return false; - } - - fprintf(stderr, "%s: encoding from preprocessed %dx%d\n", __func__, img_size, img_size); - - // EdgeTAM dispatch: build RepViT graph directly from preprocessed data - if (hp.is_edgetam()) { - state.orig_width = img_size; - state.orig_height = img_size; - - const size_t buf_size = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = {buf_size, nullptr, true}; - auto* ctx0 = ggml_init(gparams); - if (!ctx0) return false; + for (int i = 0; i < hp.fenc_layers; ++i) { + x = sam3_fenc_layer_forward(ctx, model.fenc.layers[i], x, prompt_tokens, + pos_enc, prompt_attn_bias, hp.fenc_heads); + sam3_name_tensorf(x, "fenc_layer%d_out", i); + } - auto* inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img_size, img_size, 3, 1); - ggml_set_name(inp, "input_image"); - ggml_set_input(inp); + return x; +} - struct ggml_tensor* stage_outs[4] = {}; - edgetam_build_repvit_graph(ctx0, inp, model, stage_outs); +/***************************************************************************** +** DETR decoder — graph building (6 layers) +*****************************************************************************/ - struct ggml_tensor* fpn_outs[4] = {}; - edgetam_build_fpn_neck_graph(ctx0, stage_outs, model, fpn_outs); +// inverse_sigmoid: log(x / (1 - x)), clamped to avoid inf +// Python reference uses eps=1e-3: x1 = x.clamp(min=eps), x2 = (1-x).clamp(min=eps) +static struct ggml_tensor* sam3_inverse_sigmoid(struct ggml_context* ctx, struct ggml_tensor* x) { + // clamp x to [1e-3, 1-1e-3] to match Python eps=1e-3 + x = ggml_clamp(ctx, x, 1e-3f, 1.0f - 1e-3f); + // log(x / (1 - x)) = log(x) - log(1 - x) + auto* log_x = ggml_log(ctx, x); + // Compute (1 - x) as (-1)*x + 1. We use ggml_scale_bias which takes float + // scalars (no tensor allocation needed, safe in no_alloc contexts). + auto* one_minus = ggml_scale_bias(ctx, x, -1.0f, 1.0f); + auto* log_1mx = ggml_log(ctx, one_minus); + return ggml_sub(ctx, log_x, log_1mx); +} - int n_fpn = 4 - hp.scalp; - for (int i = 0; i < n_fpn; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fpn_out_%d", i); - ggml_set_name(fpn_outs[i], name); - ggml_set_output(fpn_outs[i]); - } +// Box refinement MLP (3 layers: D→D→D→4 with ReLU) +static struct ggml_tensor* sam3_bbox_mlp(struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* w[3], + struct ggml_tensor* b[3]) { + for (int j = 0; j < 3; ++j) { + x = ggml_mul_mat(ctx, w[j], x); + x = ggml_add(ctx, x, b[j]); + if (j < 2) x = ggml_relu(ctx, x); + } + return x; +} - auto* graph = ggml_new_graph_custom(ctx0, 32768, false); - for (int i = 0; i < n_fpn; ++i) - ggml_build_forward_expand(graph, fpn_outs[i]); +// Build sinusoidal positional embedding for 4D reference points in the ggml graph. +// ref_boxes: [4, NQ, B] — (cx, cy, w, h) after sigmoid, B=1 +// sine_dim_t: [1, 64] — pre-computed angle multipliers (2π / 10000^(2i/128)) +// Returns: [512, NQ, B] sinusoidal embedding matching Python gen_sineembed_for_position +static struct ggml_tensor* sam3_build_sine_pos_embed_4d( + struct ggml_context* ctx, + struct ggml_tensor* ref_boxes, // [4, NQ, B] + struct ggml_tensor* sine_dim_t) { // [1, 64] + const int64_t NQ = ref_boxes->ne[1]; - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + // Python output order: [cy, cx, w, h] → coord indices from boxes [cx(0),cy(1),w(2),h(3)] + const int coord_order[4] = {1, 0, 2, 3}; - ggml_backend_tensor_set(inp, chw_data, 0, 3 * img_size * img_size * sizeof(float)); + struct ggml_tensor* coord_embeds[4]; - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + for (int c = 0; c < 4; ++c) { + int ci = coord_order[c]; + // Extract one coordinate: view into ref_boxes [4, NQ, 1] at element ci + auto* coord = ggml_view_2d(ctx, ref_boxes, 1, NQ, + ref_boxes->nb[1], ci * sizeof(float)); // [1, NQ] - // Copy to state (same pattern as edgetam_encode_image) - if (state.buffer) { ggml_backend_buffer_free(state.buffer); state.buffer = nullptr; } - if (state.pe_buf) { ggml_backend_buffer_free(state.pe_buf); state.pe_buf = nullptr; } - if (state.pe_ctx) { ggml_free(state.pe_ctx); state.pe_ctx = nullptr; } - if (state.ctx) { ggml_free(state.ctx); state.ctx = nullptr; } + // Outer product: angles[i, q] = dim_t[i] * coord[q] + // ggml_mul_mat(A=[1,64], B=[1,NQ]) = A^T @ B = [64,1]@[1,NQ] = [64, NQ] + auto* angles = ggml_mul_mat(ctx, sine_dim_t, coord); // [64, NQ] - size_t state_ctx_size = ggml_tensor_overhead() * 32; - struct ggml_init_params sparams = {state_ctx_size, nullptr, true}; - state.ctx = ggml_init(sparams); + auto* sin_vals = ggml_sin(ctx, angles); // [64, NQ] + auto* cos_vals = ggml_cos(ctx, angles); // [64, NQ] - for (int i = 0; i < n_fpn; ++i) { - auto* src = fpn_outs[i]; - state.neck_trk[i] = ggml_new_tensor_4d(state.ctx, GGML_TYPE_F32, - src->ne[0], src->ne[1], src->ne[2], src->ne[3]); - char nm[64]; - snprintf(nm, sizeof(nm), "neck_trk_%d", i); - ggml_set_name(state.neck_trk[i], nm); - } - for (int i = n_fpn; i < 4; ++i) state.neck_trk[i] = nullptr; + // Interleave: [sin_0, cos_0, sin_1, cos_1, ...] + auto* sin_r = ggml_reshape_3d(ctx, sin_vals, 1, 64, NQ); + auto* cos_r = ggml_reshape_3d(ctx, cos_vals, 1, 64, NQ); + auto* interleaved = ggml_concat(ctx, sin_r, cos_r, 0); // [2, 64, NQ] + coord_embeds[c] = ggml_reshape_2d(ctx, interleaved, 128, NQ); + } - state.buffer = ggml_backend_alloc_ctx_tensors(state.ctx, model.backend); - for (int i = 0; i < n_fpn; ++i) { - int64_t nb = ggml_nbytes(state.neck_trk[i]); - std::vector buf(nb); - ggml_backend_tensor_get(fpn_outs[i], buf.data(), 0, nb); - ggml_backend_tensor_set(state.neck_trk[i], buf.data(), 0, nb); - } + // Concatenate all 4 coordinates → [512, NQ] + auto* embed = ggml_concat(ctx, coord_embeds[0], coord_embeds[1], 0); // [256, NQ] + embed = ggml_concat(ctx, embed, coord_embeds[2], 0); // [384, NQ] + embed = ggml_concat(ctx, embed, coord_embeds[3], 0); // [512, NQ] - size_t pe_ctx_size = ggml_tensor_overhead() * 16; - struct ggml_init_params pe_params = {pe_ctx_size, nullptr, true}; - state.pe_ctx = ggml_init(pe_params); - for (int i = 0; i < n_fpn; ++i) { - int H = (int)state.neck_trk[i]->ne[2]; - int W = (int)state.neck_trk[i]->ne[1]; - state.neck_trk_pe[i] = ggml_new_tensor_4d(state.pe_ctx, GGML_TYPE_F32, - hp.neck_dim, W, H, 1); - } - state.pe_buf = ggml_backend_alloc_ctx_tensors(state.pe_ctx, model.backend); - for (int i = 0; i < n_fpn; ++i) { - int H = (int)state.neck_trk[i]->ne[2]; - int W = (int)state.neck_trk[i]->ne[1]; - auto pe = sam3_sinusoidal_pe_2d(H, W, hp.neck_dim); - ggml_backend_tensor_set(state.neck_trk_pe[i], pe.data(), 0, pe.size() * sizeof(float)); - } + return embed; +} - ggml_gallocr_free(galloc); - ggml_free(ctx0); +// Build query positional encoding from reference boxes via sine embed + ref_point_head MLP. +// ref_boxes: [4, NQ, 1] — after sigmoid +// sine_dim_t: [1, 64] +// Returns: [D, NQ+1, 1] (zeros for presence token at index 0, MLP output for object queries) +static struct ggml_tensor* sam3_build_query_pos( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* ref_boxes, // [4, NQ, 1] + struct ggml_tensor* sine_dim_t, // [1, 64] + int layer_idx = -1) { + const auto& tensors = model.tensors; + const int64_t NQ = ref_boxes->ne[1]; + const int D = model.hparams.neck_dim; // 256 - auto t_end = std::chrono::high_resolution_clock::now(); - auto ms = std::chrono::duration_cast(t_end - t_start).count(); - fprintf(stderr, "%s: EdgeTAM preprocessed encode done in %lld ms\n", __func__, ms); - return true; + // 1. Sine positional embedding: [512, NQ] + auto* sine_embed = sam3_build_sine_pos_embed_4d(ctx, ref_boxes, sine_dim_t); + if (layer_idx == 0) { + ggml_set_name(sine_embed, "ddec_query_sine_0"); } - // SAM2 dispatch: build a fake sam3_image and use the SAM2 encoder - if (hp.is_sam2()) { - state.orig_width = img_size; - state.orig_height = img_size; + // 2. ref_point_head MLP: 512 → 256 → 256 + // Layer 0: relu(W0 @ sine_embed + b0) + auto* h = ggml_mul_mat(ctx, tensors.at("ddec.ref_point_head.layers.0.weight"), sine_embed); + h = ggml_add(ctx, h, tensors.at("ddec.ref_point_head.layers.0.bias")); + h = ggml_relu(ctx, h); + // Layer 1: W1 @ h + b1 (no activation) + auto* qpos_obj = ggml_mul_mat(ctx, tensors.at("ddec.ref_point_head.layers.1.weight"), h); + qpos_obj = ggml_add(ctx, qpos_obj, tensors.at("ddec.ref_point_head.layers.1.bias")); + // qpos_obj: [D, NQ] + if (layer_idx == 0) { + ggml_set_name(qpos_obj, "ddec_query_pos_0"); + } - // sam2_encode_image_hiera expects an image struct, but we have raw CHW data. - // We need to bypass preprocessing and inject the data directly into the graph. - // Build the Hiera graph, set the input from chw_data (already CHW normalized). + // 3. Reshape to 3D and prepend zeros for presence token + qpos_obj = ggml_reshape_3d(ctx, qpos_obj, D, NQ, 1); + auto* qpos_pres = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, 1, 1); + ggml_set_name(qpos_pres, "ddec_query_pos_pres"); + ggml_set_input(qpos_pres); // zeros — set by caller - const size_t buf_size = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = {buf_size, nullptr, true}; - auto* ctx0 = ggml_init(gparams); - if (!ctx0) return false; + return ggml_concat(ctx, qpos_pres, qpos_obj, 1); // [D, NQ+1, 1] +} - auto* inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img_size, img_size, 3, 1); - ggml_set_name(inp, "input_image"); - ggml_set_input(inp); +// Build box-relative positional bias for DETR cross-attention. +// ref_boxes: [4, N_q, B] — (cx, cy, w, h) in [0,1] +// rpb_coords: [feat_hw] — normalized coords [0/H, 1/H, ..., (H-1)/H] (input tensor) +// Returns: bias tensor [N_kv, N_q+1, n_heads, B] for ggml_flash_attn_ext mask +// +// Python _get_rpb_matrix with boxRPB="log": +// 1. boxes → xyxy +// 2. deltas_x[q,w,:2] = [coord_w - x0, coord_w - x1] +// 3. deltas_y[q,h,:2] = [coord_h - y0, coord_h - y1] +// 4. log transform: sign(d*8) * log2(|d*8|+1) / log2(8) +// 5. MLP: [2] → [256] → [n_heads] +// 6. outer sum: B[h,w] = delta_y[h] + delta_x[w] +static struct ggml_tensor* sam3_compute_box_rpb( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* ref_boxes, // [4, N_q, B] + struct ggml_tensor* rpb_coords, // [feat_hw] — pre-filled grid coordinates + int feat_hw, + int layer_idx = -1) { + const int64_t NQ = ref_boxes->ne[1]; + const int NH = model.hparams.ddec_heads; // 8 + const int W = feat_hw; + const int H = feat_hw; + const auto& tensors = model.tensors; - struct ggml_tensor* stage_outs[4] = {}; - sam2_build_hiera_graph(ctx0, inp, model, stage_outs); + // ── 1. Convert cxcywh → xyxy ───────────────────────────────────────── + // ggml_view_2d on strided data is non-contiguous — ggml_scale requires contiguous. + // Use ggml_cont to make each coordinate slice contiguous. + auto* cx = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 0)); + auto* cy = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 1 * sizeof(float))); + auto* bw = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 2 * sizeof(float))); + auto* bh = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 3 * sizeof(float))); + // x0 = cx - w/2, x1 = cx + w/2 + auto* half_w = ggml_scale(ctx, bw, 0.5f); + auto* half_h = ggml_scale(ctx, bh, 0.5f); + auto* x0 = ggml_sub(ctx, cx, half_w); // [1, NQ] + auto* x1 = ggml_add(ctx, cx, half_w); + auto* y0 = ggml_sub(ctx, cy, half_h); + auto* y1 = ggml_add(ctx, cy, half_h); - struct ggml_tensor* fpn_outs[4] = {}; - sam2_build_fpn_neck_graph(ctx0, stage_outs, model, fpn_outs); + // ── 2. Compute deltas via outer subtract ────────────────────────────── + // coords: [W] → reshape to [W, 1] for outer subtract + auto* cw = ggml_reshape_2d(ctx, rpb_coords, W, 1); // [W, 1] - int n_fpn = 4 - hp.scalp; - for (int i = 0; i < n_fpn; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fpn_out_%d", i); - ggml_set_name(fpn_outs[i], name); - ggml_set_output(fpn_outs[i]); - } + // Outer subtract: delta[w, q] = coord[w] - edge[q] + // Use ggml_mul_mat trick: not applicable for subtraction. + // Instead: repeat coords to [W, NQ], repeat edge to [W, NQ], subtract. + auto* shape_wn = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, W, NQ); - auto* graph = ggml_new_graph_custom(ctx0, 32768, false); - for (int i = 0; i < n_fpn; ++i) - ggml_build_forward_expand(graph, fpn_outs[i]); + auto* cw_rep = ggml_repeat(ctx, cw, shape_wn); // [W, NQ] (each column = coords) + auto* x0_t = ggml_cont(ctx, ggml_transpose(ctx, x0)); // [NQ, 1] + auto* x0_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, x0_t, 1, NQ), shape_wn); + auto* x1_t = ggml_cont(ctx, ggml_transpose(ctx, x1)); + auto* x1_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, x1_t, 1, NQ), shape_wn); + auto* y0_t = ggml_cont(ctx, ggml_transpose(ctx, y0)); + auto* y0_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, y0_t, 1, NQ), shape_wn); + auto* y1_t = ggml_cont(ctx, ggml_transpose(ctx, y1)); + auto* y1_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, y1_t, 1, NQ), shape_wn); - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + auto* dx0 = ggml_sub(ctx, cw_rep, x0_rep); // [W, NQ] + auto* dx1 = ggml_sub(ctx, cw_rep, x1_rep); + auto* dy0 = ggml_sub(ctx, cw_rep, y0_rep); // reusing coords for H (H==W) + auto* dy1 = ggml_sub(ctx, cw_rep, y1_rep); - // Set input image — CHW data needs to go into ggml's [W, H, C, B] layout - // The input tensor is [img_size, img_size, 3, 1] in ggml = [W, H, C, B] - // CHW data: element (c, h, w) at c*H*W + h*W + w - // ggml data: element at w + h*W_stride... but ggml_conv_2d expects [W, H, C, B] - // which means ne[0]=W, ne[1]=H, ne[2]=C, ne[3]=B - // flat index: w + h*W + c*W*H + b*W*H*C - // CHW: c*H*W + h*W + w → same flat index! So we can just copy directly. - ggml_backend_tensor_set(inp, chw_data, 0, 3 * img_size * img_size * sizeof(float)); + // Stack into [2, W, NQ]: reshape each to [1, W, NQ], concat dim 0 + auto* dx0_r = ggml_reshape_3d(ctx, dx0, 1, W, NQ); + auto* dx1_r = ggml_reshape_3d(ctx, dx1, 1, W, NQ); + auto* deltas_x = ggml_concat(ctx, dx0_r, dx1_r, 0); // [2, W, NQ] + auto* dy0_r = ggml_reshape_3d(ctx, dy0, 1, H, NQ); + auto* dy1_r = ggml_reshape_3d(ctx, dy1, 1, H, NQ); + auto* deltas_y = ggml_concat(ctx, dy0_r, dy1_r, 0); // [2, H, NQ] - // Set positional embedding - { - int pe_H = img_size / 4, pe_W = img_size / 4; - auto pe_data = sam2_compute_pos_embed(model, pe_H, pe_W); - auto* pe_tensor = ggml_graph_get_tensor(graph, "hiera_pos_embed"); - ggml_backend_tensor_set(pe_tensor, pe_data.data(), 0, pe_data.size() * sizeof(float)); - - // Dump PE if requested - const char* dump_dir = getenv("SAM2_DUMP_DIR"); - if (dump_dir) { - char path[512]; - snprintf(path, sizeof(path), "%s/cpp_pos_embed.bin", dump_dir); - FILE* f = fopen(path, "wb"); - if (f) { - fwrite(pe_data.data(), sizeof(float), pe_data.size(), f); - fclose(f); - } - snprintf(path, sizeof(path), "%s/cpp_pos_embed.shape", dump_dir); - f = fopen(path, "w"); - if (f) { - fprintf(f, "%d,%d,%d,%d", hp.hiera_embed_dim, pe_W, pe_H, 1); - fclose(f); - } - fprintf(stderr, " [DUMP] cpp_pos_embed: [%d,%d,%d,1]\n", - hp.hiera_embed_dim, pe_W, pe_H); - } - } + // ── 3. Log transform: sign(d*8) * log2(|d*8|+1) / log2(8) ──────────── + const float scale8 = 8.0f; + const float inv_log2_8 = 1.0f / log2f(8.0f); // = 1/3 - if (ggml_backend_is_cpu(model.backend)) - ggml_backend_cpu_set_n_threads(model.backend, state.n_threads); - if (ggml_backend_graph_compute(model.backend, graph) != GGML_STATUS_SUCCESS) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + auto rpb_log = [&](struct ggml_tensor* d) -> struct ggml_tensor* { + auto* d8 = ggml_scale(ctx, d, scale8); + auto* sign_d = ggml_sgn(ctx, d8); + auto* abs_d = ggml_abs(ctx, d8); + auto* log_val = ggml_log(ctx, ggml_scale_bias(ctx, abs_d, 1.0f, 1.0f)); + // log2(x) = ln(x) / ln(2) + log_val = ggml_scale(ctx, log_val, 1.0f / logf(2.0f)); + return ggml_mul(ctx, sign_d, ggml_scale(ctx, log_val, inv_log2_8)); + }; - // Dump debug tensors if SAM2_DUMP_DIR is set - { - const char* dump_dir = getenv("SAM2_DUMP_DIR"); - if (dump_dir) { - const char* dbg_names[] = { - "dbg_patch_embed", "dbg_after_pe", - "dbg_blk0_norm1", "dbg_blk0_attn_out", "dbg_blk0_res1", - "dbg_block_0", "dbg_block_1", "dbg_block_2", - "dbg_block_5", "dbg_block_21", - "dbg_fpn_lateral_0", "dbg_fpn_lateral_1", - "dbg_fpn_lateral_2", "dbg_fpn_lateral_3", - }; - for (const char* dn : dbg_names) { - auto* t = ggml_graph_get_tensor(graph, dn); - if (!t) continue; - int64_t nb = ggml_nbytes(t); - std::vector buf(nb); - ggml_backend_tensor_get(t, buf.data(), 0, nb); - char path[512]; - snprintf(path, sizeof(path), "%s/%s.bin", dump_dir, dn); - FILE* f = fopen(path, "wb"); - if (f) { fwrite(buf.data(), 1, nb, f); fclose(f); } - snprintf(path, sizeof(path), "%s/%s.shape", dump_dir, dn); - f = fopen(path, "w"); - if (f) { - fprintf(f, "%lld,%lld,%lld,%lld", - (long long)t->ne[0], (long long)t->ne[1], - (long long)t->ne[2], (long long)t->ne[3]); - fclose(f); - } - fprintf(stderr, " [DUMP] %s: [%lld,%lld,%lld,%lld]\n", dn, - (long long)t->ne[0], (long long)t->ne[1], - (long long)t->ne[2], (long long)t->ne[3]); - } - } - } + deltas_x = rpb_log(deltas_x); // [2, W, NQ] + deltas_y = rpb_log(deltas_y); // [2, H, NQ] - // Copy to state (same as sam2_encode_image_hiera) - if (state.buffer) { ggml_backend_buffer_free(state.buffer); state.buffer = nullptr; } - if (state.pe_buf) { ggml_backend_buffer_free(state.pe_buf); state.pe_buf = nullptr; } - if (state.pe_ctx) { ggml_free(state.pe_ctx); state.pe_ctx = nullptr; } - if (state.ctx) { ggml_free(state.ctx); state.ctx = nullptr; } + // ── 4. MLP: [2, W*NQ] → [NH, W*NQ] ─────────────────────────────────── + // boxRPB_embed_x: MLP(2, 256, 8, 2) = Linear(2→256)+ReLU, Linear(256→8) + // Reshape to [2, W*NQ] so matmul treats each (w, q) pair as a sample + auto rpb_mlp = [&](struct ggml_tensor* d, const char* axis) -> struct ggml_tensor* { + int64_t spatial = d->ne[1]; + int64_t nq = d->ne[2]; + auto* flat = ggml_reshape_2d(ctx, d, 2, spatial * nq); // [2, W*NQ] + auto wn0 = std::string("ddec.boxRPB_embed_") + axis + ".layers.0.weight"; + auto bn0 = std::string("ddec.boxRPB_embed_") + axis + ".layers.0.bias"; + auto wn1 = std::string("ddec.boxRPB_embed_") + axis + ".layers.1.weight"; + auto bn1 = std::string("ddec.boxRPB_embed_") + axis + ".layers.1.bias"; + flat = ggml_mul_mat(ctx, tensors.at(wn0), flat); + flat = ggml_add(ctx, flat, tensors.at(bn0)); + flat = ggml_relu(ctx, flat); + flat = ggml_mul_mat(ctx, tensors.at(wn1), flat); + flat = ggml_add(ctx, flat, tensors.at(bn1)); + // flat: [NH, W*NQ] → reshape to [NH, spatial, NQ] + return ggml_reshape_3d(ctx, flat, NH, spatial, nq); + }; - size_t state_ctx_size = ggml_tensor_overhead() * 32; - struct ggml_init_params sparams = {state_ctx_size, nullptr, true}; - state.ctx = ggml_init(sparams); + auto* rpb_x = rpb_mlp(deltas_x, "x"); // [NH, W, NQ] + auto* rpb_y = rpb_mlp(deltas_y, "y"); // [NH, H, NQ] - for (int i = 0; i < n_fpn; ++i) { - auto* src = fpn_outs[i]; - state.neck_trk[i] = ggml_new_tensor_4d(state.ctx, GGML_TYPE_F32, - src->ne[0], src->ne[1], src->ne[2], src->ne[3]); - } - for (int i = n_fpn; i < 4; ++i) state.neck_trk[i] = nullptr; + // ── 5. Outer sum: B[nh, w, h, q] = rpb_y[nh, h, q] + rpb_x[nh, w, q] ─ + // Reshape for broadcasting: + // rpb_y → [NH, 1, H, NQ] + // rpb_x → [NH, W, 1, NQ] + // + // Keep W in ne[1] so reshaping [NH, W, H, NQ] → [NH, H*W, NQ, 1] + // preserves Python's flatten(H, W) order where W is the fast spatial axis. + auto* rpb_y_4d = ggml_reshape_4d(ctx, rpb_y, NH, 1, H, NQ); + auto* rpb_x_4d = ggml_reshape_4d(ctx, rpb_x, NH, W, 1, NQ); - state.buffer = ggml_backend_alloc_ctx_tensors(state.ctx, model.backend); - for (int i = 0; i < n_fpn; ++i) { - int64_t n_bytes = ggml_nbytes(state.neck_trk[i]); - std::vector buf(n_bytes); - ggml_backend_tensor_get(fpn_outs[i], buf.data(), 0, n_bytes); - ggml_backend_tensor_set(state.neck_trk[i], buf.data(), 0, n_bytes); - } + // ggml_add broadcasts: where one dim is 1, the other is used + auto* rpb_hw = ggml_repeat(ctx, rpb_y_4d, + ggml_new_tensor_4d(ctx, GGML_TYPE_F32, NH, W, H, NQ)); + auto* rpb_hw_x = ggml_repeat(ctx, rpb_x_4d, + ggml_new_tensor_4d(ctx, GGML_TYPE_F32, NH, W, H, NQ)); + auto* rpb = ggml_add(ctx, rpb_hw, rpb_hw_x); // [NH, W, H, NQ] - // Compute sinusoidal PE - size_t pe_ctx_size = ggml_tensor_overhead() * 16; - struct ggml_init_params pe_params = {pe_ctx_size, nullptr, true}; - state.pe_ctx = ggml_init(pe_params); - for (int i = 0; i < n_fpn; ++i) { - int H = (int)state.neck_trk[i]->ne[2]; - int W = (int)state.neck_trk[i]->ne[1]; - state.neck_trk_pe[i] = ggml_new_tensor_4d(state.pe_ctx, GGML_TYPE_F32, hp.neck_dim, W, H, 1); - } - state.pe_buf = ggml_backend_alloc_ctx_tensors(state.pe_ctx, model.backend); - for (int i = 0; i < n_fpn; ++i) { - int H = (int)state.neck_trk[i]->ne[2]; - int W = (int)state.neck_trk[i]->ne[1]; - auto pe = sam3_sinusoidal_pe_2d(H, W, hp.neck_dim); - ggml_backend_tensor_set(state.neck_trk_pe[i], pe.data(), 0, pe.size() * sizeof(float)); - } + // ── 6. Reshape to [H*W, NQ, NH, 1] for flash_attn_ext mask ─────────── + // Current: [NH, W, H, NQ]. Need: [N_kv=H*W, NQ, NH, B=1] + rpb = ggml_reshape_4d(ctx, rpb, NH, H * W, NQ, 1); + rpb = ggml_cont(ctx, ggml_permute(ctx, rpb, 2, 0, 1, 3)); // [H*W, NQ, NH, 1] - ggml_gallocr_free(galloc); - ggml_free(ctx0); + // Prepend zeros for presence token: mask for presence token has no box-relative bias + auto* pres_mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, H * W, 1, NH, 1); + ggml_set_name(pres_mask, "rpb_pres_zeros"); + ggml_set_input(pres_mask); // zeros — set by caller - fprintf(stderr, "%s: SAM2 encoding from preprocessed done\n", __func__); - return true; + // [H*W, NQ+1, NH, 1] + auto* full_rpb = ggml_concat(ctx, pres_mask, rpb, 1); + full_rpb = ggml_cont(ctx, full_rpb); + if (layer_idx == 0) { + ggml_set_name(full_rpb, "ddec_rpb_mask_0"); } - state.orig_width = img_size; - state.orig_height = img_size; - - // ── Build computation graph (SAM3 path) ── - const size_t buf_size = ggml_tensor_overhead() * 8192 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; - } + return full_rpb; +} - auto* inp = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, img_size, img_size, 3, 1); - ggml_set_name(inp, "input_image"); - ggml_set_input(inp); +// Single DETR decoder layer. +// queries: [D, N_q, B] where N_q = 201 (200 object queries + 1 presence token) +// query_pos: [D, N_q, B] positional encoding for queries +// enc_feats: [D, N_kv, B] conditioned image features from fusion encoder +// enc_pos: [D, N_kv, B] positional encoding for image features +// text_feats: [D, T, B] text features +// rpb_mask: [N_kv, N_q, n_heads, B] box-relative positional bias (or nullptr) +// Returns: updated queries [D, N_q, B] +static struct ggml_tensor* sam3_ddec_layer_forward( + struct ggml_context* ctx, + const sam3_ddec_layer& ly, + struct ggml_tensor* queries, + struct ggml_tensor* query_pos, + struct ggml_tensor* enc_feats, + struct ggml_tensor* enc_pos, + struct ggml_tensor* text_feats, + int n_heads, + struct ggml_tensor* text_attn_bias = nullptr, + struct ggml_tensor* rpb_mask = nullptr, + int layer_idx = -1) { + const int64_t D = queries->ne[0]; - auto* vit_out = sam3_build_vit_graph(ctx0, inp, model); - ggml_set_name(vit_out, "vit_output"); - ggml_set_output(vit_out); + // Python decoder layer order (all post-norm): + // 1. Self-attention → norm2 (post-norm) + // 2. Text cross-attention (ca_text) → catext_norm (post-norm) + // 3. Image cross-attention (cross_attn) → norm1 (post-norm) + // 4. FFN → norm3 (post-norm) + // + // Norm weight mapping: + // ly.norm2_w = ".norm2.weight" = Python norm2 (post-SA) + // ly.norm3_w = ".norm_ca_text.weight" = Python catext_norm (post-text-CA) + // ly.norm1_w = ".norm1.weight" = Python norm1 (post-image-CA) + // ly.norm4_w = ".norm3.weight" = Python norm3 (post-FFN) - // Mark debug intermediate tensors as outputs so graph allocator preserves them + // 1. Self-attention among queries (post-norm) { - const char* dbg_names[] = { - "dbg_patch_embed", - "dbg_after_pos_embed", - "dbg_ln_pre_norm", - "dbg_ln_pre_scale", - "dbg_after_ln_pre", - "dbg_block_15_norm1", - "dbg_block_15_qkv_proj", - "dbg_block_15_q_split", - "dbg_block_15_k_split", - "dbg_block_15_v_split", - "dbg_block_15_q_heads_base", - "dbg_block_15_k_heads_base", - "dbg_block_15_v_heads_base", - "dbg_block_15_q_heads", - "dbg_block_15_k_heads", - "dbg_block_15_v_flash", - "dbg_block_15_q_rope", - "dbg_block_15_k_rope", - "dbg_block_15_q_flash", - "dbg_block_15_k_flash", - "dbg_block_15_attn_out", - "dbg_block_15_attn_proj", - "dbg_block_15_resid1", - "dbg_block_15_norm2", - "dbg_block_15_mlp", - }; - for (const char* dn : dbg_names) { - auto* dt = ggml_get_tensor(ctx0, dn); - if (dt) ggml_set_output(dt); - } - for (int i = 0; i < (int)model.hparams.vit_depth; ++i) { - char bn[64]; - snprintf(bn, sizeof(bn), "dbg_block_%d_out", i); - auto* dt = ggml_get_tensor(ctx0, bn); - if (dt) ggml_set_output(dt); - } - } + // Q = K = queries + query_pos, V = queries (no pos) + auto* q_in = ggml_add(ctx, queries, query_pos); - struct ggml_tensor* neck_det_out[4] = {}; - struct ggml_tensor* neck_trk_out[4]; - if (!model.hparams.visual_only) { - sam3_build_neck_graph(ctx0, vit_out, model.neck_det, neck_det_out); - } - sam3_build_neck_graph(ctx0, vit_out, model.neck_trk, neck_trk_out); + auto* q_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]); + auto* q_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float)); - for (int i = 0; i < 4; ++i) { - char name[64]; - if (!model.hparams.visual_only) { - snprintf(name, sizeof(name), "neck_det_%d", i); - ggml_set_name(neck_det_out[i], name); - ggml_set_output(neck_det_out[i]); - } - snprintf(name, sizeof(name), "neck_trk_%d", i); - ggml_set_name(neck_trk_out[i], name); - ggml_set_output(neck_trk_out[i]); - } + const int64_t N = queries->ne[1]; + const int64_t B = queries->ne[2]; + const int64_t HD = D / n_heads; - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 16384, false); - for (int i = 0; i < 4; ++i) { - if (!model.hparams.visual_only) { - ggml_build_forward_expand(graph, neck_det_out[i]); - } - ggml_build_forward_expand(graph, neck_trk_out[i]); - } + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, q_in), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, queries), v_b); - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, N, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - if (!ggml_gallocr_reserve(galloc, graph)) { - fprintf(stderr, "%s: failed to reserve graph memory\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + float scale = 1.0f / sqrtf((float)HD); + auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + sa_out = ggml_reshape_3d(ctx, sa_out, D, N, B); + sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); + sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); - if (!ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; + queries = ggml_add(ctx, queries, sa_out); + // Post-norm: norm2 (Python's post-SA norm) + queries = sam3_layer_norm(ctx, queries, ly.norm2_w, ly.norm2_b); + if (layer_idx == 0) { + ggml_set_name(queries, "ddec_layer0_after_sa"); + } } - // Upload preprocessed data directly (already in CHW = ggml [W, H, C, B] layout) - const size_t data_bytes = (size_t)3 * img_size * img_size * sizeof(float); - ggml_backend_tensor_set(inp, chw_data, 0, data_bytes); - - // Compute + // 2. Cross-attention to text tokens (post-norm) { - auto t0 = std::chrono::high_resolution_clock::now(); - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } - auto t1 = std::chrono::high_resolution_clock::now(); - double ms = std::chrono::duration(t1 - t0).count(); - fprintf(stderr, "%s: graph computed in %.1f ms (%d threads)\n", - __func__, ms, state.n_threads); - } + // Q = queries + query_pos, K = V = text_feats + auto* q_in = ggml_add(ctx, queries, query_pos); - // Cache results in state - if (state.galloc) ggml_gallocr_free(state.galloc); - if (state.ctx) ggml_free(state.ctx); + auto* q_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], D * ly.ca_text_q_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], 2 * D * ly.ca_text_q_w->nb[1]); + auto* q_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, 2 * D * sizeof(float)); - state.ctx = ctx0; - state.galloc = galloc; - state.backend = model.backend; - state.vit_output = vit_out; + const int64_t N_q = queries->ne[1]; + const int64_t N_kv = text_feats->ne[1]; + const int64_t B = queries->ne[2]; + const int64_t HD = D / n_heads; - for (int i = 0; i < 4; ++i) { - state.neck_det[i] = model.hparams.visual_only ? nullptr : neck_det_out[i]; - state.neck_trk[i] = neck_trk_out[i]; - } + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, text_feats), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, text_feats), v_b); - // Compute sinusoidal PEs - { - const int neck_dim = hp.neck_dim; - const int scale_sizes[4] = { - hp.n_img_embd() * 4, - hp.n_img_embd() * 2, - hp.n_img_embd(), - hp.n_img_embd() / 2, - }; + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); - if (state.pe_buf) { - ggml_backend_buffer_free(state.pe_buf); - state.pe_buf = nullptr; - } - if (state.pe_ctx) { - ggml_free(state.pe_ctx); - state.pe_ctx = nullptr; + auto* text_mask = sam3_expand_token_attn_bias(ctx, text_attn_bias, N_q, n_heads, B); + float scale = 1.0f / sqrtf((float)HD); + auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, text_mask, scale, 0.0f, 0.0f); + ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); + ca_out = ggml_mul_mat(ctx, ly.ca_text_out_w, ca_out); + ca_out = ggml_add(ctx, ca_out, ly.ca_text_out_b); + + queries = ggml_add(ctx, queries, ca_out); + // Post-norm: catext_norm (Python's post-text-CA norm) + queries = sam3_layer_norm(ctx, queries, ly.norm3_w, ly.norm3_b); + if (layer_idx == 0) { + ggml_set_name(queries, "ddec_layer0_after_text_ca"); } + } - struct ggml_init_params pe_params = { - /*.mem_size =*/ggml_tensor_overhead() * 4 + 256, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - state.pe_ctx = ggml_init(pe_params); + // 3. Cross-attention to conditioned image features (post-norm) + { + // Q = queries + query_pos, K = enc_feats + enc_pos, V = enc_feats + auto* q_in = ggml_add(ctx, queries, query_pos); + auto* k_in = ggml_add(ctx, enc_feats, enc_pos); - struct ggml_tensor* pe_tensors[4]; - for (int i = 0; i < 4; ++i) { - const int S = scale_sizes[i]; - pe_tensors[i] = ggml_new_tensor_4d(state.pe_ctx, GGML_TYPE_F32, neck_dim, S, S, 1); - char name[64]; - snprintf(name, sizeof(name), "pe_%d", i); - ggml_set_name(pe_tensors[i], name); - } + auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); + auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); + auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); + auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); + auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); + auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); - state.pe_buf = ggml_backend_alloc_ctx_tensors(state.pe_ctx, model.backend); - if (!state.pe_buf) { - fprintf(stderr, "%s: failed to allocate PE buffer\n", __func__); + const int64_t N_q = queries->ne[1]; + const int64_t N_kv = enc_feats->ne[1]; + const int64_t B = queries->ne[2]; + const int64_t HD = D / n_heads; + + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_in), k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, enc_feats), v_b); + + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); + K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); + V = ggml_permute(ctx, V, 0, 2, 1, 3); + + float scale = 1.0f / sqrtf((float)HD); + struct ggml_tensor* ca_out = nullptr; + + if (rpb_mask) { + // Keep the box-relative positional bias in fp32. The CPU flash-attn + // kernel reads mask storage as fp16, which is good enough for token + // padding masks but introduces avoidable drift here. + auto* kq = ggml_mul_mat(ctx, K, Q); // [N_kv, N_q, NH, B] + kq = ggml_soft_max_ext(ctx, kq, rpb_mask, scale, 0.0f); // [N_kv, N_q, NH, B] + + auto* v_t = ggml_cont(ctx, ggml_transpose(ctx, V)); // [N_kv, HD, NH, B] + ca_out = ggml_mul_mat(ctx, v_t, kq); // [HD, N_q, NH, B] + ca_out = ggml_cont(ctx, ggml_permute(ctx, ca_out, 0, 2, 1, 3)); } else { - for (int i = 0; i < 4; ++i) { - const int S = scale_sizes[i]; - auto pe_data = sam3_sinusoidal_pe_2d(S, S, neck_dim); - ggml_backend_tensor_set(pe_tensors[i], pe_data.data(), 0, pe_data.size() * sizeof(float)); - state.neck_det_pe[i] = pe_tensors[i]; - state.neck_trk_pe[i] = pe_tensors[i]; - } + ca_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + } + + ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); + ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); + ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); + + queries = ggml_add(ctx, queries, ca_out); + // Post-norm: norm1 (Python's post-image-CA norm) + queries = sam3_layer_norm(ctx, queries, ly.norm1_w, ly.norm1_b); + if (layer_idx == 0) { + ggml_set_name(queries, "ddec_layer0_after_img_ca"); } } - state.pe_cache_valid = false; + // 4. FFN (post-norm, ReLU) + { + auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, queries); + ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); + ffn = ggml_relu(ctx, ffn); + ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); + ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); - auto t_end = std::chrono::high_resolution_clock::now(); - double total_ms = std::chrono::duration(t_end - t_start).count(); - fprintf(stderr, "%s: encoded successfully in %.1f ms\n", __func__, total_ms); - return true; -} + queries = ggml_add(ctx, queries, ffn); + // Post-norm: norm3 (Python's post-FFN norm) + queries = sam3_layer_norm(ctx, queries, ly.norm4_w, ly.norm4_b); + if (layer_idx == 0) { + ggml_set_name(queries, "ddec_layer0_full_out"); + } + } -/***************************************************************************** -** Multi-head attention helper (used by fusion encoder, DETR decoder, seg head) -*****************************************************************************/ + return queries; +} -// Standard multi-head attention with fused in_proj. -// q_in, k_in, v_in: [D, N, B] (if fused_qkv, only q_in is used and contains QKV stacked) -// in_proj_w: [D, 3*D] (fused Q/K/V projection) -// in_proj_b: [3*D] -// out_proj_w: [D, D], out_proj_b: [D] -// n_heads: number of attention heads -// Returns: [D, N_q, B] +// DotProductScoring: classify queries against text features via dot product. // -// If separate_kv is true, q_in/k_in/v_in are already separate (no fused proj needed). -// The in_proj is applied to form Q from q_in, and K/V from the concatenated k/v source. -static struct ggml_tensor* sam3_multihead_attn_fused( +// Python reference (DotProductScoring.forward): +// 1. prompt_mlp(prompt) → residual MLP + LN on text features +// 2. mean_pool_text(result, prompt_mask) → pooled [BS, D] (only valid tokens) +// 3. prompt_proj(pooled) → [BS, D] +// 4. hs_proj(hs) → [num_layer, BS, N_q, D] +// 5. matmul(proj_hs, proj_pooled.unsqueeze(-1)) → dot product → [num_layer, BS, N_q, 1] +// 6. scale by 1/sqrt(D) +// 7. clamp to [-12, 12] +// +// query_outputs: [D, N_q, B] — the 200 object query outputs +// text_features: [D, T, B] — text encoder output (already through resizer) +// text_valid_mask: [T, 1, B] — 1.0 for valid tokens, 0.0 for padding (or nullptr for all-valid) +// Returns: class_scores [N_q, B] (one score per query per batch) +static struct ggml_tensor* sam3_dot_product_scoring( struct ggml_context* ctx, - struct ggml_tensor* q_in, // [D, N_q, B] - struct ggml_tensor* kv_in, // [D, N_kv, B] (can be same as q_in for self-attn) - struct ggml_tensor* in_proj_w, // [D, 3*D] — fused QKV weights - struct ggml_tensor* in_proj_b, // [3*D] - struct ggml_tensor* out_proj_w, // [D, D] - struct ggml_tensor* out_proj_b, // [D] - int n_heads, - struct ggml_tensor* attn_mask = nullptr) // [N_kv, N_q] or nullptr + const sam3_model& model, + struct ggml_tensor* query_outputs, // [D, N_q, B] + struct ggml_tensor* text_features, // [D, T, B] + struct ggml_tensor* text_valid_mask) // [T, 1, B] or nullptr { - const int64_t D = q_in->ne[0]; // 256 - const int64_t N_q = q_in->ne[1]; - const int64_t B = q_in->ne[2]; - const int64_t N_kv = kv_in->ne[1]; - const int64_t HD = D / n_heads; + const auto& tensors = model.tensors; + const int64_t D = query_outputs->ne[0]; // 256 + const int64_t T = text_features->ne[1]; + const int64_t B = text_features->ne[2]; - auto* q_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], D * in_proj_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, in_proj_w, D, D, in_proj_w->nb[1], 2 * D * in_proj_w->nb[1]); + // Step 1: Apply prompt_mlp on text features (residual MLP + LayerNorm) + auto* text_mlp = text_features; // [D, T, B] + auto* orig_text = text_features; + text_mlp = ggml_mul_mat(ctx, tensors.at("scoring.prompt_mlp.layers.0.weight"), text_mlp); + text_mlp = ggml_add(ctx, text_mlp, tensors.at("scoring.prompt_mlp.layers.0.bias")); + text_mlp = ggml_relu(ctx, text_mlp); + text_mlp = ggml_mul_mat(ctx, tensors.at("scoring.prompt_mlp.layers.1.weight"), text_mlp); + text_mlp = ggml_add(ctx, text_mlp, tensors.at("scoring.prompt_mlp.layers.1.bias")); + text_mlp = ggml_add(ctx, text_mlp, orig_text); + text_mlp = sam3_layer_norm(ctx, text_mlp, + tensors.at("scoring.prompt_mlp.out_norm.weight"), + tensors.at("scoring.prompt_mlp.out_norm.bias")); + // text_mlp: [D, T, B] + ggml_set_name(text_mlp, "scoring_prompt_mlp_out"); - auto* q_b = ggml_view_1d(ctx, in_proj_b, D, 0); - auto* k_b = ggml_view_1d(ctx, in_proj_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, in_proj_b, D, 2 * D * sizeof(float)); + // Step 2: Mean-pool over valid text tokens → [D, 1, B] + // Python: pooled = (prompt * is_valid).sum(0) / num_valid + struct ggml_tensor* text_pooled; + if (text_valid_mask) { + // Permute to [T, D, B] for masked pooling + auto* tp = ggml_cont(ctx, ggml_permute(ctx, text_mlp, 1, 0, 2, 3)); // [T, D, B] + // Mask: text_valid_mask is [T, 1, B] — broadcast multiply zeros out padding + tp = ggml_mul(ctx, tp, text_valid_mask); // [T, D, B] with padding zeroed + // Sum over T dimension: pool_1d with SUM kernel=T + // ggml_pool_1d AVG divides by T; we want SUM then divide by n_valid. + // Use AVG and then scale by T/n_valid? Or use a manual approach. + // Simpler: sum via pool_1d with AVG, then scale by T/n_valid. + // But n_valid is dynamic. Instead: sum = mean * T, then divide by n_valid. + // We pass n_valid as part of the mask: text_valid_mask sums to n_valid. + // pool_1d(masked, AVG, T, T, 0) = sum(masked) / T. Multiply by T → sum(masked). + // Then divide by n_valid. But n_valid is a scalar we know CPU-side. + // For simplicity: compute AVG over ALL T positions (with padding zeroed out). + // This gives sum(valid) / T. To get sum(valid) / n_valid, scale by T / n_valid. + // We embed the scale factor into the mask: mask = (T / n_valid) for valid, 0 for pad. + // Then AVG(mask * features) = sum(valid * T/n_valid) / T = sum(valid) / n_valid. ✓ + // Caller should set mask values to T/n_valid for valid tokens, 0 for padding. + auto* pooled_t = ggml_pool_1d(ctx, tp, GGML_OP_POOL_AVG, (int)T, (int)T, 0); + text_pooled = ggml_cont(ctx, ggml_permute(ctx, pooled_t, 1, 0, 2, 3)); // [D, 1, B] + } else { + // All tokens valid — simple mean + auto* tp = ggml_cont(ctx, ggml_permute(ctx, text_mlp, 1, 0, 2, 3)); + auto* pooled_t = ggml_pool_1d(ctx, tp, GGML_OP_POOL_AVG, (int)T, (int)T, 0); + text_pooled = ggml_cont(ctx, ggml_permute(ctx, pooled_t, 1, 0, 2, 3)); + } + ggml_set_name(text_pooled, "scoring_pooled"); - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, kv_in), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, kv_in), v_b); + // Step 3: Project pooled prompt through prompt_proj: D→D + auto* proj_pooled = ggml_mul_mat(ctx, tensors.at("scoring.prompt_proj.weight"), text_pooled); + proj_pooled = ggml_add(ctx, proj_pooled, tensors.at("scoring.prompt_proj.bias")); + // proj_pooled: [D, 1, B] + ggml_set_name(proj_pooled, "scoring_proj_pooled"); - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); // [HD, N_q, NH, B] + // Step 4: Project queries through hs_proj: D→D + auto* proj_hs = ggml_mul_mat(ctx, tensors.at("scoring.hs_proj.weight"), query_outputs); + proj_hs = ggml_add(ctx, proj_hs, tensors.at("scoring.hs_proj.bias")); + // proj_hs: [D, N_q, B] + ggml_set_name(proj_hs, "scoring_proj_hs"); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); // [HD, N_kv, NH, B] + // Step 5: Dot product — for each query, dot with pooled prompt + // matmul(proj_hs, proj_pooled.unsqueeze(-1)) in Python = batched vector-matrix multiply + // ggml_mul_mat(A, B) = A^T @ B + // With A = proj_pooled [D, 1, B], B = proj_hs [D, N_q, B]: + // result = [1, N_q, B] — each element is dot product of query with pooled prompt + auto* scores = ggml_mul_mat(ctx, proj_pooled, proj_hs); // [1, N_q, B] - V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); // [HD, N_kv, NH, B] non-contiguous; flash_attn uses strides + // Step 6: Scale by 1/sqrt(D) + float scale = 1.0f / sqrtf((float)D); + scores = ggml_scale(ctx, scores, scale); - float scale = 1.0f / sqrtf((float)HD); - auto* attn_out = ggml_flash_attn_ext(ctx, Q, K, V, attn_mask, scale, 0.0f, 0.0f); + // Step 7: Clamp to [-12, 12] + scores = ggml_clamp(ctx, scores, -12.0f, 12.0f); - auto* merged = ggml_reshape_3d(ctx, attn_out, D, N_q, B); - merged = ggml_mul_mat(ctx, out_proj_w, merged); - merged = ggml_add(ctx, merged, out_proj_b); + // Reshape to [N_q, B] + const int64_t N_q = query_outputs->ne[1]; + scores = ggml_reshape_2d(ctx, scores, N_q, B); + ggml_set_name(scores, "scoring_class_scores"); - return merged; + return scores; } -static struct ggml_tensor* sam3_expand_token_attn_bias( - struct ggml_context* ctx, - struct ggml_tensor* token_bias, // [T, 1, B] or nullptr - int64_t n_q, - int n_heads, - int64_t batch) { - if (!token_bias) { - return nullptr; - } +// Build full DETR decoder graph. +// enc_feats: [D, N_kv, B] conditioned features from fusion encoder (N_kv=5184) +// enc_pos: [D, N_kv, B] positional encoding +// text_feats: [D, T, B] text features +// Returns struct with: +// queries: [D, 201, B] (all query outputs including presence token) +// pred_boxes: [4, 200, B] (cx, cy, w, h in [0,1]) +// class_scores: [200, B] +// presence_score: [1, B] +struct sam3_ddec_output { + struct ggml_tensor* queries; // [D, 201, B] + struct ggml_tensor* presence_feats; // [D, 1, B] pre-decoder-norm presence token + struct ggml_tensor* pred_boxes; // [4, 200, B] + struct ggml_tensor* class_scores; // [200, B] + struct ggml_tensor* presence_score; // [1, B] +}; - const int64_t n_kv = token_bias->ne[0]; - auto* bias_4d = ggml_reshape_4d(ctx, token_bias, n_kv, 1, 1, batch); - auto* full_bias = ggml_repeat( - ctx, - bias_4d, - ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_kv, n_q, n_heads, batch)); +static sam3_ddec_output sam3_build_ddec_graph( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* enc_feats, // [D, N_kv, B] + struct ggml_tensor* enc_pos, // [D, N_kv, B] + struct ggml_tensor* text_feats, // [D, T, B] + struct ggml_tensor* sine_dim_t, // [1, 64] — pre-computed angle multipliers + struct ggml_tensor* rpb_coords, // [feat_hw] — normalized grid coords (or nullptr) + struct ggml_tensor* text_attn_bias = nullptr, // [T, 1, B] additive text padding bias + struct ggml_tensor* text_valid_mask = nullptr) // [T, 1, B] for scoring (or nullptr) +{ + const auto& hp = model.hparams; + const auto& tensors = model.tensors; + const int D = hp.neck_dim; // 256 + const int NQ = hp.ddec_num_queries; // 200 + const int B = (int)enc_feats->ne[2]; // batch (1) + const int feat_hw = hp.n_img_embd(); // 72 - // ggml_flash_attn_ext expects mask storage in fp16. - return ggml_cont(ctx, ggml_cast(ctx, full_bias, GGML_TYPE_F16)); -} + // ── Initialize queries from query_embed ────────────────────────────── + auto* content = ggml_reshape_3d(ctx, model.ddec.query_embed, D, NQ, 1); + auto* pres_tok = ggml_reshape_3d(ctx, model.ddec.presence_token, D, 1, 1); + auto* queries = ggml_concat(ctx, pres_tok, content, 1); // [D, NQ+1, B=1] -/***************************************************************************** -** Geometry / exemplar encoder — graph building -*****************************************************************************/ + // Reference points: sigmoid → initial anchor boxes + auto* ref_pts_raw = ggml_cont(ctx, tensors.at("ddec.reference_points.weight")); // [4, NQ] + auto* ref_boxes = ggml_sigmoid(ctx, ref_pts_raw); // [4, NQ] + ref_boxes = ggml_reshape_3d(ctx, ref_boxes, 4, NQ, 1); // [4, NQ, 1] +#ifndef NDEBUG + auto* ref_boxes_dbg = ggml_cont(ctx, ref_boxes); + ggml_set_name(ref_boxes_dbg, "ddec_ref_boxes_init"); +#endif -// Sinusoidal positional encoding for box coordinates. -// Matches Python PositionEmbeddingSine.encode_boxes(cx, cy, w, h). -// Output: [258] = [pos_y(128), pos_x(128), h, w] -static void sam3_sine_encode_box(float* out, float cx, float cy, float w, float h, - int num_pos_feats, int temperature) { - const float scale = 2.0f * (float)M_PI; - float x_embed = cx * scale; - float y_embed = cy * scale; + // ── Run decoder layers ─────────────────────────────────────────────── + // Per-layer: recompute query_pos from updated ref_boxes (matching Python exactly) + struct ggml_tensor* last_presence = pres_tok; + for (int i = 0; i < hp.ddec_layers; ++i) { + // Recompute query_pos from current ref_boxes via sine embed + ref_point_head MLP + auto* query_pos = sam3_build_query_pos(ctx, model, ref_boxes, sine_dim_t, i); - // dim_t[i] = temperature^(2*(i//2)/num_pos_feats) - for (int i = 0; i < num_pos_feats; ++i) { - int div_idx = 2 * (i / 2); - float dim_t = powf((float)temperature, (float)div_idx / (float)num_pos_feats); - float px = x_embed / dim_t; - float py = y_embed / dim_t; - // Interleaved sin/cos: even indices get sin, odd get cos - if (i % 2 == 0) { - out[i] = sinf(py); // pos_y first - out[num_pos_feats + i] = sinf(px); // pos_x second - } else { - out[i] = cosf(py); - out[num_pos_feats + i] = cosf(px); + // Compute box-relative positional bias for image cross-attention + struct ggml_tensor* rpb_mask = nullptr; + if (rpb_coords) { + rpb_mask = sam3_compute_box_rpb(ctx, model, ref_boxes, rpb_coords, feat_hw, i); } - } - out[2 * num_pos_feats] = h; // h - out[2 * num_pos_feats + 1] = w; // w -} -// CPU-side ROI Align matching torchvision.ops.roi_align behavior. -// Features in ggml [C, W, H] layout. Box in XYXY format scaled to feature grid coords. -// Uses sub-sampling matching torchvision's sampling_ratio=0 (auto). -// Output: [C * roi_size * roi_size] in ggml layout. -static void sam3_roi_align_single( - const float* feats, // [C, W, H] ggml layout (C innermost, then W, then H) - int C, int W_feat, int H_feat, - float x0, float y0, float x1, float y1, - int roi_size, - float* out) // [C, roi_size, roi_size] -{ - float roi_w = std::max(x1 - x0, 1e-6f); - float roi_h = std::max(y1 - y0, 1e-6f); - float bin_w = roi_w / (float)roi_size; - float bin_h = roi_h / (float)roi_size; + queries = sam3_ddec_layer_forward(ctx, model.ddec.layers[i], + queries, query_pos, + enc_feats, enc_pos, + text_feats, hp.ddec_heads, + text_attn_bias, + rpb_mask, + i); - // Match torchvision sampling_ratio=0: use ceil(bin_size) sub-samples - int sample_y_count = std::max(1, (int)ceilf(bin_h)); - int sample_x_count = std::max(1, (int)ceilf(bin_w)); - float inv_count = 1.0f / (float)(sample_y_count * sample_x_count); + // Box refinement after each layer (on object queries only, not presence token) + auto* obj_q = ggml_view_3d(ctx, queries, D, NQ, 1, + queries->nb[1], queries->nb[2], 1 * queries->nb[1]); + obj_q = ggml_cont(ctx, obj_q); + sam3_name_tensorf(obj_q, "ddec_layer%d_out", i); - auto bilinear_sample = [&](float sx, float sy, int c) -> float { - // Clamp to [-0.5, W-0.5] then adjust — matching torchvision - if (sx < -1.0f || sx > (float)W_feat || sy < -1.0f || sy > (float)H_feat) - return 0.0f; + auto* pres_q = ggml_view_3d(ctx, queries, D, 1, 1, + queries->nb[1], queries->nb[2], 0); + pres_q = ggml_cont(ctx, pres_q); + if (i == 0) { + ggml_set_name(pres_q, "ddec_layer0_presence"); + } + last_presence = pres_q; - sy = std::max(0.0f, sy); - sx = std::max(0.0f, sx); + // Apply the final decoder norm before box refinement (use_normed_output_consistently) + auto* obj_q_normed = sam3_layer_norm(ctx, obj_q, + tensors.at("ddec.norm.weight"), + tensors.at("ddec.norm.bias")); - int y_lo = (int)sy; - int x_lo = (int)sx; - int y_hi = std::min(y_lo + 1, H_feat - 1); - int x_hi = std::min(x_lo + 1, W_feat - 1); - y_lo = std::min(y_lo, H_feat - 1); - x_lo = std::min(x_lo, W_feat - 1); + // Shared bbox_embed MLP + auto* bd = obj_q_normed; + for (int j = 0; j < 3; ++j) { + auto wn = "ddec.bbox_embed.layers." + std::to_string(j) + ".weight"; + auto bn = "ddec.bbox_embed.layers." + std::to_string(j) + ".bias"; + bd = ggml_mul_mat(ctx, tensors.at(wn), bd); + bd = ggml_add(ctx, bd, tensors.at(bn)); + if (j < 2) bd = ggml_relu(ctx, bd); + } + // bd: [4, NQ, 1] - float ly = sy - (float)y_lo; - float lx = sx - (float)x_lo; + // ref_boxes = sigmoid(inverse_sigmoid(ref_boxes) + box_delta) + auto* ref_inv_cur = sam3_inverse_sigmoid(ctx, ref_boxes); + ref_boxes = ggml_sigmoid(ctx, ggml_add(ctx, ref_inv_cur, bd)); + sam3_name_tensorf(ref_boxes, "ddec_layer%d_refboxes", i); + } - // ggml layout: feats[c + x * C + y * C * W_feat] - float v00 = feats[c + x_lo * C + y_lo * C * W_feat]; - float v10 = feats[c + x_hi * C + y_lo * C * W_feat]; - float v01 = feats[c + x_lo * C + y_hi * C * W_feat]; - float v11 = feats[c + x_hi * C + y_hi * C * W_feat]; + // ── Final normalization ────────────────────────────────────────────── + // Match Python: decoder.norm is applied to object queries only. + auto* obj_queries = ggml_view_3d(ctx, queries, D, NQ, 1, + queries->nb[1], queries->nb[2], 1 * queries->nb[1]); + obj_queries = ggml_cont(ctx, obj_queries); + obj_queries = sam3_layer_norm(ctx, obj_queries, + tensors.at("ddec.norm.weight"), + tensors.at("ddec.norm.bias")); + ggml_set_name(obj_queries, "ddec_normed_output"); - return v00 * (1 - ly) * (1 - lx) + v10 * (1 - ly) * lx + v01 * ly * (1 - lx) + v11 * ly * lx; - }; + auto* queries_for_seg = ggml_concat(ctx, last_presence, obj_queries, 1); - for (int ph = 0; ph < roi_size; ++ph) { - for (int pw = 0; pw < roi_size; ++pw) { - for (int c = 0; c < C; ++c) { - float sum = 0.0f; - for (int iy = 0; iy < sample_y_count; ++iy) { - float sy = y0 + bin_h * ((float)ph + ((float)iy + 0.5f) / (float)sample_y_count); - for (int ix = 0; ix < sample_x_count; ++ix) { - float sx = x0 + bin_w * ((float)pw + ((float)ix + 0.5f) / (float)sample_x_count); - sum += bilinear_sample(sx, sy, c); - } - } - out[c + pw * C + ph * C * roi_size] = sum * inv_count; - } - } + auto* class_scores = sam3_dot_product_scoring(ctx, model, obj_queries, text_feats, text_valid_mask); + // class_scores: [NQ, B] + + // ── Presence score ─────────────────────────────────────────────────── + // Presence token head: LN + 3-layer MLP (D→D→D→1) + auto* pres_out = sam3_layer_norm(ctx, last_presence, + tensors.at("ddec.presence_token_out_norm.weight"), + tensors.at("ddec.presence_token_out_norm.bias")); + + for (int j = 0; j < 3; ++j) { + auto wn = "ddec.presence_token_head.layers." + std::to_string(j) + ".weight"; + auto bn = "ddec.presence_token_head.layers." + std::to_string(j) + ".bias"; + pres_out = ggml_mul_mat(ctx, tensors.at(wn), pres_out); + pres_out = ggml_add(ctx, pres_out, tensors.at(bn)); + if (j < 2) pres_out = ggml_relu(ctx, pres_out); } + // Keep presence as raw logit (no sigmoid yet — applied during post-processing) + auto* presence_score = ggml_reshape_2d(ctx, pres_out, 1, 1); + // presence_score: [1, B] — raw logit + + sam3_ddec_output out; + out.queries = queries_for_seg; // [D, NQ+1, B] + out.presence_feats = last_presence; // [D, 1, B] + out.pred_boxes = ref_boxes; // [4, NQ, B] + out.class_scores = class_scores; // [NQ, B] + out.presence_score = presence_score; // [1, B] + + return out; } -// Build geometry encoder graph and pre-compute box embeddings on CPU. -// Returns the geometry features as a pre-computed input tensor [D, N_geo, 1] -// where N_geo = n_exemplar_boxes + 1 (CLS). -// For dummy prompts (no boxes), N_geo = 1 (just CLS token). -struct sam3_geom_result { - struct ggml_tensor* geo_feats; // [D, N_geo, 1] - int n_tokens; // N_geo -}; +/***************************************************************************** +** Segmentation head (MaskFormer) — graph building +*****************************************************************************/ -static sam3_geom_result sam3_build_geom_enc_graph( +// Build pixel decoder: progressively upsample FPN features. +// fpn_feats[0]: [D, 288, 288, B] (highest res) +// fpn_feats[1]: [D, 144, 144, B] +// fpn_feats[2]: [D, 72, 72, B] (lowest res) +// Returns: [D, 288, 288, B] pixel features +// +// Python PixelDecoder.forward: +// prev_fpn = backbone_feats[-1] (lowest res) +// for bb_feat in backbone_feats[:-1][::-1]: (iterate from second-lowest to highest) +// prev_fpn = bb_feat + F.interpolate(prev_fpn, size=bb_feat.shape[-2:], mode="nearest") +// prev_fpn = conv_layers[i](prev_fpn) # conv on the MERGED result +// prev_fpn = F.relu(norms[i](prev_fpn)) # GroupNorm then ReLU +// +// Python uses GroupNorm(8, 256) — we use ggml_group_norm which normalizes ne[2] +// (the channel dim) in groups. The conv output is [W, H, D, B] with D in ne[2]. +static struct ggml_tensor* sam3_pixel_decoder( struct ggml_context* ctx, const sam3_model& model, - const sam3_pcs_params& params, - struct ggml_tensor* img_feats, // [D, N_img, 1] where N_img = H*H - struct ggml_tensor* img_pe) // [D, N_img, 1] sinusoidal PE + struct ggml_tensor* fpn_feats[3]) // [D, W, H, B] at 3 scales { - const auto& ge = model.geom_enc; - const int D = model.hparams.neck_dim; // 256 - const int n_heads = 8; - const int n_boxes = (int)(params.pos_exemplars.size() + params.neg_exemplars.size()); - const int N_geo = n_boxes + 1; // +1 for CLS - - // Input tensor: pre-computed geometry embeddings after final_proj + norm [D, N_geo, 1] - // The caller pre-computes: final_proj(box_embeds + CLS) → LayerNorm - auto* x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_geo, 1); - ggml_set_name(x, "geom_post_final_proj"); // also used as upload target - ggml_set_input(x); - - // Transformer layers (3 layers: self-attn + cross-attn + FFN, pre-norm) - for (int i = 0; i < (int)ge.layers.size(); ++i) { - const auto& ly = ge.layers[i]; - - // 1. Self-attention (pre-norm, pos_enc_at_attn=False) - { - auto* shortcut = x; - auto* xn = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); - - // Q = K = V = norm(x) — no positional encoding at self-attention - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0), xn), - ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0)); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]), xn), - ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float))); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]), xn), - ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float))); - - const int64_t S = N_geo; - const int64_t HD = D / n_heads; + const auto& seg = model.seg_head; - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, S, 1); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, S, 1); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, S, 1); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + // Start from lowest resolution + auto* feat = fpn_feats[2]; // [D, 72, 72, B] - float scale = 1.0f / sqrtf((float)HD); - auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); - sa_out = ggml_reshape_3d(ctx, sa_out, D, S, 1); + // Iteration 0: merge with FPN[1] (144x144) + // prev_fpn = FPN[1] + upsample(prev_fpn) + // Permute to [W, H, D, B] for conv operations + auto* prev = ggml_cont(ctx, ggml_permute(ctx, feat, 2, 0, 1, 3)); // [72, 72, D, B] + prev = ggml_upscale(ctx, prev, 2, GGML_SCALE_MODE_NEAREST); // [144, 144, D, B] + auto* fpn1 = ggml_cont(ctx, ggml_permute(ctx, fpn_feats[1], 2, 0, 1, 3)); // [144, 144, D, B] + prev = ggml_add(ctx, fpn1, prev); // merged + // Conv 3x3 on the MERGED result (not individual FPN feat) + prev = ggml_conv_2d_s1_ph(ctx, seg.up_conv_w[0], prev); + { + auto* b3d = ggml_reshape_3d(ctx, seg.up_conv_b[0], 1, 1, seg.up_conv_b[0]->ne[0]); + prev = ggml_add(ctx, prev, ggml_repeat(ctx, b3d, prev)); + } + // GroupNorm(8, 256) then ReLU — prev is [W, H, D, B] with D in ne[2] + prev = ggml_group_norm(ctx, prev, 8, 1e-5f); + { + auto* w3d = ggml_reshape_3d(ctx, seg.up_norm_w[0], 1, 1, seg.up_norm_w[0]->ne[0]); + prev = ggml_mul(ctx, prev, ggml_repeat(ctx, w3d, prev)); + auto* bn3d = ggml_reshape_3d(ctx, seg.up_norm_b[0], 1, 1, seg.up_norm_b[0]->ne[0]); + prev = ggml_add(ctx, prev, ggml_repeat(ctx, bn3d, prev)); + } + prev = ggml_relu(ctx, prev); + ggml_set_name(prev, "seg_pixel_dec_stage0"); - sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); - sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); + // Iteration 1: merge with FPN[0] (288x288) + prev = ggml_upscale(ctx, prev, 2, GGML_SCALE_MODE_NEAREST); // [288, 288, D, B] + auto* fpn0 = ggml_cont(ctx, ggml_permute(ctx, fpn_feats[0], 2, 0, 1, 3)); // [288, 288, D, B] + prev = ggml_add(ctx, fpn0, prev); // merged + // Conv 3x3 on the MERGED result + prev = ggml_conv_2d_s1_ph(ctx, seg.up_conv_w[1], prev); + { + auto* b3d = ggml_reshape_3d(ctx, seg.up_conv_b[1], 1, 1, seg.up_conv_b[1]->ne[0]); + prev = ggml_add(ctx, prev, ggml_repeat(ctx, b3d, prev)); + } + // GroupNorm(8, 256) then ReLU + prev = ggml_group_norm(ctx, prev, 8, 1e-5f); + { + auto* w3d = ggml_reshape_3d(ctx, seg.up_norm_w[1], 1, 1, seg.up_norm_w[1]->ne[0]); + prev = ggml_mul(ctx, prev, ggml_repeat(ctx, w3d, prev)); + auto* bn3d = ggml_reshape_3d(ctx, seg.up_norm_b[1], 1, 1, seg.up_norm_b[1]->ne[0]); + prev = ggml_add(ctx, prev, ggml_repeat(ctx, bn3d, prev)); + } + prev = ggml_relu(ctx, prev); + ggml_set_name(prev, "seg_pixel_dec_stage1"); - x = ggml_add(ctx, shortcut, sa_out); - } + // Python PixelDecoder allocates 3 conv layers but only uses 2 (one per + // upsample step). The 3rd conv (up_conv_w[2]) is unused. - // 2. Cross-attention (pre-norm, Q from x, K from img+PE, V from img) - { - auto* shortcut = x; - auto* xn = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); + auto* out = ggml_cont(ctx, ggml_permute(ctx, prev, 1, 2, 0, 3)); // [D, 288, 288, B] + return out; +} - // Q from normalized geometry tokens (no pos) - // K from image features + PE - // V from image features - auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); - auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); +// Build the full segmentation head graph. +// +// Python UniversalSegmentationHead.forward: +// 1. Cross-attend encoder_hidden_states to prompt → updated encoder +// 2. _embed_pixels: replace lowest-res FPN feat with spatial portion of encoder output +// 3. Run pixel decoder on modified FPN feats +// 4. instance_seg_head (Conv1x1) +// 5. mask_predictor: einsum(mask_embed(queries), instance_embeds) +// +// enc_hidden: [D, N_spatial, B] — fusion encoder output (cross-attended in step 1) +// fpn_feats[3]: the 3 FPN features at different resolutions +// query_outputs: [D, N, B] selected object query outputs +// text_features: [D, T, B] for cross-attention (prompt) +// Returns: mask_logits [W*H, N, B] (raw logits, not sigmoid) +static struct ggml_tensor* sam3_build_seg_head_graph( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* enc_hidden, // [D, N_spatial, B] fusion encoder output + struct ggml_tensor* fpn_feats[3], // FPN features at 3 scales + struct ggml_tensor* query_outputs, // [D, N, B] + struct ggml_tensor* text_features, // [D, T, B] (for cross-attn, can be nullptr) + struct ggml_tensor* text_attn_bias = nullptr) { + const auto& seg = model.seg_head; + const auto& tensors = model.tensors; + const int64_t D = enc_hidden->ne[0]; // 256 + const int64_t B = enc_hidden->ne[2]; // 1 + const int64_t N = query_outputs->ne[1]; // number of selected queries - auto* k_input = ggml_add(ctx, img_feats, img_pe); // pos_enc_at_cross_attn_keys + auto* enc = enc_hidden; + if (text_features) { + auto* ca_norm = sam3_layer_norm(ctx, enc, + tensors.at("seg.cross_attn_norm.weight"), + tensors.at("seg.cross_attn_norm.bias")); - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, xn), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_input), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, img_feats), v_b); + auto* ca_mask = sam3_expand_token_attn_bias(ctx, text_attn_bias, enc->ne[1], 8, B); + auto* ca_out = sam3_multihead_attn_fused(ctx, ca_norm, text_features, + seg.ca_prompt_q_w, seg.ca_prompt_q_b, + seg.ca_prompt_out_w, seg.ca_prompt_out_b, + 8, ca_mask); + enc = ggml_add(ctx, enc, ca_out); + } + // enc: [D, N_spatial, B] + ggml_set_name(enc, "seg_enc_after_ca"); - const int64_t S_q = N_geo; - const int64_t S_kv = img_feats->ne[1]; - const int64_t HD = D / n_heads; + // Replace lowest-res FPN feat with spatial portion of encoder output + const int64_t feat_hw = model.hparams.n_img_embd(); // 72 + auto* enc_spatial = ggml_reshape_4d(ctx, enc, D, feat_hw, feat_hw, B); +#ifndef NDEBUG + auto* enc_spatial_dbg = ggml_cont(ctx, ggml_permute(ctx, enc_spatial, 2, 0, 1, 3)); + ggml_set_name(enc_spatial_dbg, "seg_enc_visual"); +#endif - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, S_q, 1); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, S_kv, 1); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, S_kv, 1); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + struct ggml_tensor* modified_fpn[3] = { + fpn_feats[0], + fpn_feats[1], + enc_spatial, // replaces original lowest-res FPN + }; - float scale = 1.0f / sqrtf((float)HD); - auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); - ca_out = ggml_reshape_3d(ctx, ca_out, D, S_q, 1); + auto* pixel_feats = sam3_pixel_decoder(ctx, model, modified_fpn); +#ifndef NDEBUG + auto* pixel_feats_dbg = ggml_cont(ctx, ggml_permute(ctx, pixel_feats, 2, 0, 1, 3)); + ggml_set_name(pixel_feats_dbg, "seg_pixel_decoder_out"); +#endif - ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); - ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); + const int64_t W = pixel_feats->ne[1]; // 288 + const int64_t H = pixel_feats->ne[2]; // 288 - x = ggml_add(ctx, shortcut, ca_out); - } + // Instance segmentation head (Conv1x1) + auto* pf_conv = ggml_cont(ctx, ggml_permute(ctx, pixel_feats, 2, 0, 1, 3)); + pf_conv = ggml_conv_2d_sk_p0(ctx, tensors.at("seg.instance_seg_head.weight"), pf_conv); + { + auto* b3d = ggml_reshape_3d(ctx, tensors.at("seg.instance_seg_head.bias"), + 1, 1, tensors.at("seg.instance_seg_head.bias")->ne[0]); + pf_conv = ggml_add(ctx, pf_conv, ggml_repeat(ctx, b3d, pf_conv)); + } + auto* pixel_embed = ggml_cont(ctx, ggml_permute(ctx, pf_conv, 1, 2, 0, 3)); // [D, W, H, B] +#ifndef NDEBUG + auto* pixel_embed_dbg = ggml_cont(ctx, ggml_permute(ctx, pixel_embed, 2, 0, 1, 3)); + ggml_set_name(pixel_embed_dbg, "seg_instance_embed"); +#endif - // 3. FFN (pre-norm, ReLU) - { - auto* shortcut = x; - auto* xn = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); - auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, xn); - ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); - ffn = ggml_relu(ctx, ffn); - ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); - ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); - x = ggml_add(ctx, shortcut, ffn); - } - sam3_name_tensorf(x, "geom_layer%d_out", i); + // Mask embedding MLP + auto* mask_embed = query_outputs; + for (int j = 0; j < 3; ++j) { + auto wn = "seg.mask_predictor.mask_embed.layers." + std::to_string(j) + ".weight"; + auto bn = "seg.mask_predictor.mask_embed.layers." + std::to_string(j) + ".bias"; + mask_embed = ggml_mul_mat(ctx, tensors.at(wn), mask_embed); + mask_embed = ggml_add(ctx, mask_embed, tensors.at(bn)); + if (j < 2) mask_embed = ggml_relu(ctx, mask_embed); } + ggml_set_name(mask_embed, "seg_mask_embed"); - // Final encode norm - x = sam3_layer_norm(ctx, x, ge.encode_norm_w, ge.encode_norm_b); - ggml_set_name(x, "geom_output"); - ggml_set_output(x); + // Mask prediction: einsum('bqc,bchw->bqhw') + auto* pe_flat = ggml_reshape_3d(ctx, pixel_embed, D, W * H, B); + auto* masks = ggml_mul_mat(ctx, pe_flat, mask_embed); + ggml_set_name(masks, "seg_mask_logits"); - sam3_geom_result result; - result.geo_feats = x; - result.n_tokens = N_geo; - return result; + return masks; } -// Pre-compute geometry encoder input on CPU: box embeddings + CLS token. -// Reads model weights from GPU, computes embeddings, returns float vector. -// Layout: [D, N_geo] row-major where N_geo = n_boxes + 1 (CLS last). -static std::vector sam3_precompute_geom_input( - const sam3_model& model, - const sam3_pcs_params& params, - const float* img_feats_data, // [D, W, H] ggml-layout backbone features (nullable if no boxes) - int W_feat, int H_feat) { - const auto& ge = model.geom_enc; - const int D = model.hparams.neck_dim; // 256 - const int roi_size = 7; - const int num_pos_feats = D / 2; // 128 +/***************************************************************************** +** Memory encoder (Phase 7, Step 7.1) +*****************************************************************************/ - // Collect all exemplar boxes - struct box_info { - float cx, cy, w, h; - int label; - }; - std::vector boxes; - for (const auto& b : params.pos_exemplars) { - // API provides XYXY in original image space — convert to normalized CxCyWH [0,1] - float cx = (b.x0 + b.x1) * 0.5f; - float cy = (b.y0 + b.y1) * 0.5f; - float bw = b.x1 - b.x0; - float bh = b.y1 - b.y0; - boxes.push_back({cx, cy, bw, bh, 0}); // label 0 = positive - } - for (const auto& b : params.neg_exemplars) { - float cx = (b.x0 + b.x1) * 0.5f; - float cy = (b.y0 + b.y1) * 0.5f; - float bw = b.x1 - b.x0; - float bh = b.y1 - b.y0; - boxes.push_back({cx, cy, bw, bh, 1}); // label 1 = negative - } +// CXBlock: depthwise conv + LayerNorm + pointwise MLP with residual scaling. +static struct ggml_tensor* sam3_cxblock_forward( + struct ggml_context* ctx, + struct ggml_tensor* x, // [D, H, W, B] + struct ggml_tensor* dw_w, // [7, 7, 1, D] depthwise + struct ggml_tensor* dw_b, // [D] + struct ggml_tensor* norm_w, // [D] + struct ggml_tensor* norm_b, // [D] + struct ggml_tensor* fc1_w, // [D, 1024] + struct ggml_tensor* fc1_b, // [1024] + struct ggml_tensor* fc2_w, // [1024, D] + struct ggml_tensor* fc2_b, // [D] + struct ggml_tensor* gamma) // [D] +{ + const int D = (int)x->ne[0]; + const int H = (int)x->ne[1]; + const int W = (int)x->ne[2]; - const int n_boxes = (int)boxes.size(); - const int N_geo = n_boxes + 1; + // ggml conv expects WHCB layout; internal feature maps are stored as CWHB. + auto* x_whcb = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3)); - // Read needed weights from GPU - std::vector box_proj_w_data(4 * D), box_proj_b_data(D); - std::vector type_embed_data(D * 2); - std::vector cls_data(D); - std::vector box_pos_proj_w_data(258 * D), box_pos_proj_b_data(D); - std::vector box_pool_proj_w_data(7 * 7 * D * D), box_pool_proj_b_data(D); - std::vector img_pre_norm_w_data(D), img_pre_norm_b_data(D); + // Depthwise conv (groups = D): use the direct depthwise path. + // ggml_conv_2d_dw_direct only supports f32 kernel — cast if needed. + auto* dw_w_f32 = (dw_w->type == GGML_TYPE_F32) ? dw_w : ggml_cast(ctx, dw_w, GGML_TYPE_F32); + auto* h = ggml_conv_2d_dw_direct(ctx, dw_w_f32, x_whcb, 1, 1, 3, 3, 1, 1); + h = ggml_add(ctx, h, ggml_reshape_4d(ctx, dw_b, 1, 1, D, 1)); + h = ggml_cont(ctx, ggml_permute(ctx, h, 1, 2, 0, 3)); - auto read_f32 = [](struct ggml_tensor* t, float* dst, size_t n) { - sam3_read_f32(t, dst, n); - }; + // LayerNorm2d + h = sam3_layer_norm_2d(ctx, h, norm_w, norm_b); - read_f32(ge.box_proj_w, box_proj_w_data.data(), 4 * D); - read_f32(ge.box_proj_b, box_proj_b_data.data(), D); - read_f32(ge.type_embed, type_embed_data.data(), D * 2); - read_f32(ge.cls_token, cls_data.data(), D); - read_f32(ge.box_pos_proj_w, box_pos_proj_w_data.data(), 258 * D); - read_f32(ge.box_pos_proj_b, box_pos_proj_b_data.data(), D); + // Pointwise MLP: reshape to [D, H*W, B], apply FC, reshape back + auto* flat = ggml_reshape_3d(ctx, h, D, H * W, 1); + flat = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, flat), fc1_b); + flat = ggml_gelu(ctx, flat); + flat = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, flat), fc2_b); + h = ggml_reshape_4d(ctx, flat, D, H, W, 1); - if (n_boxes > 0) { - read_f32(ge.box_pool_proj_w, box_pool_proj_w_data.data(), 7 * 7 * D * D); - read_f32(ge.box_pool_proj_b, box_pool_proj_b_data.data(), D); - read_f32(ge.img_pre_norm_w, img_pre_norm_w_data.data(), D); - read_f32(ge.img_pre_norm_b, img_pre_norm_b_data.data(), D); - } + // Residual with learnable scaling: x + gamma * h + auto* gamma_4d = ggml_reshape_4d(ctx, gamma, D, 1, 1, 1); + h = ggml_mul(ctx, h, gamma_4d); + return ggml_add(ctx, x, h); +} - // Output: [D * N_geo] in row-major [D, N_geo] ggml order - std::vector out(D * N_geo, 0.0f); +/***************************************************************************** +** Prompt building for memory attention (Phase 7) +*****************************************************************************/ - // Encode each box - for (int bi = 0; bi < n_boxes; ++bi) { - const auto& box = boxes[bi]; - float embed[256] = {}; +struct sam3_prompt_data { + std::vector prompt; // [MD * M_total] flat + std::vector prompt_pos; // [MD * M_total] flat + int M_total; // total prompt tokens + int num_obj_ptr_tokens; // pointer tokens at end (excluded from RoPE) + int M_spatial; // spatial memory tokens = M_total - num_obj_ptr_tokens +}; - // 1. Direct projection: Linear(4, D) - // box_proj_w: ggml [4, D] = PyTorch [D, 4] - // out = x @ W^T + b where x=[cx,cy,w,h] - { - float coords[4] = {box.cx, box.cy, box.w, box.h}; - for (int d = 0; d < D; ++d) { - float sum = box_proj_b_data[d]; - for (int j = 0; j < 4; ++j) - sum += coords[j] * box_proj_w_data[j + d * 4]; // ggml: [4, D] stride - embed[d] = sum; - } - } +// Build prompt and prompt_pos tensors CPU-side for memory attention. +// mem_slot_feats[i]: [MD * H * H] raw spatial features (ggml layout) +// mem_slot_pes[i]: [MD * H * H] sinusoidal spatial PE (ggml layout) +// spatial_tpos[i]: temporal position for slot i (0 = conditioning, >0 = non-cond t_pos) +// obj_ptrs[i]: [D] object pointer data +// ptr_tpos[i]: temporal position for pointer i (<0 = skip) +static sam3_prompt_data sam3_build_prompt_and_pos( + const sam3_model& model, + const std::vector>& mem_slot_feats, + const std::vector>& mem_slot_pes, + const std::vector& spatial_tpos, + const std::vector>& obj_ptrs, + const std::vector& ptr_tpos, + int eff_feat_size = 0) { + const auto& hp = model.hparams; + const int MD = hp.mem_out_dim; // 64 + const int D = hp.neck_dim; // 256 + const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); + // Per-slot token count: from actual data size (handles both HxH and perceiver 512) + const int HH = mem_slot_feats.empty() ? (H * H) + : (int)(mem_slot_feats[0].size() / MD); - // 2. ROI Align + Conv2d(D, D, 7) pool projection - if (img_feats_data) { - // Apply img_pre_norm (LayerNorm) to image features before pooling - // For simplicity, we compute LayerNorm per spatial position - // Actually, LayerNorm is applied to the [D] dimension at each spatial location - // But we need the full feature map, so let's normalize in-place copy - const int N_spatial = W_feat * H_feat; - std::vector normed_feats(D * N_spatial); + sam3_prompt_data pd; + pd.num_obj_ptr_tokens = 0; - for (int s = 0; s < N_spatial; ++s) { - // Compute mean and variance over D dimension - float mean = 0.0f; - for (int d = 0; d < D; ++d) - mean += img_feats_data[d + s * D]; - mean /= D; + // Read maskmem_tpos_enc from model (one tensor [MD, 1, 1, 7]) + std::vector tpos_all(MD * hp.num_maskmem); + if (model.mem_enc.tpos[0]) { + sam3_read_f32(model.mem_enc.tpos[0], tpos_all.data(), MD * hp.num_maskmem); + } - float var = 0.0f; - for (int d = 0; d < D; ++d) { - float diff = img_feats_data[d + s * D] - mean; - var += diff * diff; - } - var /= D; + // 1. Spatial memory tokens + for (size_t s = 0; s < mem_slot_feats.size(); ++s) { + const int slot_tokens = (int)(mem_slot_feats[s].size() / MD); + // Append spatial features + pd.prompt.insert(pd.prompt.end(), mem_slot_feats[s].begin(), mem_slot_feats[s].end()); - float inv_std = 1.0f / sqrtf(var + 1e-5f); - for (int d = 0; d < D; ++d) { - normed_feats[d + s * D] = - (img_feats_data[d + s * D] - mean) * inv_std * img_pre_norm_w_data[d] + img_pre_norm_b_data[d]; - } - } + // Build positional encoding: spatial PE + temporal PE + std::vector pos = mem_slot_pes[s]; // copy spatial PE + int tpos_idx = spatial_tpos[s]; + if (tpos_idx >= 0 && tpos_idx < hp.num_maskmem) { + int enc_idx = hp.num_maskmem - tpos_idx - 1; // Python: maskmem_tpos_enc[num_maskmem - tpos - 1] + for (int n = 0; n < slot_tokens; ++n) + for (int d = 0; d < MD; ++d) + pos[d + n * MD] += tpos_all[d + enc_idx * MD]; + } + pd.prompt_pos.insert(pd.prompt_pos.end(), pos.begin(), pos.end()); + } + pd.M_spatial = (int)mem_slot_feats.size() * HH; - // Convert CxCyWH [0,1] → XYXY in feature grid coordinates - float fx0 = (box.cx - box.w * 0.5f) * (float)W_feat; - float fy0 = (box.cy - box.h * 0.5f) * (float)H_feat; - float fx1 = (box.cx + box.w * 0.5f) * (float)W_feat; - float fy1 = (box.cy + box.h * 0.5f) * (float)H_feat; + // 2. Object pointer tokens (split each D=256 pointer into D/MD=4 tokens of MD=64) + const int split = D / MD; // 4 - // ROI Align - std::vector roi_data(D * roi_size * roi_size); - sam3_roi_align_single(normed_feats.data(), D, W_feat, H_feat, - fx0, fy0, fx1, fy1, roi_size, roi_data.data()); + // Read obj_ptr_tpos_proj weights for CPU-side matmul + std::vector tpos_w(D * MD), tpos_b(MD); + if (model.obj_ptr_tpos_w) { + sam3_read_f32(model.obj_ptr_tpos_w, tpos_w.data(), D * MD); + sam3_read_f32(model.obj_ptr_tpos_b, tpos_b.data(), MD); + } - // Conv2d(D, D, 7): kernel [7, 7, D, D] in ggml = [D_out, D_in, kH, kW] in PyTorch - // Since roi_size = 7 = kernel_size, output is [D, 1, 1] - // This is effectively a matrix multiply: out[d_out] = sum over (d_in, kh, kw) - for (int d_out = 0; d_out < D; ++d_out) { - float sum = box_pool_proj_b_data[d_out]; - for (int kh = 0; kh < roi_size; ++kh) { - for (int kw = 0; kw < roi_size; ++kw) { - for (int d_in = 0; d_in < D; ++d_in) { - // ggml weight layout: [kW=7, kH=7, D_in=256, D_out=256] - int w_idx = kw + kh * 7 + d_in * 7 * 7 + d_out * 7 * 7 * D; - // roi_data layout: [C, W, H] = [D_in, kW, kH] - int r_idx = d_in + kw * D + kh * D * roi_size; - sum += box_pool_proj_w_data[w_idx] * roi_data[r_idx]; - } - } - } - embed[d_out] += sum; - } - } + for (size_t p = 0; p < obj_ptrs.size(); ++p) { + int rel = ptr_tpos[p]; + if (rel < 0) continue; - // 3. Sinusoidal positional encoding + Linear(258, D) - { - float pos_enc[258]; - sam3_sine_encode_box(pos_enc, box.cx, box.cy, box.w, box.h, - num_pos_feats, 10000); + // 1D sine PE for temporal position (normalized by max_obj_ptrs - 1 = 15) + std::vector sine_pe(D); + sam3_get_1d_sine_pe(sine_pe.data(), (float)rel / 15.0f, D); - for (int d = 0; d < D; ++d) { - float sum = box_pos_proj_b_data[d]; - for (int j = 0; j < 258; ++j) - sum += pos_enc[j] * box_pos_proj_w_data[j + d * 258]; - embed[d] += sum; - } + // Project 256-dim sine PE → 64-dim via obj_ptr_tpos_proj (CPU matmul) + // W is [D=256, MD=64] in ggml; y[j] = sum_i W[i + j*D] * x[i] + b[j] + std::vector proj_pe(MD, 0.0f); + for (int j = 0; j < MD; ++j) { + float sum = tpos_b[j]; + for (int i = 0; i < D; ++i) + sum += tpos_w[i + j * D] * sine_pe[i]; + proj_pe[j] = sum; } - // 4. Label embedding - { - int label = box.label; - for (int d = 0; d < D; ++d) - embed[d] += type_embed_data[d + label * D]; + // Split pointer [256] → 4 tokens of [64] + // Python: ptr.view(1,1,4,64).permute(1,2,0,3).reshape(4,1,64) + const auto& ptr = obj_ptrs[p]; + for (int t = 0; t < split; ++t) { + for (int d = 0; d < MD; ++d) { + pd.prompt.push_back(ptr[t * MD + d]); + pd.prompt_pos.push_back(proj_pe[d]); + } + pd.num_obj_ptr_tokens++; } + } - // Store in output: ggml layout [D, N_geo] — embed goes at column bi - for (int d = 0; d < D; ++d) - out[d + bi * D] = embed[d]; - } - - // CLS token goes at position n_boxes (last) - for (int d = 0; d < D; ++d) - out[d + n_boxes * D] = cls_data[d]; - - // Apply final_proj (Linear(D,D)) + LayerNorm on CPU - std::vector proj_w(D * D), proj_b(D); - std::vector ln_w(D), ln_b(D); - read_f32(ge.post_proj_w, proj_w.data(), D * D); - read_f32(ge.post_proj_b, proj_b.data(), D); - read_f32(ge.norm_w, ln_w.data(), D); - read_f32(ge.norm_b, ln_b.data(), D); - - std::vector projected(D * N_geo); - for (int t = 0; t < N_geo; ++t) { - // Linear: y = W @ x + b - // W is stored in PyTorch row-major [D_out, D_in] → ggml [D_in, D_out] - // proj_w[d_in + d_out * D] = W_py[d_out, d_in] - for (int d_out = 0; d_out < D; ++d_out) { - float sum = proj_b[d_out]; - for (int d_in = 0; d_in < D; ++d_in) - sum += out[d_in + t * D] * proj_w[d_in + d_out * D]; - projected[d_out + t * D] = sum; - } - } - if (n_boxes > 0) { - fprintf(stderr, "%s: %d exemplar boxes encoded (%d pos, %d neg)\n", - __func__, n_boxes, - (int)params.pos_exemplars.size(), - (int)params.neg_exemplars.size()); - } - - // LayerNorm over D dimension for each token - for (int t = 0; t < N_geo; ++t) { - float mean = 0.0f; - for (int d = 0; d < D; ++d) - mean += projected[d + t * D]; - mean /= D; - - float var = 0.0f; - for (int d = 0; d < D; ++d) { - float diff = projected[d + t * D] - mean; - var += diff * diff; - } - var /= D; - - float inv_std = 1.0f / sqrtf(var + 1e-5f); - for (int d = 0; d < D; ++d) { - float normalized = (projected[d + t * D] - mean) * inv_std; - out[d + t * D] = normalized * ln_w[d] + ln_b[d]; - } - } - return out; + pd.M_total = pd.M_spatial + pd.num_obj_ptr_tokens; + return pd; } /***************************************************************************** -** Fusion encoder — graph building (6 layers) +** Memory attention (Phase 7, Step 7.2) *****************************************************************************/ -// Single fusion encoder layer. -// x: [D, N, B] image features (N=5184), prompt: [D, T, B] text/exemplar tokens, pos: [D, N, B] -// Returns: updated x [D, N, B] -static struct ggml_tensor* sam3_fenc_layer_forward( +// Build memory attention graph. +// curr_tokens: [D, N, 1] — current frame image tokens (flattened from 72x72 = 5184) +// src_pos: [D, N, 1] — sinusoidal PE for current tokens (pos_enc_at_input) +// prompt: [MD, M_total, 1] — concatenated memory features + split pointer tokens +// prompt_pos: [MD, M_total, 1] — positional encodings for prompt +// rope_freqs: [2, D/2, N] — axial RoPE frequencies for Q and self-attn K +// rope_k_freqs:[2, D/2, M_spatial] — repeated axial RoPE for cross-attn spatial K +// num_obj_ptr_tokens: number of pointer tokens at end of prompt (excluded from RoPE) +// Returns: conditioned tokens [D, N, 1] +static struct ggml_tensor* sam3_build_mem_attn_graph( struct ggml_context* ctx, - const sam3_fenc_layer& ly, - struct ggml_tensor* x, - struct ggml_tensor* prompt, - struct ggml_tensor* pos, - struct ggml_tensor* prompt_attn_bias, - int n_heads) { - // Self-attention: Q/K get positional encoding, V does not - { - auto* shortcut = x; - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); - auto* q_in = ggml_add(ctx, x_norm, pos); - auto* k_in = ggml_add(ctx, x_norm, pos); - - const int64_t D = x->ne[0]; + const sam3_model& model, + struct ggml_tensor* curr_tokens, // [D, N, 1] + struct ggml_tensor* src_pos, // [D, N, 1] + struct ggml_tensor* prompt, // [MD, M_total, 1] + struct ggml_tensor* prompt_pos, // [MD, M_total, 1] + struct ggml_tensor* rope_freqs, // [2, D/2, N] + struct ggml_tensor* rope_k_freqs, // [2, D/2, M_spatial] or nullptr + int num_obj_ptr_tokens) { + const auto& ma = model.mem_attn; + const int D = model.hparams.neck_dim; // 256 + const int N = (int)curr_tokens->ne[1]; + const int M_total = (int)prompt->ne[1]; + const int M_spatial = M_total - num_obj_ptr_tokens; - auto* q_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]); + // pos_enc_at_input: x = curr + 0.1 * src_pos + auto* x = ggml_add(ctx, curr_tokens, ggml_scale(ctx, src_pos, 0.1f)); + ggml_set_name(x, "phase7_mem_attn_input"); - auto* q_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float)); + for (int l = 0; l < (int)ma.layers.size(); ++l) { + const auto& ly = ma.layers[l]; - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_in), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, x_norm), v_b); + // ── Self-attention with RoPE ────────────────────────────────────── + { + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); + auto* q = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_q_w, x_norm), ly.sa_q_b); + auto* k = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_k_w, x_norm), ly.sa_k_b); + auto* v = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_v_w, x_norm), ly.sa_v_b); - const int64_t N = x->ne[1]; - const int64_t B = x->ne[2]; - const int64_t HD = D / n_heads; + // Apply RoPE to Q and K (single-head, [D, N, 1]) + q = sam3_apply_rope(ctx, q, rope_freqs); + k = sam3_apply_rope(ctx, k, rope_freqs); - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, N, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + // Single-head attention (256-dim) + q = ggml_reshape_4d(ctx, q, D, 1, N, 1); + k = ggml_reshape_4d(ctx, k, D, 1, N, 1); + v = ggml_reshape_4d(ctx, v, D, 1, N, 1); + q = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); + k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 2, 1, 3)); + v = ggml_permute(ctx, v, 0, 2, 1, 3); - float scale = 1.0f / sqrtf((float)HD); - auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); - sa_out = ggml_reshape_3d(ctx, sa_out, D, N, B); + float scale = 1.0f / sqrtf((float)D); + auto* sa_out = ggml_flash_attn_ext(ctx, q, k, v, nullptr, scale, 0.0f, 0.0f); + sa_out = ggml_reshape_3d(ctx, sa_out, D, N, 1); + sa_out = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_out_w, sa_out), ly.sa_out_b); + x = ggml_add(ctx, x, sa_out); + sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_sa", l); + } - sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); - sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); + // ── Cross-attention to memory (kv_dim=64→256) with RoPE ────────── + { + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); + auto* q = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_q_w, x_norm), ly.ca_q_b); - x = ggml_add(ctx, shortcut, sa_out); - } + // K: project from (prompt + prompt_pos) — pos enc added before projection + auto* kv_with_pos = ggml_add(ctx, prompt, prompt_pos); + auto* k = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_k_w, kv_with_pos), ly.ca_k_b); + // V: project from prompt only (no pos enc) + auto* v = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_v_w, prompt), ly.ca_v_b); - // Cross-attention: Q from image features, K/V from prompt tokens. - // ca_q_w stores fused [D, 3*D] weights split as Q-proj, K-proj, V-proj. - { - auto* shortcut = x; - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); - const int64_t D = x->ne[0]; + // Apply RoPE to Q + q = sam3_apply_rope(ctx, q, rope_freqs); - auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); + // Apply RoPE to spatial K only (exclude num_obj_ptr_tokens tail keys) + if (M_spatial > 0 && rope_k_freqs) { + auto* k_spatial = ggml_cont(ctx, ggml_view_3d(ctx, k, + D, M_spatial, 1, k->nb[1], k->nb[2], 0)); + k_spatial = sam3_apply_rope(ctx, k_spatial, rope_k_freqs); + if (num_obj_ptr_tokens > 0) { + auto* k_ptr = ggml_cont(ctx, ggml_view_3d(ctx, k, + D, num_obj_ptr_tokens, 1, k->nb[1], k->nb[2], + (size_t)M_spatial * k->nb[1])); + k = ggml_concat(ctx, k_spatial, k_ptr, 1); + } else { + k = k_spatial; + } + } - auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); + // Flash attention (single head) + q = ggml_reshape_4d(ctx, q, D, 1, N, 1); + k = ggml_reshape_4d(ctx, k, D, 1, M_total, 1); + v = ggml_reshape_4d(ctx, v, D, 1, M_total, 1); + q = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); + k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 2, 1, 3)); + v = ggml_permute(ctx, v, 0, 2, 1, 3); - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, x_norm), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, prompt), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, prompt), v_b); + float scale = 1.0f / sqrtf((float)D); + auto* ca_out = ggml_flash_attn_ext(ctx, q, k, v, nullptr, scale, 0.0f, 0.0f); + ca_out = ggml_reshape_3d(ctx, ca_out, D, N, 1); + ca_out = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_out_w, ca_out), ly.ca_out_b); + x = ggml_add(ctx, x, ca_out); + sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_ca", l); + } - const int64_t N_q = x->ne[1]; - const int64_t N_kv = prompt->ne[1]; - const int64_t B = x->ne[2]; - const int64_t HD = D / n_heads; + // ── FFN ─────────────────────────────────────────────────────────── + { + auto* x_norm = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); + auto* ffn = ggml_add(ctx, ggml_mul_mat(ctx, ly.ffn_fc1_w, x_norm), ly.ffn_fc1_b); + ffn = ggml_relu(ctx, ffn); + ffn = ggml_add(ctx, ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn), ly.ffn_fc2_b); + x = ggml_add(ctx, x, ffn); + sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_ffn", l); + } + } - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + // Final norm + auto* norm_w = model.tensors.at("mem_attn.norm.weight"); + auto* norm_b = model.tensors.at("mem_attn.norm.bias"); + x = sam3_layer_norm(ctx, x, norm_w, norm_b); + ggml_set_name(x, "phase7_mem_attn_output"); - auto* ca_mask = sam3_expand_token_attn_bias(ctx, prompt_attn_bias, N_q, n_heads, B); - float scale = 1.0f / sqrtf((float)HD); - auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, ca_mask, scale, 0.0f, 0.0f); - ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); + return x; // [D, N, 1] +} - ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); - ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); +/***************************************************************************** +** Object pointer extraction (Phase 7, Step 7.3) +*****************************************************************************/ - x = ggml_add(ctx, shortcut, ca_out); - } +// Extract object pointer from SAM output token via 3-layer MLP (CPU-side). +static void sam3_extract_obj_ptr_cpu( + const sam3_model& model, + const float* sam_token_data, // [D] + float obj_score, + float* out_ptr) // [D] +{ + const auto& hp = model.hparams; + const int D = hp.neck_dim; - { - auto* shortcut = x; - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); + // SAM2/EdgeTAM with fixed_no_obj_ptr: blend projected ptr with no_obj_ptr + // based on presence score λ. + // SAM3 / SAM2 without fixed_no_obj_ptr: binary threshold. + if ((hp.is_sam2() || hp.is_edgetam()) && hp.fixed_no_obj_ptr) { + // λ = (obj_score > 0) ? 1.0 : 0.0 (hard threshold, not sigmoid) + float lambda = (obj_score > 0.0f) ? 1.0f : 0.0f; - auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, x_norm); - ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); - ffn = ggml_relu(ctx, ffn); - ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); - ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); + // Project token through MLP + std::vector h(D), tmp(D); + std::copy(sam_token_data, sam_token_data + D, h.data()); + for (int j = 0; j < 3; ++j) { + auto* w = model.obj_ptr_proj_w[j]; + auto* b = model.obj_ptr_proj_b[j]; + int nel_w = (int)(w->ne[0] * w->ne[1]); + std::vector w_data(nel_w), b_data(D); + sam3_read_f32(w, w_data.data(), nel_w); + sam3_read_f32(b, b_data.data(), D); + for (int o = 0; o < D; ++o) { + float sum = b_data[o]; + for (int i = 0; i < D; ++i) sum += w_data[o * D + i] * h[i]; + tmp[o] = (j < 2) ? std::max(0.0f, sum) : sum; + } + std::swap(h, tmp); + } - x = ggml_add(ctx, shortcut, ffn); + // Blend: obj_ptr = λ * projected + (1-λ) * no_obj_ptr + std::vector no_ptr(D); + ggml_backend_tensor_get(model.no_obj_ptr, no_ptr.data(), 0, D * sizeof(float)); + for (int i = 0; i < D; ++i) + out_ptr[i] = lambda * h[i] + (1.0f - lambda) * no_ptr[i]; + return; } - return x; -} + // SAM3 / default path: binary threshold + if (obj_score <= 0.0f) { + ggml_backend_tensor_get(model.no_obj_ptr, out_ptr, 0, D * sizeof(float)); + return; + } -// Build full fusion encoder graph (6 layers). -// image_feats: [D, N, B] where N=5184 (72*72), D=256 -// prompt_tokens: [D, T, B] text/exemplar features -// pos_enc: [D, N, B] sinusoidal positional encoding for image features -// Returns: conditioned_features [D, N, B] -static struct ggml_tensor* sam3_build_fenc_graph( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* image_feats, - struct ggml_tensor* prompt_tokens, - struct ggml_tensor* pos_enc, - struct ggml_tensor* prompt_attn_bias = nullptr) { - const auto& hp = model.hparams; - auto* x = image_feats; + std::vector h(D), tmp(D); + std::copy(sam_token_data, sam_token_data + D, h.data()); - for (int i = 0; i < hp.fenc_layers; ++i) { - x = sam3_fenc_layer_forward(ctx, model.fenc.layers[i], x, prompt_tokens, - pos_enc, prompt_attn_bias, hp.fenc_heads); - sam3_name_tensorf(x, "fenc_layer%d_out", i); - } + for (int j = 0; j < 3; ++j) { + auto* w = model.obj_ptr_proj_w[j]; + auto* b = model.obj_ptr_proj_b[j]; - return x; + int nel_w = (int)(w->ne[0] * w->ne[1]); + std::vector w_data(nel_w); + sam3_read_f32(w, w_data.data(), nel_w); + + std::vector b_data(D); + sam3_read_f32(b, b_data.data(), D); + + for (int o = 0; o < D; ++o) { + float sum = b_data[o]; + for (int i = 0; i < D; ++i) { + sum += w_data[o * D + i] * h[i]; + } + tmp[o] = (j < 2) ? std::max(0.0f, sum) : sum; + } + std::swap(h, tmp); + } + std::copy(h.begin(), h.end(), out_ptr); } /***************************************************************************** -** DETR decoder — graph building (6 layers) +** Tracker infrastructure (Phase 7, Step 7.4) *****************************************************************************/ -// inverse_sigmoid: log(x / (1 - x)), clamped to avoid inf -// Python reference uses eps=1e-3: x1 = x.clamp(min=eps), x2 = (1-x).clamp(min=eps) -static struct ggml_tensor* sam3_inverse_sigmoid(struct ggml_context* ctx, struct ggml_tensor* x) { - // clamp x to [1e-3, 1-1e-3] to match Python eps=1e-3 - x = ggml_clamp(ctx, x, 1e-3f, 1.0f - 1e-3f); - // log(x / (1 - x)) = log(x) - log(1 - x) - auto* log_x = ggml_log(ctx, x); - // Compute (1 - x) as (-1)*x + 1. We use ggml_scale_bias which takes float - // scalars (no tensor allocation needed, safe in no_alloc contexts). - auto* one_minus = ggml_scale_bias(ctx, x, -1.0f, 1.0f); - auto* log_1mx = ggml_log(ctx, one_minus); - return ggml_sub(ctx, log_x, log_1mx); +// Select memory frames for propagation (most recent + evenly spaced). +static std::vector sam3_select_memory_frames( + const std::vector& bank, + int max_slots) { + if ((int)bank.size() <= max_slots) { + std::vector all(bank.size()); + for (int i = 0; i < (int)bank.size(); ++i) all[i] = i; + return all; + } + std::vector selected; + selected.push_back(0); + selected.push_back((int)bank.size() - 1); + int remaining = max_slots - 2; + if (remaining > 0) { + float step = (float)(bank.size() - 2) / (remaining + 1); + for (int i = 0; i < remaining; ++i) { + int idx = 1 + (int)((i + 1) * step); + idx = std::min(idx, (int)bank.size() - 2); + selected.push_back(idx); + } + } + std::sort(selected.begin(), selected.end()); + selected.erase(std::unique(selected.begin(), selected.end()), selected.end()); + return selected; } -// Box refinement MLP (3 layers: D→D→D→4 with ReLU) -static struct ggml_tensor* sam3_bbox_mlp(struct ggml_context* ctx, - struct ggml_tensor* x, - struct ggml_tensor* w[3], - struct ggml_tensor* b[3]) { - for (int j = 0; j < 3; ++j) { - x = ggml_mul_mat(ctx, w[j], x); - x = ggml_add(ctx, x, b[j]); - if (j < 2) x = ggml_relu(ctx, x); +// Compute mask IoU between two binary masks. +static float sam3_mask_iou(const uint8_t* a, const uint8_t* b, int n) { + int inter = 0, uni = 0; + for (int i = 0; i < n; ++i) { + bool va = a[i] > 127; + bool vb = b[i] > 127; + if (va && vb) ++inter; + if (va || vb) ++uni; } - return x; + return (uni > 0) ? (float)inter / uni : 0.0f; } -// Build sinusoidal positional embedding for 4D reference points in the ggml graph. -// ref_boxes: [4, NQ, B] — (cx, cy, w, h) after sigmoid, B=1 -// sine_dim_t: [1, 64] — pre-computed angle multipliers (2π / 10000^(2i/128)) -// Returns: [512, NQ, B] sinusoidal embedding matching Python gen_sineembed_for_position -static struct ggml_tensor* sam3_build_sine_pos_embed_4d( - struct ggml_context* ctx, - struct ggml_tensor* ref_boxes, // [4, NQ, B] - struct ggml_tensor* sine_dim_t) { // [1, 64] - const int64_t NQ = ref_boxes->ne[1]; - - // Python output order: [cy, cx, w, h] → coord indices from boxes [cx(0),cy(1),w(2),h(3)] - const int coord_order[4] = {1, 0, 2, 3}; - - struct ggml_tensor* coord_embeds[4]; - - for (int c = 0; c < 4; ++c) { - int ci = coord_order[c]; - // Extract one coordinate: view into ref_boxes [4, NQ, 1] at element ci - auto* coord = ggml_view_2d(ctx, ref_boxes, 1, NQ, - ref_boxes->nb[1], ci * sizeof(float)); // [1, NQ] - - // Outer product: angles[i, q] = dim_t[i] * coord[q] - // ggml_mul_mat(A=[1,64], B=[1,NQ]) = A^T @ B = [64,1]@[1,NQ] = [64, NQ] - auto* angles = ggml_mul_mat(ctx, sine_dim_t, coord); // [64, NQ] +/***************************************************************************** +** Post-processing: hole filling and sprinkle removal (Phase 7, Step 7.8) +*****************************************************************************/ - auto* sin_vals = ggml_sin(ctx, angles); // [64, NQ] - auto* cos_vals = ggml_cos(ctx, angles); // [64, NQ] +// Fill small holes in a binary mask using BFS connected components. +static void sam3_fill_holes(uint8_t* mask, int w, int h, int area_threshold) { + const int n = w * h; + std::vector labels(n, -1); + int next_label = 0; + std::vector component_sizes; - // Interleave: [sin_0, cos_0, sin_1, cos_1, ...] - auto* sin_r = ggml_reshape_3d(ctx, sin_vals, 1, 64, NQ); - auto* cos_r = ggml_reshape_3d(ctx, cos_vals, 1, 64, NQ); - auto* interleaved = ggml_concat(ctx, sin_r, cos_r, 0); // [2, 64, NQ] - coord_embeds[c] = ggml_reshape_2d(ctx, interleaved, 128, NQ); + for (int i = 0; i < n; ++i) { + if (mask[i] > 127 || labels[i] >= 0) continue; + int label = next_label++; + component_sizes.push_back(0); + std::vector queue; + queue.push_back(i); + labels[i] = label; + int head = 0; + bool touches_border = false; + while (head < (int)queue.size()) { + int p = queue[head++]; + component_sizes[label]++; + int px = p % w, py = p / w; + if (px == 0 || px == w - 1 || py == 0 || py == h - 1) touches_border = true; + int dx[] = {-1, 1, 0, 0}; + int dy[] = {0, 0, -1, 1}; + for (int d = 0; d < 4; ++d) { + int nx2 = px + dx[d], ny2 = py + dy[d]; + if (nx2 < 0 || nx2 >= w || ny2 < 0 || ny2 >= h) continue; + int ni = ny2 * w + nx2; + if (mask[ni] <= 127 && labels[ni] < 0) { + labels[ni] = label; + queue.push_back(ni); + } + } + } + if (touches_border) component_sizes[label] = area_threshold + 1; + } + for (int i = 0; i < n; ++i) { + if (mask[i] <= 127 && labels[i] >= 0 && component_sizes[labels[i]] <= area_threshold) { + mask[i] = 255; + } } - - // Concatenate all 4 coordinates → [512, NQ] - auto* embed = ggml_concat(ctx, coord_embeds[0], coord_embeds[1], 0); // [256, NQ] - embed = ggml_concat(ctx, embed, coord_embeds[2], 0); // [384, NQ] - embed = ggml_concat(ctx, embed, coord_embeds[3], 0); // [512, NQ] - - return embed; } -// Build query positional encoding from reference boxes via sine embed + ref_point_head MLP. -// ref_boxes: [4, NQ, 1] — after sigmoid -// sine_dim_t: [1, 64] -// Returns: [D, NQ+1, 1] (zeros for presence token at index 0, MLP output for object queries) -static struct ggml_tensor* sam3_build_query_pos( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* ref_boxes, // [4, NQ, 1] - struct ggml_tensor* sine_dim_t, // [1, 64] - int layer_idx = -1) { - const auto& tensors = model.tensors; - const int64_t NQ = ref_boxes->ne[1]; - const int D = model.hparams.neck_dim; // 256 - - // 1. Sine positional embedding: [512, NQ] - auto* sine_embed = sam3_build_sine_pos_embed_4d(ctx, ref_boxes, sine_dim_t); - if (layer_idx == 0) { - ggml_set_name(sine_embed, "ddec_query_sine_0"); - } +// Remove small foreground sprinkles. +static void sam3_remove_sprinkles(uint8_t* mask, int w, int h, int area_threshold) { + const int n = w * h; + std::vector labels(n, -1); + int next_label = 0; + std::vector component_sizes; - // 2. ref_point_head MLP: 512 → 256 → 256 - // Layer 0: relu(W0 @ sine_embed + b0) - auto* h = ggml_mul_mat(ctx, tensors.at("ddec.ref_point_head.layers.0.weight"), sine_embed); - h = ggml_add(ctx, h, tensors.at("ddec.ref_point_head.layers.0.bias")); - h = ggml_relu(ctx, h); - // Layer 1: W1 @ h + b1 (no activation) - auto* qpos_obj = ggml_mul_mat(ctx, tensors.at("ddec.ref_point_head.layers.1.weight"), h); - qpos_obj = ggml_add(ctx, qpos_obj, tensors.at("ddec.ref_point_head.layers.1.bias")); - // qpos_obj: [D, NQ] - if (layer_idx == 0) { - ggml_set_name(qpos_obj, "ddec_query_pos_0"); + for (int i = 0; i < n; ++i) { + if (mask[i] <= 127 || labels[i] >= 0) continue; + int label = next_label++; + component_sizes.push_back(0); + std::vector queue; + queue.push_back(i); + labels[i] = label; + int head = 0; + while (head < (int)queue.size()) { + int p = queue[head++]; + component_sizes[label]++; + int px = p % w, py = p / w; + int dx[] = {-1, 1, 0, 0}; + int dy[] = {0, 0, -1, 1}; + for (int d = 0; d < 4; ++d) { + int nx2 = px + dx[d], ny2 = py + dy[d]; + if (nx2 < 0 || nx2 >= w || ny2 < 0 || ny2 >= h) continue; + int ni = ny2 * w + nx2; + if (mask[ni] > 127 && labels[ni] < 0) { + labels[ni] = label; + queue.push_back(ni); + } + } + } + } + for (int i = 0; i < n; ++i) { + if (mask[i] > 127 && labels[i] >= 0 && component_sizes[labels[i]] <= area_threshold) { + mask[i] = 0; + } } - - // 3. Reshape to 3D and prepend zeros for presence token - qpos_obj = ggml_reshape_3d(ctx, qpos_obj, D, NQ, 1); - auto* qpos_pres = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, 1, 1); - ggml_set_name(qpos_pres, "ddec_query_pos_pres"); - ggml_set_input(qpos_pres); // zeros — set by caller - - return ggml_concat(ctx, qpos_pres, qpos_obj, 1); // [D, NQ+1, 1] } -// Build box-relative positional bias for DETR cross-attention. -// ref_boxes: [4, N_q, B] — (cx, cy, w, h) in [0,1] -// rpb_coords: [feat_hw] — normalized coords [0/H, 1/H, ..., (H-1)/H] (input tensor) -// Returns: bias tensor [N_kv, N_q+1, n_heads, B] for ggml_flash_attn_ext mask -// -// Python _get_rpb_matrix with boxRPB="log": -// 1. boxes → xyxy -// 2. deltas_x[q,w,:2] = [coord_w - x0, coord_w - x1] -// 3. deltas_y[q,h,:2] = [coord_h - y0, coord_h - y1] -// 4. log transform: sign(d*8) * log2(|d*8|+1) / log2(8) -// 5. MLP: [2] → [256] → [n_heads] -// 6. outer sum: B[h,w] = delta_y[h] + delta_x[w] -static struct ggml_tensor* sam3_compute_box_rpb( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* ref_boxes, // [4, N_q, B] - struct ggml_tensor* rpb_coords, // [feat_hw] — pre-filled grid coordinates - int feat_hw, - int layer_idx = -1) { - const int64_t NQ = ref_boxes->ne[1]; - const int NH = model.hparams.ddec_heads; // 8 - const int W = feat_hw; - const int H = feat_hw; - const auto& tensors = model.tensors; +// Resolve overlapping masks: higher-scoring instances take priority. +static void sam3_resolve_overlaps(std::vector& dets) { + if (dets.size() <= 1) return; + const int w = dets[0].mask.width; + const int h = dets[0].mask.height; + if (w == 0 || h == 0) return; + std::sort(dets.begin(), dets.end(), [](const sam3_detection& a, const sam3_detection& b) { + return a.score > b.score; + }); + const int n = w * h; + for (int i = 0; i < n; ++i) { + bool claimed = false; + for (auto& det : dets) { + if (det.mask.data.empty()) continue; + if (claimed) { + det.mask.data[i] = 0; + } else if (det.mask.data[i] > 127) { + claimed = true; + } + } + } +} - // ── 1. Convert cxcywh → xyxy ───────────────────────────────────────── - // ggml_view_2d on strided data is non-contiguous — ggml_scale requires contiguous. - // Use ggml_cont to make each coordinate slice contiguous. - auto* cx = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 0)); - auto* cy = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 1 * sizeof(float))); - auto* bw = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 2 * sizeof(float))); - auto* bh = ggml_cont(ctx, ggml_view_2d(ctx, ref_boxes, 1, NQ, ref_boxes->nb[1], 3 * sizeof(float))); - // x0 = cx - w/2, x1 = cx + w/2 - auto* half_w = ggml_scale(ctx, bw, 0.5f); - auto* half_h = ggml_scale(ctx, bh, 0.5f); - auto* x0 = ggml_sub(ctx, cx, half_w); // [1, NQ] - auto* x1 = ggml_add(ctx, cx, half_w); - auto* y0 = ggml_sub(ctx, cy, half_h); - auto* y1 = ggml_add(ctx, cy, half_h); +/***************************************************************************** +** Post-processing: NMS, bilinear interpolation, mask binarization +*****************************************************************************/ - // ── 2. Compute deltas via outer subtract ────────────────────────────── - // coords: [W] → reshape to [W, 1] for outer subtract - auto* cw = ggml_reshape_2d(ctx, rpb_coords, W, 1); // [W, 1] +// Compute IoU between two boxes [x0, y0, x1, y1]. +static float sam3_box_iou(const sam3_box& a, const sam3_box& b) { + float x0 = std::max(a.x0, b.x0); + float y0 = std::max(a.y0, b.y0); + float x1 = std::min(a.x1, b.x1); + float y1 = std::min(a.y1, b.y1); + float inter = std::max(0.0f, x1 - x0) * std::max(0.0f, y1 - y0); + float area_a = (a.x1 - a.x0) * (a.y1 - a.y0); + float area_b = (b.x1 - b.x0) * (b.y1 - b.y0); + float uni = area_a + area_b - inter; + return (uni > 0.0f) ? inter / uni : 0.0f; +} - // Outer subtract: delta[w, q] = coord[w] - edge[q] - // Use ggml_mul_mat trick: not applicable for subtraction. - // Instead: repeat coords to [W, NQ], repeat edge to [W, NQ], subtract. - auto* shape_wn = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, W, NQ); +// Non-maximum suppression on detections, sorted by score descending. +// Returns indices of kept detections. +static std::vector sam3_nms(const std::vector& dets, float iou_thresh) { + // Sort indices by score descending + std::vector indices(dets.size()); + for (int i = 0; i < (int)dets.size(); ++i) indices[i] = i; + std::sort(indices.begin(), indices.end(), [&](int a, int b) { + return dets[a].score > dets[b].score; + }); - auto* cw_rep = ggml_repeat(ctx, cw, shape_wn); // [W, NQ] (each column = coords) - auto* x0_t = ggml_cont(ctx, ggml_transpose(ctx, x0)); // [NQ, 1] - auto* x0_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, x0_t, 1, NQ), shape_wn); - auto* x1_t = ggml_cont(ctx, ggml_transpose(ctx, x1)); - auto* x1_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, x1_t, 1, NQ), shape_wn); - auto* y0_t = ggml_cont(ctx, ggml_transpose(ctx, y0)); - auto* y0_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, y0_t, 1, NQ), shape_wn); - auto* y1_t = ggml_cont(ctx, ggml_transpose(ctx, y1)); - auto* y1_rep = ggml_repeat(ctx, ggml_reshape_2d(ctx, y1_t, 1, NQ), shape_wn); + std::vector suppressed(dets.size(), false); + std::vector keep; - auto* dx0 = ggml_sub(ctx, cw_rep, x0_rep); // [W, NQ] - auto* dx1 = ggml_sub(ctx, cw_rep, x1_rep); - auto* dy0 = ggml_sub(ctx, cw_rep, y0_rep); // reusing coords for H (H==W) - auto* dy1 = ggml_sub(ctx, cw_rep, y1_rep); + for (int idx : indices) { + if (suppressed[idx]) continue; + keep.push_back(idx); + for (int j : indices) { + if (suppressed[j] || j == idx) continue; + if (sam3_box_iou(dets[idx].box, dets[j].box) > iou_thresh) { + suppressed[j] = true; + } + } + } - // Stack into [2, W, NQ]: reshape each to [1, W, NQ], concat dim 0 - auto* dx0_r = ggml_reshape_3d(ctx, dx0, 1, W, NQ); - auto* dx1_r = ggml_reshape_3d(ctx, dx1, 1, W, NQ); - auto* deltas_x = ggml_concat(ctx, dx0_r, dx1_r, 0); // [2, W, NQ] - auto* dy0_r = ggml_reshape_3d(ctx, dy0, 1, H, NQ); - auto* dy1_r = ggml_reshape_3d(ctx, dy1, 1, H, NQ); - auto* deltas_y = ggml_concat(ctx, dy0_r, dy1_r, 0); // [2, H, NQ] + return keep; +} - // ── 3. Log transform: sign(d*8) * log2(|d*8|+1) / log2(8) ──────────── - const float scale8 = 8.0f; - const float inv_log2_8 = 1.0f / log2f(8.0f); // = 1/3 +// Bilinear interpolation of a flat mask [H_in * W_in] to [H_out * W_out]. +// Uses double for coordinate math to match PyTorch F.interpolate precision. +static std::vector sam3_bilinear_interpolate(const float* src, int src_w, int src_h, + int dst_w, int dst_h) { + std::vector dst(dst_w * dst_h); + const double sx = (double)src_w / dst_w; + const double sy = (double)src_h / dst_h; - auto rpb_log = [&](struct ggml_tensor* d) -> struct ggml_tensor* { - auto* d8 = ggml_scale(ctx, d, scale8); - auto* sign_d = ggml_sgn(ctx, d8); - auto* abs_d = ggml_abs(ctx, d8); - auto* log_val = ggml_log(ctx, ggml_scale_bias(ctx, abs_d, 1.0f, 1.0f)); - // log2(x) = ln(x) / ln(2) - log_val = ggml_scale(ctx, log_val, 1.0f / logf(2.0f)); - return ggml_mul(ctx, sign_d, ggml_scale(ctx, log_val, inv_log2_8)); - }; + for (int y = 0; y < dst_h; ++y) { + double fy = (y + 0.5) * sy - 0.5; + fy = std::max(0.0, std::min(fy, (double)(src_h - 1))); + const int y0 = std::min((int)fy, src_h - 2); + const int y1 = y0 + 1; + const float wy = (float)(fy - y0); - deltas_x = rpb_log(deltas_x); // [2, W, NQ] - deltas_y = rpb_log(deltas_y); // [2, H, NQ] + for (int x = 0; x < dst_w; ++x) { + double fx = (x + 0.5) * sx - 0.5; + fx = std::max(0.0, std::min(fx, (double)(src_w - 1))); + const int x0 = std::min((int)fx, src_w - 2); + const int x1 = x0 + 1; + const float wx = (float)(fx - x0); - // ── 4. MLP: [2, W*NQ] → [NH, W*NQ] ─────────────────────────────────── - // boxRPB_embed_x: MLP(2, 256, 8, 2) = Linear(2→256)+ReLU, Linear(256→8) - // Reshape to [2, W*NQ] so matmul treats each (w, q) pair as a sample - auto rpb_mlp = [&](struct ggml_tensor* d, const char* axis) -> struct ggml_tensor* { - int64_t spatial = d->ne[1]; - int64_t nq = d->ne[2]; - auto* flat = ggml_reshape_2d(ctx, d, 2, spatial * nq); // [2, W*NQ] - auto wn0 = std::string("ddec.boxRPB_embed_") + axis + ".layers.0.weight"; - auto bn0 = std::string("ddec.boxRPB_embed_") + axis + ".layers.0.bias"; - auto wn1 = std::string("ddec.boxRPB_embed_") + axis + ".layers.1.weight"; - auto bn1 = std::string("ddec.boxRPB_embed_") + axis + ".layers.1.bias"; - flat = ggml_mul_mat(ctx, tensors.at(wn0), flat); - flat = ggml_add(ctx, flat, tensors.at(bn0)); - flat = ggml_relu(ctx, flat); - flat = ggml_mul_mat(ctx, tensors.at(wn1), flat); - flat = ggml_add(ctx, flat, tensors.at(bn1)); - // flat: [NH, W*NQ] → reshape to [NH, spatial, NQ] - return ggml_reshape_3d(ctx, flat, NH, spatial, nq); - }; + float v = (1 - wy) * ((1 - wx) * src[y0 * src_w + x0] + wx * src[y0 * src_w + x1]) + wy * ((1 - wx) * src[y1 * src_w + x0] + wx * src[y1 * src_w + x1]); + dst[y * dst_w + x] = v; + } + } + return dst; +} - auto* rpb_x = rpb_mlp(deltas_x, "x"); // [NH, W, NQ] - auto* rpb_y = rpb_mlp(deltas_y, "y"); // [NH, H, NQ] +// Convert (cx, cy, w, h) in [0,1] to (x0, y0, x1, y1) in pixel coordinates. +static sam3_box sam3_cxcywh_to_xyxy(float cx, float cy, float w, float h, + int img_w, int img_h) { + sam3_box box; + box.x0 = (cx - w * 0.5f) * img_w; + box.y0 = (cy - h * 0.5f) * img_h; + box.x1 = (cx + w * 0.5f) * img_w; + box.y1 = (cy + h * 0.5f) * img_h; + // Clamp to image bounds + box.x0 = std::max(0.0f, std::min(box.x0, (float)img_w)); + box.y0 = std::max(0.0f, std::min(box.y0, (float)img_h)); + box.x1 = std::max(0.0f, std::min(box.x1, (float)img_w)); + box.y1 = std::max(0.0f, std::min(box.y1, (float)img_h)); + return box; +} - // ── 5. Outer sum: B[nh, w, h, q] = rpb_y[nh, h, q] + rpb_x[nh, w, q] ─ - // Reshape for broadcasting: - // rpb_y → [NH, 1, H, NQ] - // rpb_x → [NH, W, 1, NQ] - // - // Keep W in ne[1] so reshaping [NH, W, H, NQ] → [NH, H*W, NQ, 1] - // preserves Python's flatten(H, W) order where W is the fast spatial axis. - auto* rpb_y_4d = ggml_reshape_4d(ctx, rpb_y, NH, 1, H, NQ); - auto* rpb_x_4d = ggml_reshape_4d(ctx, rpb_x, NH, W, 1, NQ); +/***************************************************************************** +** Image segmentation — PCS (text-prompted) +*****************************************************************************/ - // ggml_add broadcasts: where one dim is 1, the other is used - auto* rpb_hw = ggml_repeat(ctx, rpb_y_4d, - ggml_new_tensor_4d(ctx, GGML_TYPE_F32, NH, W, H, NQ)); - auto* rpb_hw_x = ggml_repeat(ctx, rpb_x_4d, - ggml_new_tensor_4d(ctx, GGML_TYPE_F32, NH, W, H, NQ)); - auto* rpb = ggml_add(ctx, rpb_hw, rpb_hw_x); // [NH, W, H, NQ] - - // ── 6. Reshape to [H*W, NQ, NH, 1] for flash_attn_ext mask ─────────── - // Current: [NH, W, H, NQ]. Need: [N_kv=H*W, NQ, NH, B=1] - rpb = ggml_reshape_4d(ctx, rpb, NH, H * W, NQ, 1); - rpb = ggml_cont(ctx, ggml_permute(ctx, rpb, 2, 0, 1, 3)); // [H*W, NQ, NH, 1] +sam3_result sam3_segment_pcs(sam3_state& state, + const sam3_model& model, + const sam3_pcs_params& params) { + if (model.hparams.visual_only || model.hparams.is_sam2()) { + fprintf(stderr, "%s: ERROR: PCS not available on %s model\n", + __func__, model.hparams.is_sam2() ? "SAM2" : "visual-only"); + return sam3_result{}; + } - // Prepend zeros for presence token: mask for presence token has no box-relative bias - auto* pres_mask = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, H * W, 1, NH, 1); - ggml_set_name(pres_mask, "rpb_pres_zeros"); - ggml_set_input(pres_mask); // zeros — set by caller +#if SAM3_LOG_LEVEL >= 1 + auto t_start = std::chrono::high_resolution_clock::now(); +#endif + const auto& hp = model.hparams; + const int D = hp.neck_dim; // 256 + const int H = hp.n_img_embd(); // 72 + const int L = hp.text_ctx_len; // 32 + const int NQ = hp.ddec_num_queries; // 200 + const int N_spatial = H * H; // 5184 + sam3_result result; - // [H*W, NQ+1, NH, 1] - auto* full_rpb = ggml_concat(ctx, pres_mask, rpb, 1); - full_rpb = ggml_cont(ctx, full_rpb); - if (layer_idx == 0) { - ggml_set_name(full_rpb, "ddec_rpb_mask_0"); + // ── Check that image has been encoded ──────────────────────────────── + if (!state.neck_det[0]) { + fprintf(stderr, "%s: image not encoded — call sam3_encode_image first\n", __func__); + return result; } - return full_rpb; -} - -// Single DETR decoder layer. -// queries: [D, N_q, B] where N_q = 201 (200 object queries + 1 presence token) -// query_pos: [D, N_q, B] positional encoding for queries -// enc_feats: [D, N_kv, B] conditioned image features from fusion encoder -// enc_pos: [D, N_kv, B] positional encoding for image features -// text_feats: [D, T, B] text features -// rpb_mask: [N_kv, N_q, n_heads, B] box-relative positional bias (or nullptr) -// Returns: updated queries [D, N_q, B] -static struct ggml_tensor* sam3_ddec_layer_forward( - struct ggml_context* ctx, - const sam3_ddec_layer& ly, - struct ggml_tensor* queries, - struct ggml_tensor* query_pos, - struct ggml_tensor* enc_feats, - struct ggml_tensor* enc_pos, - struct ggml_tensor* text_feats, - int n_heads, - struct ggml_tensor* text_attn_bias = nullptr, - struct ggml_tensor* rpb_mask = nullptr, - int layer_idx = -1) { - const int64_t D = queries->ne[0]; + // ── Tokenize text prompt ───────────────────────────────────────────── + auto token_ids = sam3_tokenize(const_cast(model.tokenizer), + params.text_prompt, L); + if (token_ids.empty()) { + fprintf(stderr, "%s: failed to tokenize text prompt\n", __func__); + return result; + } - // Python decoder layer order (all post-norm): - // 1. Self-attention → norm2 (post-norm) - // 2. Text cross-attention (ca_text) → catext_norm (post-norm) - // 3. Image cross-attention (cross_attn) → norm1 (post-norm) - // 4. FFN → norm3 (post-norm) - // - // Norm weight mapping: - // ly.norm2_w = ".norm2.weight" = Python norm2 (post-SA) - // ly.norm3_w = ".norm_ca_text.weight" = Python catext_norm (post-text-CA) - // ly.norm1_w = ".norm1.weight" = Python norm1 (post-image-CA) - // ly.norm4_w = ".norm3.weight" = Python norm3 (post-FFN) + SAM3_LOG(2, "%s: text='%s', %zu tokens\n", __func__, + params.text_prompt.c_str(), token_ids.size()); - // 1. Self-attention among queries (post-norm) - { - // Q = K = queries + query_pos, V = queries (no pos) - auto* q_in = ggml_add(ctx, queries, query_pos); + // ── Helper: run a sub-graph with its own context and allocator ────── + // Each stage below follows this exact pattern: + // 1. Create fresh ggml_context + graph + allocator + // 2. Create INPUT tensors (ggml_set_input) + // 3. Build the computation graph + // 4. Mark outputs (ggml_set_output) + // 5. Allocate → set input data → compute → read output data + // 6. Free allocator + context + // This ensures ZERO buffer sharing between stages. - auto* q_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], D * ly.sa_in_proj_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.sa_in_proj_w, D, D, ly.sa_in_proj_w->nb[1], 2 * D * ly.sa_in_proj_w->nb[1]); - auto* q_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.sa_in_proj_b, D, 2 * D * sizeof(float)); + // ── Pre-compute shared CPU data used by multiple stages ───────────── + const int n_boxes = (int)(params.pos_exemplars.size() + params.neg_exemplars.size()); + const int N_geo = n_boxes + 1; // +1 for CLS + const int T = L + N_geo; // total prompt tokens (text + geometry) - const int64_t N = queries->ne[1]; - const int64_t B = queries->ne[2]; - const int64_t HD = D / n_heads; + // Read image features and PE from state into CPU buffers (used by multiple stages) + std::vector img_feats_cpu(D * N_spatial); + std::vector img_pe_cpu(D * N_spatial); + ggml_backend_tensor_get(state.neck_det[2], img_feats_cpu.data(), 0, + D * N_spatial * sizeof(float)); + ggml_backend_tensor_get(state.neck_det_pe[2], img_pe_cpu.data(), 0, + D * N_spatial * sizeof(float)); - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, q_in), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, queries), v_b); + // Pre-compute constant input vectors + std::vector sine_dim_t_cpu(64); + for (int i = 0; i < 64; ++i) + sine_dim_t_cpu[i] = 2.0f * (float)M_PI / powf(10000.0f, 2.0f * (float)i / 128.0f); - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, N, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + std::vector rpb_coords_cpu(H); + for (int i = 0; i < H; ++i) rpb_coords_cpu[i] = (float)i / (float)H; - float scale = 1.0f / sqrtf((float)HD); - auto* sa_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); - sa_out = ggml_reshape_3d(ctx, sa_out, D, N, B); - sa_out = ggml_mul_mat(ctx, ly.sa_out_proj_w, sa_out); - sa_out = ggml_add(ctx, sa_out, ly.sa_out_proj_b); + // Build combined attention bias for the prompt: [T] = [L + N_geo] + // 0.0 for valid tokens, -1e9 for padding text tokens, 0.0 for geo tokens + std::vector combined_bias_cpu(T); + for (int i = 0; i < L; ++i) + combined_bias_cpu[i] = (token_ids[i] != 0) ? 0.0f : -1.0e9f; + for (int i = 0; i < N_geo; ++i) + combined_bias_cpu[L + i] = 0.0f; - queries = ggml_add(ctx, queries, sa_out); - // Post-norm: norm2 (Python's post-SA norm) - queries = sam3_layer_norm(ctx, queries, ly.norm2_w, ly.norm2_b); - if (layer_idx == 0) { - ggml_set_name(queries, "ddec_layer0_after_sa"); - } - } + // Build text validity mask for DotProductScoring: [T] + int n_valid_tokens = 0; + for (int i = 0; i < L; ++i) + if (token_ids[i] != 0) ++n_valid_tokens; + n_valid_tokens += N_geo; + if (n_valid_tokens == 0) n_valid_tokens = 1; + float tvm_scale = (float)T / (float)n_valid_tokens; + std::vector text_valid_mask_cpu(T); + for (int i = 0; i < L; ++i) + text_valid_mask_cpu[i] = (token_ids[i] != 0) ? tvm_scale : 0.0f; + for (int i = 0; i < N_geo; ++i) + text_valid_mask_cpu[L + i] = tvm_scale; - // 2. Cross-attention to text tokens (post-norm) + /* + ** ── SUB-GRAPH 1: Text Encoder ──────────────────────────────────── + */ + std::vector text_feats_cpu(D * L); { - // Q = queries + query_pos, K = V = text_feats - auto* q_in = ggml_add(ctx, queries, query_pos); + const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; + struct ggml_init_params gp = {sz, nullptr, true}; + auto* ctx = ggml_init(gp); - auto* q_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], D * ly.ca_text_q_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.ca_text_q_w, D, D, ly.ca_text_q_w->nb[1], 2 * D * ly.ca_text_q_w->nb[1]); - auto* q_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.ca_text_q_b, D, 2 * D * sizeof(float)); + auto* inp = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, L); + ggml_set_name(inp, "text_token_ids"); + ggml_set_input(inp); - const int64_t N_q = queries->ne[1]; - const int64_t N_kv = text_feats->ne[1]; - const int64_t B = queries->ne[2]; - const int64_t HD = D / n_heads; + auto* out = sam3_build_text_encoder_graph(ctx, inp, model); + ggml_set_output(out); - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, text_feats), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, text_feats), v_b); + auto* graph = ggml_new_graph_custom(ctx, 16384, false); + ggml_build_forward_expand(graph, out); - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + auto* causal = ggml_get_tensor(ctx, "causal_mask"); + auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { + fprintf(stderr, "%s: text encoder alloc failed\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } - auto* text_mask = sam3_expand_token_attn_bias(ctx, text_attn_bias, N_q, n_heads, B); - float scale = 1.0f / sqrtf((float)HD); - auto* ca_out = ggml_flash_attn_ext(ctx, Q, K, V, text_mask, scale, 0.0f, 0.0f); - ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); - ca_out = ggml_mul_mat(ctx, ly.ca_text_out_w, ca_out); - ca_out = ggml_add(ctx, ca_out, ly.ca_text_out_b); + if (inp->buffer) { + ggml_backend_tensor_set(inp, token_ids.data(), 0, L * sizeof(int32_t)); + } else { + fprintf(stderr, "%s: ERROR: text encoder input tensor has no buffer!\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } + if (causal && causal->buffer) { + std::vector cm(L * L); + sam3_fill_causal_mask(cm.data(), L); + ggml_backend_tensor_set(causal, cm.data(), 0, L * L * sizeof(ggml_fp16_t)); + } - queries = ggml_add(ctx, queries, ca_out); - // Post-norm: catext_norm (Python's post-text-CA norm) - queries = sam3_layer_norm(ctx, queries, ly.norm3_w, ly.norm3_b); - if (layer_idx == 0) { - ggml_set_name(queries, "ddec_layer0_after_text_ca"); + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; } + ggml_backend_tensor_get(out, text_feats_cpu.data(), 0, D * L * sizeof(float)); + + ggml_gallocr_free(alloc); + ggml_free(ctx); } - // 3. Cross-attention to conditioned image features (post-norm) + SAM3_LOG(2, "%s: text encoder done\n", __func__); + + /* + ** ── SUB-GRAPH 2: Geometry Encoder ──────────────────────────────── + */ + std::vector geo_feats_cpu(D * N_geo); { - // Q = queries + query_pos, K = enc_feats + enc_pos, V = enc_feats - auto* q_in = ggml_add(ctx, queries, query_pos); - auto* k_in = ggml_add(ctx, enc_feats, enc_pos); + const size_t sz = ggml_tensor_overhead() * 4096 + ggml_graph_overhead() * 2; + struct ggml_init_params gp = {sz, nullptr, true}; + auto* ctx = ggml_init(gp); - auto* q_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 0); - auto* k_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], D * ly.ca_q_w->nb[1]); - auto* v_w = ggml_view_2d(ctx, ly.ca_q_w, D, D, ly.ca_q_w->nb[1], 2 * D * ly.ca_q_w->nb[1]); - auto* q_b = ggml_view_1d(ctx, ly.ca_q_b, D, 0); - auto* k_b = ggml_view_1d(ctx, ly.ca_q_b, D, D * sizeof(float)); - auto* v_b = ggml_view_1d(ctx, ly.ca_q_b, D, 2 * D * sizeof(float)); - - const int64_t N_q = queries->ne[1]; - const int64_t N_kv = enc_feats->ne[1]; - const int64_t B = queries->ne[2]; - const int64_t HD = D / n_heads; - - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, q_w, q_in), q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, k_w, k_in), k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, v_w, enc_feats), v_b); - - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); - V = ggml_permute(ctx, V, 0, 2, 1, 3); + auto* g_img = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(g_img, "geo_img"); + ggml_set_input(g_img); + auto* g_pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(g_pe, "geo_pe"); + ggml_set_input(g_pe); - float scale = 1.0f / sqrtf((float)HD); - struct ggml_tensor* ca_out = nullptr; + auto gr = sam3_build_geom_enc_graph(ctx, model, params, g_img, g_pe); + ggml_set_output(gr.geo_feats); - if (rpb_mask) { - // Keep the box-relative positional bias in fp32. The CPU flash-attn - // kernel reads mask storage as fp16, which is good enough for token - // padding masks but introduces avoidable drift here. - auto* kq = ggml_mul_mat(ctx, K, Q); // [N_kv, N_q, NH, B] - kq = ggml_soft_max_ext(ctx, kq, rpb_mask, scale, 0.0f); // [N_kv, N_q, NH, B] + auto* graph = ggml_new_graph_custom(ctx, 4096, false); + ggml_build_forward_expand(graph, gr.geo_feats); - auto* v_t = ggml_cont(ctx, ggml_transpose(ctx, V)); // [N_kv, HD, NH, B] - ca_out = ggml_mul_mat(ctx, v_t, kq); // [HD, N_q, NH, B] - ca_out = ggml_cont(ctx, ggml_permute(ctx, ca_out, 0, 2, 1, 3)); - } else { - ca_out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { + fprintf(stderr, "%s: geometry encoder alloc failed\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; } - ca_out = ggml_reshape_3d(ctx, ca_out, D, N_q, B); - ca_out = ggml_mul_mat(ctx, ly.ca_out_w, ca_out); - ca_out = ggml_add(ctx, ca_out, ly.ca_out_b); + ggml_backend_tensor_set(g_img, img_feats_cpu.data(), 0, D * N_spatial * sizeof(float)); + ggml_backend_tensor_set(g_pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); - queries = ggml_add(ctx, queries, ca_out); - // Post-norm: norm1 (Python's post-image-CA norm) - queries = sam3_layer_norm(ctx, queries, ly.norm1_w, ly.norm1_b); - if (layer_idx == 0) { - ggml_set_name(queries, "ddec_layer0_after_img_ca"); + // Pre-transformer input (CLS + optional box embeddings) + { + const float* feats_ptr = n_boxes > 0 ? img_feats_cpu.data() : nullptr; + auto geom_data = sam3_precompute_geom_input(model, params, feats_ptr, H, H); + auto* gi = ggml_get_tensor(ctx, "geom_post_final_proj"); + if (gi) ggml_backend_tensor_set(gi, geom_data.data(), 0, geom_data.size() * sizeof(float)); } - } - - // 4. FFN (post-norm, ReLU) - { - auto* ffn = ggml_mul_mat(ctx, ly.ffn_fc1_w, queries); - ffn = ggml_add(ctx, ffn, ly.ffn_fc1_b); - ffn = ggml_relu(ctx, ffn); - ffn = ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn); - ffn = ggml_add(ctx, ffn, ly.ffn_fc2_b); - queries = ggml_add(ctx, queries, ffn); - // Post-norm: norm3 (Python's post-FFN norm) - queries = sam3_layer_norm(ctx, queries, ly.norm4_w, ly.norm4_b); - if (layer_idx == 0) { - ggml_set_name(queries, "ddec_layer0_full_out"); + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; } - } + ggml_backend_tensor_get(gr.geo_feats, geo_feats_cpu.data(), 0, + D * N_geo * sizeof(float)); - return queries; -} + ggml_gallocr_free(alloc); + ggml_free(ctx); + } -// DotProductScoring: classify queries against text features via dot product. -// -// Python reference (DotProductScoring.forward): -// 1. prompt_mlp(prompt) → residual MLP + LN on text features -// 2. mean_pool_text(result, prompt_mask) → pooled [BS, D] (only valid tokens) -// 3. prompt_proj(pooled) → [BS, D] -// 4. hs_proj(hs) → [num_layer, BS, N_q, D] -// 5. matmul(proj_hs, proj_pooled.unsqueeze(-1)) → dot product → [num_layer, BS, N_q, 1] -// 6. scale by 1/sqrt(D) -// 7. clamp to [-12, 12] -// -// query_outputs: [D, N_q, B] — the 200 object query outputs -// text_features: [D, T, B] — text encoder output (already through resizer) -// text_valid_mask: [T, 1, B] — 1.0 for valid tokens, 0.0 for padding (or nullptr for all-valid) -// Returns: class_scores [N_q, B] (one score per query per batch) -static struct ggml_tensor* sam3_dot_product_scoring( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* query_outputs, // [D, N_q, B] - struct ggml_tensor* text_features, // [D, T, B] - struct ggml_tensor* text_valid_mask) // [T, 1, B] or nullptr -{ - const auto& tensors = model.tensors; - const int64_t D = query_outputs->ne[0]; // 256 - const int64_t T = text_features->ne[1]; - const int64_t B = text_features->ne[2]; + SAM3_LOG(2, "%s: geometry encoder done\n", __func__); - // Step 1: Apply prompt_mlp on text features (residual MLP + LayerNorm) - auto* text_mlp = text_features; // [D, T, B] - auto* orig_text = text_features; - text_mlp = ggml_mul_mat(ctx, tensors.at("scoring.prompt_mlp.layers.0.weight"), text_mlp); - text_mlp = ggml_add(ctx, text_mlp, tensors.at("scoring.prompt_mlp.layers.0.bias")); - text_mlp = ggml_relu(ctx, text_mlp); - text_mlp = ggml_mul_mat(ctx, tensors.at("scoring.prompt_mlp.layers.1.weight"), text_mlp); - text_mlp = ggml_add(ctx, text_mlp, tensors.at("scoring.prompt_mlp.layers.1.bias")); - text_mlp = ggml_add(ctx, text_mlp, orig_text); - text_mlp = sam3_layer_norm(ctx, text_mlp, - tensors.at("scoring.prompt_mlp.out_norm.weight"), - tensors.at("scoring.prompt_mlp.out_norm.bias")); - // text_mlp: [D, T, B] - ggml_set_name(text_mlp, "scoring_prompt_mlp_out"); + // Build combined prompt on CPU: [text_feats, geo_feats] → [D * T] + std::vector combined_prompt_cpu(D * T); + memcpy(combined_prompt_cpu.data(), text_feats_cpu.data(), D * L * sizeof(float)); + memcpy(combined_prompt_cpu.data() + D * L, geo_feats_cpu.data(), D * N_geo * sizeof(float)); - // Step 2: Mean-pool over valid text tokens → [D, 1, B] - // Python: pooled = (prompt * is_valid).sum(0) / num_valid - struct ggml_tensor* text_pooled; - if (text_valid_mask) { - // Permute to [T, D, B] for masked pooling - auto* tp = ggml_cont(ctx, ggml_permute(ctx, text_mlp, 1, 0, 2, 3)); // [T, D, B] - // Mask: text_valid_mask is [T, 1, B] — broadcast multiply zeros out padding - tp = ggml_mul(ctx, tp, text_valid_mask); // [T, D, B] with padding zeroed - // Sum over T dimension: pool_1d with SUM kernel=T - // ggml_pool_1d AVG divides by T; we want SUM then divide by n_valid. - // Use AVG and then scale by T/n_valid? Or use a manual approach. - // Simpler: sum via pool_1d with AVG, then scale by T/n_valid. - // But n_valid is dynamic. Instead: sum = mean * T, then divide by n_valid. - // We pass n_valid as part of the mask: text_valid_mask sums to n_valid. - // pool_1d(masked, AVG, T, T, 0) = sum(masked) / T. Multiply by T → sum(masked). - // Then divide by n_valid. But n_valid is a scalar we know CPU-side. - // For simplicity: compute AVG over ALL T positions (with padding zeroed out). - // This gives sum(valid) / T. To get sum(valid) / n_valid, scale by T / n_valid. - // We embed the scale factor into the mask: mask = (T / n_valid) for valid, 0 for pad. - // Then AVG(mask * features) = sum(valid * T/n_valid) / T = sum(valid) / n_valid. ✓ - // Caller should set mask values to T/n_valid for valid tokens, 0 for padding. - auto* pooled_t = ggml_pool_1d(ctx, tp, GGML_OP_POOL_AVG, (int)T, (int)T, 0); - text_pooled = ggml_cont(ctx, ggml_permute(ctx, pooled_t, 1, 0, 2, 3)); // [D, 1, B] - } else { - // All tokens valid — simple mean - auto* tp = ggml_cont(ctx, ggml_permute(ctx, text_mlp, 1, 0, 2, 3)); - auto* pooled_t = ggml_pool_1d(ctx, tp, GGML_OP_POOL_AVG, (int)T, (int)T, 0); - text_pooled = ggml_cont(ctx, ggml_permute(ctx, pooled_t, 1, 0, 2, 3)); - } - ggml_set_name(text_pooled, "scoring_pooled"); + SAM3_LOG(2, "%s: starting fusion encoder\n", __func__); + /* + ** ── SUB-GRAPH 3: Fusion Encoder ────────────────────────────────── + */ + std::vector fenc_output_cpu(D * N_spatial); + { + const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; + struct ggml_init_params gp = {sz, nullptr, true}; + auto* ctx = ggml_init(gp); - // Step 3: Project pooled prompt through prompt_proj: D→D - auto* proj_pooled = ggml_mul_mat(ctx, tensors.at("scoring.prompt_proj.weight"), text_pooled); - proj_pooled = ggml_add(ctx, proj_pooled, tensors.at("scoring.prompt_proj.bias")); - // proj_pooled: [D, 1, B] - ggml_set_name(proj_pooled, "scoring_proj_pooled"); + auto* img = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(img, "fenc_img"); + ggml_set_input(img); + auto* pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(pe, "fenc_pe"); + ggml_set_input(pe); + auto* prompt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); + ggml_set_name(prompt, "fenc_prompt"); + ggml_set_input(prompt); + auto* bias = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); + ggml_set_name(bias, "fenc_bias"); + ggml_set_input(bias); - // Step 4: Project queries through hs_proj: D→D - auto* proj_hs = ggml_mul_mat(ctx, tensors.at("scoring.hs_proj.weight"), query_outputs); - proj_hs = ggml_add(ctx, proj_hs, tensors.at("scoring.hs_proj.bias")); - // proj_hs: [D, N_q, B] - ggml_set_name(proj_hs, "scoring_proj_hs"); + auto* out = sam3_build_fenc_graph(ctx, model, img, prompt, pe, bias); + ggml_set_output(out); - // Step 5: Dot product — for each query, dot with pooled prompt - // matmul(proj_hs, proj_pooled.unsqueeze(-1)) in Python = batched vector-matrix multiply - // ggml_mul_mat(A, B) = A^T @ B - // With A = proj_pooled [D, 1, B], B = proj_hs [D, N_q, B]: - // result = [1, N_q, B] — each element is dot product of query with pooled prompt - auto* scores = ggml_mul_mat(ctx, proj_pooled, proj_hs); // [1, N_q, B] + auto* graph = ggml_new_graph_custom(ctx, 16384, false); + ggml_build_forward_expand(graph, out); - // Step 6: Scale by 1/sqrt(D) - float scale = 1.0f / sqrtf((float)D); - scores = ggml_scale(ctx, scores, scale); + auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { + fprintf(stderr, "%s: fusion encoder alloc failed\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } - // Step 7: Clamp to [-12, 12] - scores = ggml_clamp(ctx, scores, -12.0f, 12.0f); + ggml_backend_tensor_set(img, img_feats_cpu.data(), 0, D * N_spatial * sizeof(float)); + ggml_backend_tensor_set(pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); + ggml_backend_tensor_set(prompt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); + ggml_backend_tensor_set(bias, combined_bias_cpu.data(), 0, T * sizeof(float)); - // Reshape to [N_q, B] - const int64_t N_q = query_outputs->ne[1]; - scores = ggml_reshape_2d(ctx, scores, N_q, B); - ggml_set_name(scores, "scoring_class_scores"); + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } + ggml_backend_tensor_get(out, fenc_output_cpu.data(), 0, D * N_spatial * sizeof(float)); - return scores; -} + SAM3_LOG(2, "%s: fenc_out[0..4] = [%.6f, %.6f, %.6f, %.6f, %.6f]\n", + __func__, fenc_output_cpu[0], fenc_output_cpu[1], fenc_output_cpu[2], + fenc_output_cpu[3], fenc_output_cpu[4]); -// Build full DETR decoder graph. -// enc_feats: [D, N_kv, B] conditioned features from fusion encoder (N_kv=5184) -// enc_pos: [D, N_kv, B] positional encoding -// text_feats: [D, T, B] text features -// Returns struct with: -// queries: [D, 201, B] (all query outputs including presence token) -// pred_boxes: [4, 200, B] (cx, cy, w, h in [0,1]) -// class_scores: [200, B] -// presence_score: [1, B] -struct sam3_ddec_output { - struct ggml_tensor* queries; // [D, 201, B] - struct ggml_tensor* presence_feats; // [D, 1, B] pre-decoder-norm presence token - struct ggml_tensor* pred_boxes; // [4, 200, B] - struct ggml_tensor* class_scores; // [200, B] - struct ggml_tensor* presence_score; // [1, B] -}; + ggml_gallocr_free(alloc); + ggml_free(ctx); + } -static sam3_ddec_output sam3_build_ddec_graph( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* enc_feats, // [D, N_kv, B] - struct ggml_tensor* enc_pos, // [D, N_kv, B] - struct ggml_tensor* text_feats, // [D, T, B] - struct ggml_tensor* sine_dim_t, // [1, 64] — pre-computed angle multipliers - struct ggml_tensor* rpb_coords, // [feat_hw] — normalized grid coords (or nullptr) - struct ggml_tensor* text_attn_bias = nullptr, // [T, 1, B] additive text padding bias - struct ggml_tensor* text_valid_mask = nullptr) // [T, 1, B] for scoring (or nullptr) -{ - const auto& hp = model.hparams; - const auto& tensors = model.tensors; - const int D = hp.neck_dim; // 256 - const int NQ = hp.ddec_num_queries; // 200 - const int B = (int)enc_feats->ne[2]; // batch (1) - const int feat_hw = hp.n_img_embd(); // 72 + SAM3_LOG(2, "%s: fusion encoder done\n", __func__); + /* + ** ── SUB-GRAPH 4: DETR Decoder + Scoring ────────────────────────── + */ + std::vector scores_data(NQ); + std::vector boxes_data(4 * NQ); + std::vector queries_data(D * (NQ + 1)); // 201 = NQ + presence token + float presence_logit = 0.0f; + { + const size_t sz = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; + struct ggml_init_params gp = {sz, nullptr, true}; + auto* ctx = ggml_init(gp); - // ── Initialize queries from query_embed ────────────────────────────── - auto* content = ggml_reshape_3d(ctx, model.ddec.query_embed, D, NQ, 1); - auto* pres_tok = ggml_reshape_3d(ctx, model.ddec.presence_token, D, 1, 1); - auto* queries = ggml_concat(ctx, pres_tok, content, 1); // [D, NQ+1, B=1] + auto* enc = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(enc, "ddec_enc"); + ggml_set_input(enc); + auto* pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(pe, "ddec_pe"); + ggml_set_input(pe); + auto* txt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); + ggml_set_name(txt, "ddec_text"); + ggml_set_input(txt); + auto* sdt = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, 64); + ggml_set_name(sdt, "sine_dim_t"); + ggml_set_input(sdt); + auto* rpb = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H); + ggml_set_name(rpb, "rpb_coords"); + ggml_set_input(rpb); + auto* tab = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); + ggml_set_name(tab, "text_attn_bias"); + ggml_set_input(tab); + auto* tvm = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); + ggml_set_name(tvm, "text_valid_mask"); + ggml_set_input(tvm); - // Reference points: sigmoid → initial anchor boxes - auto* ref_pts_raw = ggml_cont(ctx, tensors.at("ddec.reference_points.weight")); // [4, NQ] - auto* ref_boxes = ggml_sigmoid(ctx, ref_pts_raw); // [4, NQ] - ref_boxes = ggml_reshape_3d(ctx, ref_boxes, 4, NQ, 1); // [4, NQ, 1] -#ifndef NDEBUG - auto* ref_boxes_dbg = ggml_cont(ctx, ref_boxes); - ggml_set_name(ref_boxes_dbg, "ddec_ref_boxes_init"); -#endif + auto dout = sam3_build_ddec_graph(ctx, model, enc, pe, txt, sdt, rpb, tab, tvm); + ggml_set_output(dout.class_scores); + ggml_set_output(dout.pred_boxes); + ggml_set_output(dout.presence_score); + ggml_set_output(dout.queries); - // ── Run decoder layers ─────────────────────────────────────────────── - // Per-layer: recompute query_pos from updated ref_boxes (matching Python exactly) - struct ggml_tensor* last_presence = pres_tok; - for (int i = 0; i < hp.ddec_layers; ++i) { - // Recompute query_pos from current ref_boxes via sine embed + ref_point_head MLP - auto* query_pos = sam3_build_query_pos(ctx, model, ref_boxes, sine_dim_t, i); + auto* graph = ggml_new_graph_custom(ctx, 65536, false); + ggml_build_forward_expand(graph, dout.class_scores); + ggml_build_forward_expand(graph, dout.pred_boxes); + ggml_build_forward_expand(graph, dout.presence_score); + ggml_build_forward_expand(graph, dout.queries); - // Compute box-relative positional bias for image cross-attention - struct ggml_tensor* rpb_mask = nullptr; - if (rpb_coords) { - rpb_mask = sam3_compute_box_rpb(ctx, model, ref_boxes, rpb_coords, feat_hw, i); + auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { + fprintf(stderr, "%s: DETR decoder alloc failed\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; } - queries = sam3_ddec_layer_forward(ctx, model.ddec.layers[i], - queries, query_pos, - enc_feats, enc_pos, - text_feats, hp.ddec_heads, - text_attn_bias, - rpb_mask, - i); - - // Box refinement after each layer (on object queries only, not presence token) - auto* obj_q = ggml_view_3d(ctx, queries, D, NQ, 1, - queries->nb[1], queries->nb[2], 1 * queries->nb[1]); - obj_q = ggml_cont(ctx, obj_q); - sam3_name_tensorf(obj_q, "ddec_layer%d_out", i); + ggml_backend_tensor_set(enc, fenc_output_cpu.data(), 0, D * N_spatial * sizeof(float)); + ggml_backend_tensor_set(pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); + ggml_backend_tensor_set(txt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); + ggml_backend_tensor_set(sdt, sine_dim_t_cpu.data(), 0, 64 * sizeof(float)); + ggml_backend_tensor_set(rpb, rpb_coords_cpu.data(), 0, H * sizeof(float)); + ggml_backend_tensor_set(tab, combined_bias_cpu.data(), 0, T * sizeof(float)); + ggml_backend_tensor_set(tvm, text_valid_mask_cpu.data(), 0, T * sizeof(float)); - auto* pres_q = ggml_view_3d(ctx, queries, D, 1, 1, - queries->nb[1], queries->nb[2], 0); - pres_q = ggml_cont(ctx, pres_q); - if (i == 0) { - ggml_set_name(pres_q, "ddec_layer0_presence"); + // Presence token position encoding: zeros + auto* qpos = ggml_get_tensor(ctx, "ddec_query_pos_pres"); + if (qpos) { + std::vector z(D, 0.0f); + ggml_backend_tensor_set(qpos, z.data(), 0, D * sizeof(float)); + } + auto* rpbz = ggml_get_tensor(ctx, "rpb_pres_zeros"); + if (rpbz) { + int n = (int)(rpbz->ne[0] * rpbz->ne[1] * rpbz->ne[2] * rpbz->ne[3]); + std::vector z(n, 0.0f); + ggml_backend_tensor_set(rpbz, z.data(), 0, n * sizeof(float)); } - last_presence = pres_q; - - // Apply the final decoder norm before box refinement (use_normed_output_consistently) - auto* obj_q_normed = sam3_layer_norm(ctx, obj_q, - tensors.at("ddec.norm.weight"), - tensors.at("ddec.norm.bias")); - // Shared bbox_embed MLP - auto* bd = obj_q_normed; - for (int j = 0; j < 3; ++j) { - auto wn = "ddec.bbox_embed.layers." + std::to_string(j) + ".weight"; - auto bn = "ddec.bbox_embed.layers." + std::to_string(j) + ".bias"; - bd = ggml_mul_mat(ctx, tensors.at(wn), bd); - bd = ggml_add(ctx, bd, tensors.at(bn)); - if (j < 2) bd = ggml_relu(ctx, bd); + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; } - // bd: [4, NQ, 1] - // ref_boxes = sigmoid(inverse_sigmoid(ref_boxes) + box_delta) - auto* ref_inv_cur = sam3_inverse_sigmoid(ctx, ref_boxes); - ref_boxes = ggml_sigmoid(ctx, ggml_add(ctx, ref_inv_cur, bd)); - sam3_name_tensorf(ref_boxes, "ddec_layer%d_refboxes", i); - } + ggml_backend_tensor_get(dout.class_scores, scores_data.data(), 0, NQ * sizeof(float)); + ggml_backend_tensor_get(dout.pred_boxes, boxes_data.data(), 0, 4 * NQ * sizeof(float)); + ggml_backend_tensor_get(dout.presence_score, &presence_logit, 0, sizeof(float)); + ggml_backend_tensor_get(dout.queries, queries_data.data(), 0, + D * (NQ + 1) * sizeof(float)); - // ── Final normalization ────────────────────────────────────────────── - // Match Python: decoder.norm is applied to object queries only. - auto* obj_queries = ggml_view_3d(ctx, queries, D, NQ, 1, - queries->nb[1], queries->nb[2], 1 * queries->nb[1]); - obj_queries = ggml_cont(ctx, obj_queries); - obj_queries = sam3_layer_norm(ctx, obj_queries, - tensors.at("ddec.norm.weight"), - tensors.at("ddec.norm.bias")); - ggml_set_name(obj_queries, "ddec_normed_output"); + ggml_gallocr_free(alloc); + ggml_free(ctx); + } - auto* queries_for_seg = ggml_concat(ctx, last_presence, obj_queries, 1); + float presence_prob = 1.0f / (1.0f + expf(-presence_logit)); - auto* class_scores = sam3_dot_product_scoring(ctx, model, obj_queries, text_feats, text_valid_mask); - // class_scores: [NQ, B] + SAM3_LOG(2, "%s: DETR decoder done\n", __func__); + /* + ** ── SUB-GRAPH 5: Segmentation Head ─────────────────────────────── + */ + const int mask_hw = H * 4; // 288 for SAM3 + std::vector all_masks(NQ * mask_hw * mask_hw); + { + const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; + struct ggml_init_params gp = {sz, nullptr, true}; + auto* ctx = ggml_init(gp); - // ── Presence score ─────────────────────────────────────────────────── - // Presence token head: LN + 3-layer MLP (D→D→D→1) - auto* pres_out = sam3_layer_norm(ctx, last_presence, - tensors.at("ddec.presence_token_out_norm.weight"), - tensors.at("ddec.presence_token_out_norm.bias")); + auto* enc_h = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); + ggml_set_name(enc_h, "seg_enc"); + ggml_set_input(enc_h); - for (int j = 0; j < 3; ++j) { - auto wn = "ddec.presence_token_head.layers." + std::to_string(j) + ".weight"; - auto bn = "ddec.presence_token_head.layers." + std::to_string(j) + ".bias"; - pres_out = ggml_mul_mat(ctx, tensors.at(wn), pres_out); - pres_out = ggml_add(ctx, pres_out, tensors.at(bn)); - if (j < 2) pres_out = ggml_relu(ctx, pres_out); - } - // Keep presence as raw logit (no sigmoid yet — applied during post-processing) - auto* presence_score = ggml_reshape_2d(ctx, pres_out, 1, 1); - // presence_score: [1, B] — raw logit + // FPN features at 3 scales — create fresh inputs + const int H0 = H * 4, H1 = H * 2; + auto* fpn0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H0, H0, 1); + ggml_set_name(fpn0, "seg_fpn0"); + ggml_set_input(fpn0); + auto* fpn1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H1, H1, 1); + ggml_set_name(fpn1, "seg_fpn1"); + ggml_set_input(fpn1); + auto* fpn2 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(fpn2, "seg_fpn2"); + ggml_set_input(fpn2); + struct ggml_tensor* fpn_feats[3] = {fpn0, fpn1, fpn2}; - sam3_ddec_output out; - out.queries = queries_for_seg; // [D, NQ+1, B] - out.presence_feats = last_presence; // [D, 1, B] - out.pred_boxes = ref_boxes; // [4, NQ, B] - out.class_scores = class_scores; // [NQ, B] - out.presence_score = presence_score; // [1, B] + // Object queries (skip presence token at index 0 → start at index 1) + auto* oq = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, NQ, 1); + ggml_set_name(oq, "seg_queries"); + ggml_set_input(oq); - return out; -} + auto* txt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); + ggml_set_name(txt, "seg_text"); + ggml_set_input(txt); + auto* tab = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); + ggml_set_name(tab, "seg_bias"); + ggml_set_input(tab); -/***************************************************************************** -** Segmentation head (MaskFormer) — graph building -*****************************************************************************/ + auto* out = sam3_build_seg_head_graph(ctx, model, enc_h, fpn_feats, + oq, txt, tab); + ggml_set_output(out); -// Build pixel decoder: progressively upsample FPN features. -// fpn_feats[0]: [D, 288, 288, B] (highest res) -// fpn_feats[1]: [D, 144, 144, B] -// fpn_feats[2]: [D, 72, 72, B] (lowest res) -// Returns: [D, 288, 288, B] pixel features -// -// Python PixelDecoder.forward: -// prev_fpn = backbone_feats[-1] (lowest res) -// for bb_feat in backbone_feats[:-1][::-1]: (iterate from second-lowest to highest) -// prev_fpn = bb_feat + F.interpolate(prev_fpn, size=bb_feat.shape[-2:], mode="nearest") -// prev_fpn = conv_layers[i](prev_fpn) # conv on the MERGED result -// prev_fpn = F.relu(norms[i](prev_fpn)) # GroupNorm then ReLU -// -// Python uses GroupNorm(8, 256) — we use ggml_group_norm which normalizes ne[2] -// (the channel dim) in groups. The conv output is [W, H, D, B] with D in ne[2]. -static struct ggml_tensor* sam3_pixel_decoder( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* fpn_feats[3]) // [D, W, H, B] at 3 scales -{ - const auto& seg = model.seg_head; + auto* graph = ggml_new_graph_custom(ctx, 32768, false); + ggml_build_forward_expand(graph, out); - // Start from lowest resolution - auto* feat = fpn_feats[2]; // [D, 72, 72, B] + SAM3_LOG(2, "%s: seg head graph: %d nodes\n", __func__, ggml_graph_n_nodes(graph)); - // Iteration 0: merge with FPN[1] (144x144) - // prev_fpn = FPN[1] + upsample(prev_fpn) - // Permute to [W, H, D, B] for conv operations - auto* prev = ggml_cont(ctx, ggml_permute(ctx, feat, 2, 0, 1, 3)); // [72, 72, D, B] - prev = ggml_upscale(ctx, prev, 2, GGML_SCALE_MODE_NEAREST); // [144, 144, D, B] - auto* fpn1 = ggml_cont(ctx, ggml_permute(ctx, fpn_feats[1], 2, 0, 1, 3)); // [144, 144, D, B] - prev = ggml_add(ctx, fpn1, prev); // merged - // Conv 3x3 on the MERGED result (not individual FPN feat) - prev = ggml_conv_2d_s1_ph(ctx, seg.up_conv_w[0], prev); - { - auto* b3d = ggml_reshape_3d(ctx, seg.up_conv_b[0], 1, 1, seg.up_conv_b[0]->ne[0]); - prev = ggml_add(ctx, prev, ggml_repeat(ctx, b3d, prev)); - } - // GroupNorm(8, 256) then ReLU — prev is [W, H, D, B] with D in ne[2] - prev = ggml_group_norm(ctx, prev, 8, 1e-5f); - { - auto* w3d = ggml_reshape_3d(ctx, seg.up_norm_w[0], 1, 1, seg.up_norm_w[0]->ne[0]); - prev = ggml_mul(ctx, prev, ggml_repeat(ctx, w3d, prev)); - auto* bn3d = ggml_reshape_3d(ctx, seg.up_norm_b[0], 1, 1, seg.up_norm_b[0]->ne[0]); - prev = ggml_add(ctx, prev, ggml_repeat(ctx, bn3d, prev)); - } - prev = ggml_relu(ctx, prev); - ggml_set_name(prev, "seg_pixel_dec_stage0"); + auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { + fprintf(stderr, "%s: segmentation head alloc failed\n", __func__); + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } - // Iteration 1: merge with FPN[0] (288x288) - prev = ggml_upscale(ctx, prev, 2, GGML_SCALE_MODE_NEAREST); // [288, 288, D, B] - auto* fpn0 = ggml_cont(ctx, ggml_permute(ctx, fpn_feats[0], 2, 0, 1, 3)); // [288, 288, D, B] - prev = ggml_add(ctx, fpn0, prev); // merged - // Conv 3x3 on the MERGED result - prev = ggml_conv_2d_s1_ph(ctx, seg.up_conv_w[1], prev); - { - auto* b3d = ggml_reshape_3d(ctx, seg.up_conv_b[1], 1, 1, seg.up_conv_b[1]->ne[0]); - prev = ggml_add(ctx, prev, ggml_repeat(ctx, b3d, prev)); - } - // GroupNorm(8, 256) then ReLU - prev = ggml_group_norm(ctx, prev, 8, 1e-5f); - { - auto* w3d = ggml_reshape_3d(ctx, seg.up_norm_w[1], 1, 1, seg.up_norm_w[1]->ne[0]); - prev = ggml_mul(ctx, prev, ggml_repeat(ctx, w3d, prev)); - auto* bn3d = ggml_reshape_3d(ctx, seg.up_norm_b[1], 1, 1, seg.up_norm_b[1]->ne[0]); - prev = ggml_add(ctx, prev, ggml_repeat(ctx, bn3d, prev)); - } - prev = ggml_relu(ctx, prev); - ggml_set_name(prev, "seg_pixel_dec_stage1"); + ggml_backend_tensor_set(enc_h, fenc_output_cpu.data(), 0, + D * N_spatial * sizeof(float)); - // Python PixelDecoder allocates 3 conv layers but only uses 2 (one per - // upsample step). The 3rd conv (up_conv_w[2]) is unused. + // Copy FPN features from state + { + size_t s0 = (size_t)D * H0 * H0 * sizeof(float); + size_t s1 = (size_t)D * H1 * H1 * sizeof(float); + size_t s2 = (size_t)D * H * H * sizeof(float); + std::vector b0(D * H0 * H0), b1(D * H1 * H1), b2(D * H * H); + ggml_backend_tensor_get(state.neck_det[0], b0.data(), 0, s0); + if (fpn0->buffer) ggml_backend_tensor_set(fpn0, b0.data(), 0, s0); + ggml_backend_tensor_get(state.neck_det[1], b1.data(), 0, s1); + if (fpn1->buffer) ggml_backend_tensor_set(fpn1, b1.data(), 0, s1); + ggml_backend_tensor_get(state.neck_det[2], b2.data(), 0, s2); + if (fpn2->buffer) ggml_backend_tensor_set(fpn2, b2.data(), 0, s2); + } - auto* out = ggml_cont(ctx, ggml_permute(ctx, prev, 1, 2, 0, 3)); // [D, 288, 288, B] - return out; -} + // Object queries: extract from DETR queries (skip presence token at slot 0) + // queries_data is flat [D * 201], presence token is at positions [0..D-1] + // Object queries start at position [D..D*(NQ+1)-1] + ggml_backend_tensor_set(oq, queries_data.data() + D, 0, D * NQ * sizeof(float)); -// Build the full segmentation head graph. -// -// Python UniversalSegmentationHead.forward: -// 1. Cross-attend encoder_hidden_states to prompt → updated encoder -// 2. _embed_pixels: replace lowest-res FPN feat with spatial portion of encoder output -// 3. Run pixel decoder on modified FPN feats -// 4. instance_seg_head (Conv1x1) -// 5. mask_predictor: einsum(mask_embed(queries), instance_embeds) -// -// enc_hidden: [D, N_spatial, B] — fusion encoder output (cross-attended in step 1) -// fpn_feats[3]: the 3 FPN features at different resolutions -// query_outputs: [D, N, B] selected object query outputs -// text_features: [D, T, B] for cross-attention (prompt) -// Returns: mask_logits [W*H, N, B] (raw logits, not sigmoid) -static struct ggml_tensor* sam3_build_seg_head_graph( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* enc_hidden, // [D, N_spatial, B] fusion encoder output - struct ggml_tensor* fpn_feats[3], // FPN features at 3 scales - struct ggml_tensor* query_outputs, // [D, N, B] - struct ggml_tensor* text_features, // [D, T, B] (for cross-attn, can be nullptr) - struct ggml_tensor* text_attn_bias = nullptr) { - const auto& seg = model.seg_head; - const auto& tensors = model.tensors; - const int64_t D = enc_hidden->ne[0]; // 256 - const int64_t B = enc_hidden->ne[2]; // 1 - const int64_t N = query_outputs->ne[1]; // number of selected queries + ggml_backend_tensor_set(txt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); + ggml_backend_tensor_set(tab, combined_bias_cpu.data(), 0, T * sizeof(float)); - auto* enc = enc_hidden; - if (text_features) { - auto* ca_norm = sam3_layer_norm(ctx, enc, - tensors.at("seg.cross_attn_norm.weight"), - tensors.at("seg.cross_attn_norm.bias")); + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(alloc); + ggml_free(ctx); + return result; + } + ggml_backend_tensor_get(out, all_masks.data(), 0, all_masks.size() * sizeof(float)); - auto* ca_mask = sam3_expand_token_attn_bias(ctx, text_attn_bias, enc->ne[1], 8, B); - auto* ca_out = sam3_multihead_attn_fused(ctx, ca_norm, text_features, - seg.ca_prompt_q_w, seg.ca_prompt_q_b, - seg.ca_prompt_out_w, seg.ca_prompt_out_b, - 8, ca_mask); - enc = ggml_add(ctx, enc, ca_out); + ggml_gallocr_free(alloc); + ggml_free(ctx); } - // enc: [D, N_spatial, B] - ggml_set_name(enc, "seg_enc_after_ca"); - // Replace lowest-res FPN feat with spatial portion of encoder output - const int64_t feat_hw = model.hparams.n_img_embd(); // 72 - auto* enc_spatial = ggml_reshape_4d(ctx, enc, D, feat_hw, feat_hw, B); -#ifndef NDEBUG - auto* enc_spatial_dbg = ggml_cont(ctx, ggml_permute(ctx, enc_spatial, 2, 0, 1, 3)); - ggml_set_name(enc_spatial_dbg, "seg_enc_visual"); -#endif + /* + ** ── Post-processing: thresholding + NMS + mask resize ──────────── + */ + std::vector dets; + for (int q = 0; q < NQ; ++q) { + float class_prob = 1.0f / (1.0f + expf(-scores_data[q])); + float score = class_prob * presence_prob; + if (score < params.score_threshold) continue; - struct ggml_tensor* modified_fpn[3] = { - fpn_feats[0], - fpn_feats[1], - enc_spatial, // replaces original lowest-res FPN - }; + sam3_detection det; + float cx = boxes_data[0 + q * 4]; + float cy = boxes_data[1 + q * 4]; + float bw = boxes_data[2 + q * 4]; + float bh = boxes_data[3 + q * 4]; - auto* pixel_feats = sam3_pixel_decoder(ctx, model, modified_fpn); -#ifndef NDEBUG - auto* pixel_feats_dbg = ggml_cont(ctx, ggml_permute(ctx, pixel_feats, 2, 0, 1, 3)); - ggml_set_name(pixel_feats_dbg, "seg_pixel_decoder_out"); -#endif + det.box = sam3_cxcywh_to_xyxy(cx, cy, bw, bh, state.orig_width, state.orig_height); + det.score = score; - const int64_t W = pixel_feats->ne[1]; // 288 - const int64_t H = pixel_feats->ne[2]; // 288 + const float* mask_ptr = all_masks.data() + q * mask_hw * mask_hw; + auto mask_resized = sam3_bilinear_interpolate(mask_ptr, mask_hw, mask_hw, + state.orig_width, state.orig_height); + det.mask.width = state.orig_width; + det.mask.height = state.orig_height; + det.mask.data.resize(state.orig_width * state.orig_height); + for (int i = 0; i < (int)mask_resized.size(); ++i) + det.mask.data[i] = (mask_resized[i] > 0.0f) ? 255 : 0; + det.mask.iou_score = score; - // Instance segmentation head (Conv1x1) - auto* pf_conv = ggml_cont(ctx, ggml_permute(ctx, pixel_feats, 2, 0, 1, 3)); - pf_conv = ggml_conv_2d_sk_p0(ctx, tensors.at("seg.instance_seg_head.weight"), pf_conv); - { - auto* b3d = ggml_reshape_3d(ctx, tensors.at("seg.instance_seg_head.bias"), - 1, 1, tensors.at("seg.instance_seg_head.bias")->ne[0]); - pf_conv = ggml_add(ctx, pf_conv, ggml_repeat(ctx, b3d, pf_conv)); + dets.push_back(std::move(det)); } - auto* pixel_embed = ggml_cont(ctx, ggml_permute(ctx, pf_conv, 1, 2, 0, 3)); // [D, W, H, B] -#ifndef NDEBUG - auto* pixel_embed_dbg = ggml_cont(ctx, ggml_permute(ctx, pixel_embed, 2, 0, 1, 3)); - ggml_set_name(pixel_embed_dbg, "seg_instance_embed"); -#endif - // Mask embedding MLP - auto* mask_embed = query_outputs; - for (int j = 0; j < 3; ++j) { - auto wn = "seg.mask_predictor.mask_embed.layers." + std::to_string(j) + ".weight"; - auto bn = "seg.mask_predictor.mask_embed.layers." + std::to_string(j) + ".bias"; - mask_embed = ggml_mul_mat(ctx, tensors.at(wn), mask_embed); - mask_embed = ggml_add(ctx, mask_embed, tensors.at(bn)); - if (j < 2) mask_embed = ggml_relu(ctx, mask_embed); + SAM3_LOG(2, "%s: %zu detections above threshold %.2f (presence=%.3f, logit=%.3f)\n", + __func__, dets.size(), params.score_threshold, presence_prob, presence_logit); + + auto keep = sam3_nms(dets, params.nms_threshold); + for (int i = 0; i < (int)keep.size(); ++i) { + dets[keep[i]].instance_id = i + 1; + result.detections.push_back(std::move(dets[keep[i]])); } - ggml_set_name(mask_embed, "seg_mask_embed"); - // Mask prediction: einsum('bqc,bchw->bqhw') - auto* pe_flat = ggml_reshape_3d(ctx, pixel_embed, D, W * H, B); - auto* masks = ggml_mul_mat(ctx, pe_flat, mask_embed); - ggml_set_name(masks, "seg_mask_logits"); + SAM3_LOG(2, "%s: %zu detections after NMS\n", __func__, result.detections.size()); - return masks; +#if SAM3_LOG_LEVEL >= 1 + auto t_end = std::chrono::high_resolution_clock::now(); + double total_ms = std::chrono::duration(t_end - t_start).count(); + SAM3_LOG(1, "%s: completed in %.1f ms\n", __func__, total_ms); +#endif + + return result; } /***************************************************************************** -** Memory encoder (Phase 7, Step 7.1) +** SAM attention helper (separate Q, K, V weight/bias) *****************************************************************************/ -// CXBlock: depthwise conv + LayerNorm + pointwise MLP with residual scaling. -static struct ggml_tensor* sam3_cxblock_forward( +static struct ggml_tensor* sam3_sam_attention( struct ggml_context* ctx, - struct ggml_tensor* x, // [D, H, W, B] - struct ggml_tensor* dw_w, // [7, 7, 1, D] depthwise - struct ggml_tensor* dw_b, // [D] - struct ggml_tensor* norm_w, // [D] - struct ggml_tensor* norm_b, // [D] - struct ggml_tensor* fc1_w, // [D, 1024] - struct ggml_tensor* fc1_b, // [1024] - struct ggml_tensor* fc2_w, // [1024, D] - struct ggml_tensor* fc2_b, // [D] - struct ggml_tensor* gamma) // [D] -{ - const int D = (int)x->ne[0]; - const int H = (int)x->ne[1]; - const int W = (int)x->ne[2]; - - // ggml conv expects WHCB layout; internal feature maps are stored as CWHB. - auto* x_whcb = ggml_cont(ctx, ggml_permute(ctx, x, 2, 0, 1, 3)); - - // Depthwise conv (groups = D): use the direct depthwise path. - // ggml_conv_2d_dw_direct only supports f32 kernel — cast if needed. - auto* dw_w_f32 = (dw_w->type == GGML_TYPE_F32) ? dw_w : ggml_cast(ctx, dw_w, GGML_TYPE_F32); - auto* h = ggml_conv_2d_dw_direct(ctx, dw_w_f32, x_whcb, 1, 1, 3, 3, 1, 1); - h = ggml_add(ctx, h, ggml_reshape_4d(ctx, dw_b, 1, 1, D, 1)); - h = ggml_cont(ctx, ggml_permute(ctx, h, 1, 2, 0, 3)); + struct ggml_tensor* q_in, // [D, N_q, B] + struct ggml_tensor* k_in, // [D, N_kv, B] + struct ggml_tensor* v_in, // [D, N_kv, B] + const sam3_sam_attn& attn, + int n_heads) { + const int64_t N_q = q_in->ne[1]; + const int64_t B = q_in->ne[2]; + const int64_t N_kv = k_in->ne[1]; - // LayerNorm2d - h = sam3_layer_norm_2d(ctx, h, norm_w, norm_b); + // Project + auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, attn.q_w, q_in), attn.q_b); + auto* K = ggml_add(ctx, ggml_mul_mat(ctx, attn.k_w, k_in), attn.k_b); + auto* V = ggml_add(ctx, ggml_mul_mat(ctx, attn.v_w, v_in), attn.v_b); - // Pointwise MLP: reshape to [D, H*W, B], apply FC, reshape back - auto* flat = ggml_reshape_3d(ctx, h, D, H * W, 1); - flat = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, flat), fc1_b); - flat = ggml_gelu(ctx, flat); - flat = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, flat), fc2_b); - h = ggml_reshape_4d(ctx, flat, D, H, W, 1); + // Debug: mark projections for the first SA call (N_q=8 tokens, block 0) + static int _sa_call_count = 0; + if (_sa_call_count == 0 && N_q <= 16) { + ggml_set_name(Q, "dbg_sa0_Q_proj"); + ggml_set_output(Q); + ggml_set_name(V, "dbg_sa0_V_proj"); + ggml_set_output(V); + } + _sa_call_count++; - // Residual with learnable scaling: x + gamma * h - auto* gamma_4d = ggml_reshape_4d(ctx, gamma, D, 1, 1, 1); - h = ggml_mul(ctx, h, gamma_4d); - return ggml_add(ctx, x, h); -} + // internal_dim = out_proj cols = attn.q_w->ne[1] + const int64_t ID = attn.q_w->ne[1]; + const int64_t HD = ID / n_heads; -/***************************************************************************** -** Prompt building for memory attention (Phase 7) -*****************************************************************************/ + // Reshape to multi-head: [HD, N, NH, B] + Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); + Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); // [HD, N_q, NH, B] -struct sam3_prompt_data { - std::vector prompt; // [MD * M_total] flat - std::vector prompt_pos; // [MD * M_total] flat - int M_total; // total prompt tokens - int num_obj_ptr_tokens; // pointer tokens at end (excluded from RoPE) - int M_spatial; // spatial memory tokens = M_total - num_obj_ptr_tokens -}; + K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); + K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); // [HD, N_kv, NH, B] -// Build prompt and prompt_pos tensors CPU-side for memory attention. -// mem_slot_feats[i]: [MD * H * H] raw spatial features (ggml layout) -// mem_slot_pes[i]: [MD * H * H] sinusoidal spatial PE (ggml layout) -// spatial_tpos[i]: temporal position for slot i (0 = conditioning, >0 = non-cond t_pos) -// obj_ptrs[i]: [D] object pointer data -// ptr_tpos[i]: temporal position for pointer i (<0 = skip) -static sam3_prompt_data sam3_build_prompt_and_pos( - const sam3_model& model, - const std::vector>& mem_slot_feats, - const std::vector>& mem_slot_pes, - const std::vector& spatial_tpos, - const std::vector>& obj_ptrs, - const std::vector& ptr_tpos, - int eff_feat_size = 0) { - const auto& hp = model.hparams; - const int MD = hp.mem_out_dim; // 64 - const int D = hp.neck_dim; // 256 - const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); - // Per-slot token count: from actual data size (handles both HxH and perceiver 512) - const int HH = mem_slot_feats.empty() ? (H * H) - : (int)(mem_slot_feats[0].size() / MD); + V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); + V = ggml_cont(ctx, ggml_permute(ctx, V, 0, 2, 1, 3)); // [HD, N_kv, NH, B] contiguous - sam3_prompt_data pd; - pd.num_obj_ptr_tokens = 0; + // Attention + float scale = 1.0f / sqrtf((float)HD); + auto* out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); + // out: [HD, NH, N_q, B] (flash_attn_ext swaps dims 1,2 vs input) - // Read maskmem_tpos_enc from model (one tensor [MD, 1, 1, 7]) - std::vector tpos_all(MD * hp.num_maskmem); - if (model.mem_enc.tpos[0]) { - sam3_read_f32(model.mem_enc.tpos[0], tpos_all.data(), MD * hp.num_maskmem); - } +#if 0 // Manual SDPA (for debugging only) + auto* Q3 = ggml_reshape_3d(ctx, Q, HD, N_q, n_heads * B); + auto* K3 = ggml_reshape_3d(ctx, K, HD, N_kv, n_heads * B); + auto* V3 = ggml_reshape_3d(ctx, V, HD, N_kv, n_heads * B); + // QK^T: ggml_mul_mat(K, Q) → K^T @ Q → [N_kv, N_q, NH*B] + auto* attn_scores = ggml_mul_mat(ctx, K3, Q3); + attn_scores = ggml_scale(ctx, attn_scores, scale); + attn_scores = ggml_soft_max(ctx, attn_scores); - // 1. Spatial memory tokens - for (size_t s = 0; s < mem_slot_feats.size(); ++s) { - const int slot_tokens = (int)(mem_slot_feats[s].size() / MD); - // Append spatial features - pd.prompt.insert(pd.prompt.end(), mem_slot_feats[s].begin(), mem_slot_feats[s].end()); + // attn @ V: need attn^T [N_q, N_kv] and V^T [HD, N_kv] + // ggml_mul_mat(attn^T, V) = (attn^T)^T @ V = attn @ V = [N_q, HD]... no. + // ggml_mul_mat(A, B) = A^T @ B where A=[K, M], B=[K, N] → [M, N] + // Want: output[q, d] = sum_k attn[q, k] * V[k, d] + // = (V^T @ attn^T)^T... let me think differently. + // attn_scores is [N_kv, N_q, NH*B]. For each head: + // attn[k, q] = attn_scores[k, q] (col q has the weights for query q) + // V3 is [HD, N_kv, NH*B]. + // Want: out[d, q] = sum_k V[d, k] * attn[k, q] = V @ attn + // = ggml_mul_mat? mul_mat(A, B) = A^T B with A=[K, M], B=[K, N] → [M, N] + // V has ne=[HD, N_kv, ...]. attn has ne=[N_kv, N_q, ...]. + // If A=V3 (ne0=HD, ne1=N_kv) and B=attn_scores (ne0=N_kv, ne1=N_q): + // Shared dim ne0: V3 ne0=HD ≠ attn ne0=N_kv. Mismatch! + // + // Need to transpose V: V^T is [N_kv, HD]. Then A=V^T, B=attn_scores. + // A ne0=N_kv, B ne0=N_kv → shared. A^T B = V @ attn → [HD, N_q]. ✓ + auto* VT = ggml_permute(ctx, V3, 1, 0, 2, 3); // [N_kv, HD, NH*B] + VT = ggml_cont(ctx, VT); + auto* out3 = ggml_mul_mat(ctx, VT, attn_scores); // [HD, N_q, NH*B] - // Build positional encoding: spatial PE + temporal PE - std::vector pos = mem_slot_pes[s]; // copy spatial PE - int tpos_idx = spatial_tpos[s]; - if (tpos_idx >= 0 && tpos_idx < hp.num_maskmem) { - int enc_idx = hp.num_maskmem - tpos_idx - 1; // Python: maskmem_tpos_enc[num_maskmem - tpos - 1] - for (int n = 0; n < slot_tokens; ++n) - for (int d = 0; d < MD; ++d) - pos[d + n * MD] += tpos_all[d + enc_idx * MD]; - } - pd.prompt_pos.insert(pd.prompt_pos.end(), pos.begin(), pos.end()); - } - pd.M_spatial = (int)mem_slot_feats.size() * HH; + // Reshape back to 4D: [HD, N_q, NH, B] + auto* out = ggml_reshape_4d(ctx, out3, HD, N_q, n_heads, B); + // Permute to [HD, NH, N_q, B] to match flash_attn_ext output convention + out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); +#endif - // 2. Object pointer tokens (split each D=256 pointer into D/MD=4 tokens of MD=64) - const int split = D / MD; // 4 + // Merge heads: [ID=HD*NH, N_q, B] + auto* merged = ggml_reshape_3d(ctx, out, ID, N_q, B); - // Read obj_ptr_tpos_proj weights for CPU-side matmul - std::vector tpos_w(D * MD), tpos_b(MD); - if (model.obj_ptr_tpos_w) { - sam3_read_f32(model.obj_ptr_tpos_w, tpos_w.data(), D * MD); - sam3_read_f32(model.obj_ptr_tpos_b, tpos_b.data(), MD); + // Debug: mark merged attention output for first SA call + static int _sa_merge_count = 0; + if (_sa_merge_count == 0 && N_q <= 16) { + ggml_set_name(merged, "dbg_sa0_merged"); + ggml_set_output(merged); } + _sa_merge_count++; - for (size_t p = 0; p < obj_ptrs.size(); ++p) { - int rel = ptr_tpos[p]; - if (rel < 0) continue; - - // 1D sine PE for temporal position (normalized by max_obj_ptrs - 1 = 15) - std::vector sine_pe(D); - sam3_get_1d_sine_pe(sine_pe.data(), (float)rel / 15.0f, D); - - // Project 256-dim sine PE → 64-dim via obj_ptr_tpos_proj (CPU matmul) - // W is [D=256, MD=64] in ggml; y[j] = sum_i W[i + j*D] * x[i] + b[j] - std::vector proj_pe(MD, 0.0f); - for (int j = 0; j < MD; ++j) { - float sum = tpos_b[j]; - for (int i = 0; i < D; ++i) - sum += tpos_w[i + j * D] * sine_pe[i]; - proj_pe[j] = sum; - } - - // Split pointer [256] → 4 tokens of [64] - // Python: ptr.view(1,1,4,64).permute(1,2,0,3).reshape(4,1,64) - const auto& ptr = obj_ptrs[p]; - for (int t = 0; t < split; ++t) { - for (int d = 0; d < MD; ++d) { - pd.prompt.push_back(ptr[t * MD + d]); - pd.prompt_pos.push_back(proj_pe[d]); - } - pd.num_obj_ptr_tokens++; - } - } + // Output projection + out = ggml_mul_mat(ctx, attn.out_w, merged); + out = ggml_add(ctx, out, attn.out_b); - pd.M_total = pd.M_spatial + pd.num_obj_ptr_tokens; - return pd; + return out; } /***************************************************************************** -** Memory attention (Phase 7, Step 7.2) +** SAM prompt encoder — graph building (Phase 6, Step 6.1) *****************************************************************************/ -// Build memory attention graph. -// curr_tokens: [D, N, 1] — current frame image tokens (flattened from 72x72 = 5184) -// src_pos: [D, N, 1] — sinusoidal PE for current tokens (pos_enc_at_input) -// prompt: [MD, M_total, 1] — concatenated memory features + split pointer tokens -// prompt_pos: [MD, M_total, 1] — positional encodings for prompt -// rope_freqs: [2, D/2, N] — axial RoPE frequencies for Q and self-attn K -// rope_k_freqs:[2, D/2, M_spatial] — repeated axial RoPE for cross-attn spatial K -// num_obj_ptr_tokens: number of pointer tokens at end of prompt (excluded from RoPE) -// Returns: conditioned tokens [D, N, 1] -static struct ggml_tensor* sam3_build_mem_attn_graph( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* curr_tokens, // [D, N, 1] - struct ggml_tensor* src_pos, // [D, N, 1] - struct ggml_tensor* prompt, // [MD, M_total, 1] - struct ggml_tensor* prompt_pos, // [MD, M_total, 1] - struct ggml_tensor* rope_freqs, // [2, D/2, N] - struct ggml_tensor* rope_k_freqs, // [2, D/2, M_spatial] or nullptr - int num_obj_ptr_tokens) { - const auto& ma = model.mem_attn; - const int D = model.hparams.neck_dim; // 256 - const int N = (int)curr_tokens->ne[1]; - const int M_total = (int)prompt->ne[1]; - const int M_spatial = M_total - num_obj_ptr_tokens; - - // pos_enc_at_input: x = curr + 0.1 * src_pos - auto* x = ggml_add(ctx, curr_tokens, ggml_scale(ctx, src_pos, 0.1f)); - ggml_set_name(x, "phase7_mem_attn_input"); - - for (int l = 0; l < (int)ma.layers.size(); ++l) { - const auto& ly = ma.layers[l]; - - // ── Self-attention with RoPE ────────────────────────────────────── - { - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm1_w, ly.norm1_b); - auto* q = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_q_w, x_norm), ly.sa_q_b); - auto* k = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_k_w, x_norm), ly.sa_k_b); - auto* v = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_v_w, x_norm), ly.sa_v_b); - - // Apply RoPE to Q and K (single-head, [D, N, 1]) - q = sam3_apply_rope(ctx, q, rope_freqs); - k = sam3_apply_rope(ctx, k, rope_freqs); +// Random Fourier positional encoding for a single (x, y) coordinate +// coords_norm: normalized to [0, 1], pe_gaussian: PyTorch row-major [2, 128] +// Output: [256] = [sin(128); cos(128)] +static void sam3_pe_encode_coord(float* out, float x_norm, float y_norm, + const float* pe_gauss, int num_pos_feats) { + // Map [0,1] → [-1,1] + float coords[2] = {2.0f * x_norm - 1.0f, 2.0f * y_norm - 1.0f}; - // Single-head attention (256-dim) - q = ggml_reshape_4d(ctx, q, D, 1, N, 1); - k = ggml_reshape_4d(ctx, k, D, 1, N, 1); - v = ggml_reshape_4d(ctx, v, D, 1, N, 1); - q = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); - k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 2, 1, 3)); - v = ggml_permute(ctx, v, 0, 2, 1, 3); + // coords @ pe_gaussian → [128] + for (int i = 0; i < num_pos_feats; ++i) { + float dot = coords[0] * pe_gauss[i] + + coords[1] * pe_gauss[num_pos_feats + i]; + dot *= 2.0f * (float)M_PI; + out[i] = sinf(dot); + out[i + num_pos_feats] = cosf(dot); + } +} - float scale = 1.0f / sqrtf((float)D); - auto* sa_out = ggml_flash_attn_ext(ctx, q, k, v, nullptr, scale, 0.0f, 0.0f); - sa_out = ggml_reshape_3d(ctx, sa_out, D, N, 1); - sa_out = ggml_add(ctx, ggml_mul_mat(ctx, ly.sa_out_w, sa_out), ly.sa_out_b); - x = ggml_add(ctx, x, sa_out); - sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_sa", l); - } +// Read SAM prompt encoder weights from GPU and cache them in state. +// Also pre-computes the dense PE grid and no-mask tiled embedding. +// These never change between PVS calls for the same model. +static void sam3_populate_pe_cache(sam3_state& state, const sam3_model& model) { + if (state.pe_cache_valid) return; - // ── Cross-attention to memory (kv_dim=64→256) with RoPE ────────── - { - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm2_w, ly.norm2_b); - auto* q = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_q_w, x_norm), ly.ca_q_b); + const int D = model.hparams.sam_embed_dim; // 256 + const int H = sam3_eff_feat_size(state, model.hparams); + const int num_pos_feats = D / 2; // 128 + const int pe_nel = 2 * num_pos_feats; // 256 + const auto& pe = model.sam_pe; - // K: project from (prompt + prompt_pos) — pos enc added before projection - auto* kv_with_pos = ggml_add(ctx, prompt, prompt_pos); - auto* k = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_k_w, kv_with_pos), ly.ca_k_b); - // V: project from prompt only (no pos enc) - auto* v = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_v_w, prompt), ly.ca_v_b); - - // Apply RoPE to Q - q = sam3_apply_rope(ctx, q, rope_freqs); - - // Apply RoPE to spatial K only (exclude num_obj_ptr_tokens tail keys) - if (M_spatial > 0 && rope_k_freqs) { - auto* k_spatial = ggml_cont(ctx, ggml_view_3d(ctx, k, - D, M_spatial, 1, k->nb[1], k->nb[2], 0)); - k_spatial = sam3_apply_rope(ctx, k_spatial, rope_k_freqs); - if (num_obj_ptr_tokens > 0) { - auto* k_ptr = ggml_cont(ctx, ggml_view_3d(ctx, k, - D, num_obj_ptr_tokens, 1, k->nb[1], k->nb[2], - (size_t)M_spatial * k->nb[1])); - k = ggml_concat(ctx, k_spatial, k_ptr, 1); - } else { - k = k_spatial; - } - } - - // Flash attention (single head) - q = ggml_reshape_4d(ctx, q, D, 1, N, 1); - k = ggml_reshape_4d(ctx, k, D, 1, M_total, 1); - v = ggml_reshape_4d(ctx, v, D, 1, M_total, 1); - q = ggml_cont(ctx, ggml_permute(ctx, q, 0, 2, 1, 3)); - k = ggml_cont(ctx, ggml_permute(ctx, k, 0, 2, 1, 3)); - v = ggml_permute(ctx, v, 0, 2, 1, 3); - - float scale = 1.0f / sqrtf((float)D); - auto* ca_out = ggml_flash_attn_ext(ctx, q, k, v, nullptr, scale, 0.0f, 0.0f); - ca_out = ggml_reshape_3d(ctx, ca_out, D, N, 1); - ca_out = ggml_add(ctx, ggml_mul_mat(ctx, ly.ca_out_w, ca_out), ly.ca_out_b); - x = ggml_add(ctx, x, ca_out); - sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_ca", l); - } - - // ── FFN ─────────────────────────────────────────────────────────── - { - auto* x_norm = sam3_layer_norm(ctx, x, ly.norm3_w, ly.norm3_b); - auto* ffn = ggml_add(ctx, ggml_mul_mat(ctx, ly.ffn_fc1_w, x_norm), ly.ffn_fc1_b); - ffn = ggml_relu(ctx, ffn); - ffn = ggml_add(ctx, ggml_mul_mat(ctx, ly.ffn_fc2_w, ffn), ly.ffn_fc2_b); - x = ggml_add(ctx, x, ffn); - sam3_name_tensorf(x, "phase7_mem_attn_layer%d_after_ffn", l); - } - } - - // Final norm - auto* norm_w = model.tensors.at("mem_attn.norm.weight"); - auto* norm_b = model.tensors.at("mem_attn.norm.bias"); - x = sam3_layer_norm(ctx, x, norm_w, norm_b); - ggml_set_name(x, "phase7_mem_attn_output"); - - return x; // [D, N, 1] -} - -/***************************************************************************** -** Object pointer extraction (Phase 7, Step 7.3) -*****************************************************************************/ - -// Extract object pointer from SAM output token via 3-layer MLP (CPU-side). -static void sam3_extract_obj_ptr_cpu( - const sam3_model& model, - const float* sam_token_data, // [D] - float obj_score, - float* out_ptr) // [D] -{ - const auto& hp = model.hparams; - const int D = hp.neck_dim; - - // SAM2/EdgeTAM with fixed_no_obj_ptr: blend projected ptr with no_obj_ptr - // based on presence score λ. - // SAM3 / SAM2 without fixed_no_obj_ptr: binary threshold. - if ((hp.is_sam2() || hp.is_edgetam()) && hp.fixed_no_obj_ptr) { - // λ = (obj_score > 0) ? 1.0 : 0.0 (hard threshold, not sigmoid) - float lambda = (obj_score > 0.0f) ? 1.0f : 0.0f; - - // Project token through MLP - std::vector h(D), tmp(D); - std::copy(sam_token_data, sam_token_data + D, h.data()); - for (int j = 0; j < 3; ++j) { - auto* w = model.obj_ptr_proj_w[j]; - auto* b = model.obj_ptr_proj_b[j]; - int nel_w = (int)(w->ne[0] * w->ne[1]); - std::vector w_data(nel_w), b_data(D); - sam3_read_f32(w, w_data.data(), nel_w); - sam3_read_f32(b, b_data.data(), D); - for (int o = 0; o < D; ++o) { - float sum = b_data[o]; - for (int i = 0; i < D; ++i) sum += w_data[o * D + i] * h[i]; - tmp[o] = (j < 2) ? std::max(0.0f, sum) : sum; - } - std::swap(h, tmp); - } - - // Blend: obj_ptr = λ * projected + (1-λ) * no_obj_ptr - std::vector no_ptr(D); - ggml_backend_tensor_get(model.no_obj_ptr, no_ptr.data(), 0, D * sizeof(float)); - for (int i = 0; i < D; ++i) - out_ptr[i] = lambda * h[i] + (1.0f - lambda) * no_ptr[i]; - return; - } - - // SAM3 / default path: binary threshold - if (obj_score <= 0.0f) { - ggml_backend_tensor_get(model.no_obj_ptr, out_ptr, 0, D * sizeof(float)); - return; - } - - std::vector h(D), tmp(D); - std::copy(sam_token_data, sam_token_data + D, h.data()); - - for (int j = 0; j < 3; ++j) { - auto* w = model.obj_ptr_proj_w[j]; - auto* b = model.obj_ptr_proj_b[j]; - - int nel_w = (int)(w->ne[0] * w->ne[1]); - std::vector w_data(nel_w); - sam3_read_f32(w, w_data.data(), nel_w); - - std::vector b_data(D); - sam3_read_f32(b, b_data.data(), D); - - for (int o = 0; o < D; ++o) { - float sum = b_data[o]; - for (int i = 0; i < D; ++i) { - sum += w_data[o * D + i] * h[i]; - } - tmp[o] = (j < 2) ? std::max(0.0f, sum) : sum; - } - std::swap(h, tmp); - } - std::copy(h.begin(), h.end(), out_ptr); -} - -/***************************************************************************** -** Tracker infrastructure (Phase 7, Step 7.4) -*****************************************************************************/ - -// Select memory frames for propagation (most recent + evenly spaced). -static std::vector sam3_select_memory_frames( - const std::vector& bank, - int max_slots) { - if ((int)bank.size() <= max_slots) { - std::vector all(bank.size()); - for (int i = 0; i < (int)bank.size(); ++i) all[i] = i; - return all; - } - std::vector selected; - selected.push_back(0); - selected.push_back((int)bank.size() - 1); - int remaining = max_slots - 2; - if (remaining > 0) { - float step = (float)(bank.size() - 2) / (remaining + 1); - for (int i = 0; i < remaining; ++i) { - int idx = 1 + (int)((i + 1) * step); - idx = std::min(idx, (int)bank.size() - 2); - selected.push_back(idx); - } - } - std::sort(selected.begin(), selected.end()); - selected.erase(std::unique(selected.begin(), selected.end()), selected.end()); - return selected; -} - -// Compute mask IoU between two binary masks. -static float sam3_mask_iou(const uint8_t* a, const uint8_t* b, int n) { - int inter = 0, uni = 0; - for (int i = 0; i < n; ++i) { - bool va = a[i] > 127; - bool vb = b[i] > 127; - if (va && vb) ++inter; - if (va || vb) ++uni; - } - return (uni > 0) ? (float)inter / uni : 0.0f; -} - -/***************************************************************************** -** Post-processing: hole filling and sprinkle removal (Phase 7, Step 7.8) -*****************************************************************************/ - -// Fill small holes in a binary mask using BFS connected components. -static void sam3_fill_holes(uint8_t* mask, int w, int h, int area_threshold) { - const int n = w * h; - std::vector labels(n, -1); - int next_label = 0; - std::vector component_sizes; - - for (int i = 0; i < n; ++i) { - if (mask[i] > 127 || labels[i] >= 0) continue; - int label = next_label++; - component_sizes.push_back(0); - std::vector queue; - queue.push_back(i); - labels[i] = label; - int head = 0; - bool touches_border = false; - while (head < (int)queue.size()) { - int p = queue[head++]; - component_sizes[label]++; - int px = p % w, py = p / w; - if (px == 0 || px == w - 1 || py == 0 || py == h - 1) touches_border = true; - int dx[] = {-1, 1, 0, 0}; - int dy[] = {0, 0, -1, 1}; - for (int d = 0; d < 4; ++d) { - int nx2 = px + dx[d], ny2 = py + dy[d]; - if (nx2 < 0 || nx2 >= w || ny2 < 0 || ny2 >= h) continue; - int ni = ny2 * w + nx2; - if (mask[ni] <= 127 && labels[ni] < 0) { - labels[ni] = label; - queue.push_back(ni); - } - } - } - if (touches_border) component_sizes[label] = area_threshold + 1; - } - for (int i = 0; i < n; ++i) { - if (mask[i] <= 127 && labels[i] >= 0 && component_sizes[labels[i]] <= area_threshold) { - mask[i] = 255; - } - } -} - -// Remove small foreground sprinkles. -static void sam3_remove_sprinkles(uint8_t* mask, int w, int h, int area_threshold) { - const int n = w * h; - std::vector labels(n, -1); - int next_label = 0; - std::vector component_sizes; - - for (int i = 0; i < n; ++i) { - if (mask[i] <= 127 || labels[i] >= 0) continue; - int label = next_label++; - component_sizes.push_back(0); - std::vector queue; - queue.push_back(i); - labels[i] = label; - int head = 0; - while (head < (int)queue.size()) { - int p = queue[head++]; - component_sizes[label]++; - int px = p % w, py = p / w; - int dx[] = {-1, 1, 0, 0}; - int dy[] = {0, 0, -1, 1}; - for (int d = 0; d < 4; ++d) { - int nx2 = px + dx[d], ny2 = py + dy[d]; - if (nx2 < 0 || nx2 >= w || ny2 < 0 || ny2 >= h) continue; - int ni = ny2 * w + nx2; - if (mask[ni] > 127 && labels[ni] < 0) { - labels[ni] = label; - queue.push_back(ni); - } - } - } - } - for (int i = 0; i < n; ++i) { - if (mask[i] > 127 && labels[i] >= 0 && component_sizes[labels[i]] <= area_threshold) { - mask[i] = 0; - } - } -} - -// Resolve overlapping masks: higher-scoring instances take priority. -static void sam3_resolve_overlaps(std::vector& dets) { - if (dets.size() <= 1) return; - const int w = dets[0].mask.width; - const int h = dets[0].mask.height; - if (w == 0 || h == 0) return; - std::sort(dets.begin(), dets.end(), [](const sam3_detection& a, const sam3_detection& b) { - return a.score > b.score; - }); - const int n = w * h; - for (int i = 0; i < n; ++i) { - bool claimed = false; - for (auto& det : dets) { - if (det.mask.data.empty()) continue; - if (claimed) { - det.mask.data[i] = 0; - } else if (det.mask.data[i] > 127) { - claimed = true; - } - } - } -} - -/***************************************************************************** -** Post-processing: NMS, bilinear interpolation, mask binarization -*****************************************************************************/ - -// Compute IoU between two boxes [x0, y0, x1, y1]. -static float sam3_box_iou(const sam3_box& a, const sam3_box& b) { - float x0 = std::max(a.x0, b.x0); - float y0 = std::max(a.y0, b.y0); - float x1 = std::min(a.x1, b.x1); - float y1 = std::min(a.y1, b.y1); - float inter = std::max(0.0f, x1 - x0) * std::max(0.0f, y1 - y0); - float area_a = (a.x1 - a.x0) * (a.y1 - a.y0); - float area_b = (b.x1 - b.x0) * (b.y1 - b.y0); - float uni = area_a + area_b - inter; - return (uni > 0.0f) ? inter / uni : 0.0f; -} - -// Non-maximum suppression on detections, sorted by score descending. -// Returns indices of kept detections. -static std::vector sam3_nms(const std::vector& dets, float iou_thresh) { - // Sort indices by score descending - std::vector indices(dets.size()); - for (int i = 0; i < (int)dets.size(); ++i) indices[i] = i; - std::sort(indices.begin(), indices.end(), [&](int a, int b) { - return dets[a].score > dets[b].score; - }); - - std::vector suppressed(dets.size(), false); - std::vector keep; - - for (int idx : indices) { - if (suppressed[idx]) continue; - keep.push_back(idx); - for (int j : indices) { - if (suppressed[j] || j == idx) continue; - if (sam3_box_iou(dets[idx].box, dets[j].box) > iou_thresh) { - suppressed[j] = true; - } - } - } - - return keep; -} - -// Bilinear interpolation of a flat mask [H_in * W_in] to [H_out * W_out]. -// Uses double for coordinate math to match PyTorch F.interpolate precision. -static std::vector sam3_bilinear_interpolate(const float* src, int src_w, int src_h, - int dst_w, int dst_h) { - std::vector dst(dst_w * dst_h); - const double sx = (double)src_w / dst_w; - const double sy = (double)src_h / dst_h; - - for (int y = 0; y < dst_h; ++y) { - double fy = (y + 0.5) * sy - 0.5; - fy = std::max(0.0, std::min(fy, (double)(src_h - 1))); - const int y0 = std::min((int)fy, src_h - 2); - const int y1 = y0 + 1; - const float wy = (float)(fy - y0); - - for (int x = 0; x < dst_w; ++x) { - double fx = (x + 0.5) * sx - 0.5; - fx = std::max(0.0, std::min(fx, (double)(src_w - 1))); - const int x0 = std::min((int)fx, src_w - 2); - const int x1 = x0 + 1; - const float wx = (float)(fx - x0); - - float v = (1 - wy) * ((1 - wx) * src[y0 * src_w + x0] + wx * src[y0 * src_w + x1]) + wy * ((1 - wx) * src[y1 * src_w + x0] + wx * src[y1 * src_w + x1]); - dst[y * dst_w + x] = v; - } - } - return dst; -} - -// Convert (cx, cy, w, h) in [0,1] to (x0, y0, x1, y1) in pixel coordinates. -static sam3_box sam3_cxcywh_to_xyxy(float cx, float cy, float w, float h, - int img_w, int img_h) { - sam3_box box; - box.x0 = (cx - w * 0.5f) * img_w; - box.y0 = (cy - h * 0.5f) * img_h; - box.x1 = (cx + w * 0.5f) * img_w; - box.y1 = (cy + h * 0.5f) * img_h; - // Clamp to image bounds - box.x0 = std::max(0.0f, std::min(box.x0, (float)img_w)); - box.y0 = std::max(0.0f, std::min(box.y0, (float)img_h)); - box.x1 = std::max(0.0f, std::min(box.x1, (float)img_w)); - box.y1 = std::max(0.0f, std::min(box.y1, (float)img_h)); - return box; -} - -/***************************************************************************** -** Image segmentation — PCS (text-prompted) -*****************************************************************************/ - -sam3_result sam3_segment_pcs(sam3_state& state, - const sam3_model& model, - const sam3_pcs_params& params) { - if (model.hparams.visual_only || model.hparams.is_sam2()) { - fprintf(stderr, "%s: ERROR: PCS not available on %s model\n", - __func__, model.hparams.is_sam2() ? "SAM2" : "visual-only"); - return sam3_result{}; - } - -#if SAM3_LOG_LEVEL >= 1 - auto t_start = std::chrono::high_resolution_clock::now(); -#endif - const auto& hp = model.hparams; - const int D = hp.neck_dim; // 256 - const int H = hp.n_img_embd(); // 72 - const int L = hp.text_ctx_len; // 32 - const int NQ = hp.ddec_num_queries; // 200 - const int N_spatial = H * H; // 5184 - sam3_result result; - - // ── Check that image has been encoded ──────────────────────────────── - if (!state.neck_det[0]) { - fprintf(stderr, "%s: image not encoded — call sam3_encode_image first\n", __func__); - return result; - } - - // ── Tokenize text prompt ───────────────────────────────────────────── - auto token_ids = sam3_tokenize(const_cast(model.tokenizer), - params.text_prompt, L); - if (token_ids.empty()) { - fprintf(stderr, "%s: failed to tokenize text prompt\n", __func__); - return result; - } - - SAM3_LOG(2, "%s: text='%s', %zu tokens\n", __func__, - params.text_prompt.c_str(), token_ids.size()); - - // ── Helper: run a sub-graph with its own context and allocator ────── - // Each stage below follows this exact pattern: - // 1. Create fresh ggml_context + graph + allocator - // 2. Create INPUT tensors (ggml_set_input) - // 3. Build the computation graph - // 4. Mark outputs (ggml_set_output) - // 5. Allocate → set input data → compute → read output data - // 6. Free allocator + context - // This ensures ZERO buffer sharing between stages. - - // ── Pre-compute shared CPU data used by multiple stages ───────────── - const int n_boxes = (int)(params.pos_exemplars.size() + params.neg_exemplars.size()); - const int N_geo = n_boxes + 1; // +1 for CLS - const int T = L + N_geo; // total prompt tokens (text + geometry) - - // Read image features and PE from state into CPU buffers (used by multiple stages) - std::vector img_feats_cpu(D * N_spatial); - std::vector img_pe_cpu(D * N_spatial); - ggml_backend_tensor_get(state.neck_det[2], img_feats_cpu.data(), 0, - D * N_spatial * sizeof(float)); - ggml_backend_tensor_get(state.neck_det_pe[2], img_pe_cpu.data(), 0, - D * N_spatial * sizeof(float)); - - // Pre-compute constant input vectors - std::vector sine_dim_t_cpu(64); - for (int i = 0; i < 64; ++i) - sine_dim_t_cpu[i] = 2.0f * (float)M_PI / powf(10000.0f, 2.0f * (float)i / 128.0f); - - std::vector rpb_coords_cpu(H); - for (int i = 0; i < H; ++i) rpb_coords_cpu[i] = (float)i / (float)H; - - // Build combined attention bias for the prompt: [T] = [L + N_geo] - // 0.0 for valid tokens, -1e9 for padding text tokens, 0.0 for geo tokens - std::vector combined_bias_cpu(T); - for (int i = 0; i < L; ++i) - combined_bias_cpu[i] = (token_ids[i] != 0) ? 0.0f : -1.0e9f; - for (int i = 0; i < N_geo; ++i) - combined_bias_cpu[L + i] = 0.0f; - - // Build text validity mask for DotProductScoring: [T] - int n_valid_tokens = 0; - for (int i = 0; i < L; ++i) - if (token_ids[i] != 0) ++n_valid_tokens; - n_valid_tokens += N_geo; - if (n_valid_tokens == 0) n_valid_tokens = 1; - float tvm_scale = (float)T / (float)n_valid_tokens; - std::vector text_valid_mask_cpu(T); - for (int i = 0; i < L; ++i) - text_valid_mask_cpu[i] = (token_ids[i] != 0) ? tvm_scale : 0.0f; - for (int i = 0; i < N_geo; ++i) - text_valid_mask_cpu[L + i] = tvm_scale; - - /* - ** ── SUB-GRAPH 1: Text Encoder ──────────────────────────────────── - */ - std::vector text_feats_cpu(D * L); - { - const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gp = {sz, nullptr, true}; - auto* ctx = ggml_init(gp); - - auto* inp = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, L); - ggml_set_name(inp, "text_token_ids"); - ggml_set_input(inp); - - auto* out = sam3_build_text_encoder_graph(ctx, inp, model); - ggml_set_output(out); - - auto* graph = ggml_new_graph_custom(ctx, 16384, false); - ggml_build_forward_expand(graph, out); - - auto* causal = ggml_get_tensor(ctx, "causal_mask"); - auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { - fprintf(stderr, "%s: text encoder alloc failed\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - if (inp->buffer) { - ggml_backend_tensor_set(inp, token_ids.data(), 0, L * sizeof(int32_t)); - } else { - fprintf(stderr, "%s: ERROR: text encoder input tensor has no buffer!\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - if (causal && causal->buffer) { - std::vector cm(L * L); - sam3_fill_causal_mask(cm.data(), L); - ggml_backend_tensor_set(causal, cm.data(), 0, L * L * sizeof(ggml_fp16_t)); - } - - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - ggml_backend_tensor_get(out, text_feats_cpu.data(), 0, D * L * sizeof(float)); - - ggml_gallocr_free(alloc); - ggml_free(ctx); - } - - SAM3_LOG(2, "%s: text encoder done\n", __func__); - - /* - ** ── SUB-GRAPH 2: Geometry Encoder ──────────────────────────────── - */ - std::vector geo_feats_cpu(D * N_geo); - { - const size_t sz = ggml_tensor_overhead() * 4096 + ggml_graph_overhead() * 2; - struct ggml_init_params gp = {sz, nullptr, true}; - auto* ctx = ggml_init(gp); - - auto* g_img = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(g_img, "geo_img"); - ggml_set_input(g_img); - auto* g_pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(g_pe, "geo_pe"); - ggml_set_input(g_pe); - - auto gr = sam3_build_geom_enc_graph(ctx, model, params, g_img, g_pe); - ggml_set_output(gr.geo_feats); - - auto* graph = ggml_new_graph_custom(ctx, 4096, false); - ggml_build_forward_expand(graph, gr.geo_feats); - - auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { - fprintf(stderr, "%s: geometry encoder alloc failed\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(g_img, img_feats_cpu.data(), 0, D * N_spatial * sizeof(float)); - ggml_backend_tensor_set(g_pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); - - // Pre-transformer input (CLS + optional box embeddings) - { - const float* feats_ptr = n_boxes > 0 ? img_feats_cpu.data() : nullptr; - auto geom_data = sam3_precompute_geom_input(model, params, feats_ptr, H, H); - auto* gi = ggml_get_tensor(ctx, "geom_post_final_proj"); - if (gi) ggml_backend_tensor_set(gi, geom_data.data(), 0, geom_data.size() * sizeof(float)); - } - - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - ggml_backend_tensor_get(gr.geo_feats, geo_feats_cpu.data(), 0, - D * N_geo * sizeof(float)); - - ggml_gallocr_free(alloc); - ggml_free(ctx); - } - - SAM3_LOG(2, "%s: geometry encoder done\n", __func__); - - // Build combined prompt on CPU: [text_feats, geo_feats] → [D * T] - std::vector combined_prompt_cpu(D * T); - memcpy(combined_prompt_cpu.data(), text_feats_cpu.data(), D * L * sizeof(float)); - memcpy(combined_prompt_cpu.data() + D * L, geo_feats_cpu.data(), D * N_geo * sizeof(float)); - - SAM3_LOG(2, "%s: starting fusion encoder\n", __func__); - /* - ** ── SUB-GRAPH 3: Fusion Encoder ────────────────────────────────── - */ - std::vector fenc_output_cpu(D * N_spatial); - { - const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gp = {sz, nullptr, true}; - auto* ctx = ggml_init(gp); - - auto* img = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(img, "fenc_img"); - ggml_set_input(img); - auto* pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(pe, "fenc_pe"); - ggml_set_input(pe); - auto* prompt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); - ggml_set_name(prompt, "fenc_prompt"); - ggml_set_input(prompt); - auto* bias = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); - ggml_set_name(bias, "fenc_bias"); - ggml_set_input(bias); - - auto* out = sam3_build_fenc_graph(ctx, model, img, prompt, pe, bias); - ggml_set_output(out); - - auto* graph = ggml_new_graph_custom(ctx, 16384, false); - ggml_build_forward_expand(graph, out); - - auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { - fprintf(stderr, "%s: fusion encoder alloc failed\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(img, img_feats_cpu.data(), 0, D * N_spatial * sizeof(float)); - ggml_backend_tensor_set(pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); - ggml_backend_tensor_set(prompt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); - ggml_backend_tensor_set(bias, combined_bias_cpu.data(), 0, T * sizeof(float)); - - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - ggml_backend_tensor_get(out, fenc_output_cpu.data(), 0, D * N_spatial * sizeof(float)); - - SAM3_LOG(2, "%s: fenc_out[0..4] = [%.6f, %.6f, %.6f, %.6f, %.6f]\n", - __func__, fenc_output_cpu[0], fenc_output_cpu[1], fenc_output_cpu[2], - fenc_output_cpu[3], fenc_output_cpu[4]); - - ggml_gallocr_free(alloc); - ggml_free(ctx); - } - - SAM3_LOG(2, "%s: fusion encoder done\n", __func__); - /* - ** ── SUB-GRAPH 4: DETR Decoder + Scoring ────────────────────────── - */ - std::vector scores_data(NQ); - std::vector boxes_data(4 * NQ); - std::vector queries_data(D * (NQ + 1)); // 201 = NQ + presence token - float presence_logit = 0.0f; - { - const size_t sz = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; - struct ggml_init_params gp = {sz, nullptr, true}; - auto* ctx = ggml_init(gp); - - auto* enc = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(enc, "ddec_enc"); - ggml_set_input(enc); - auto* pe = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(pe, "ddec_pe"); - ggml_set_input(pe); - auto* txt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); - ggml_set_name(txt, "ddec_text"); - ggml_set_input(txt); - auto* sdt = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, 64); - ggml_set_name(sdt, "sine_dim_t"); - ggml_set_input(sdt); - auto* rpb = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, H); - ggml_set_name(rpb, "rpb_coords"); - ggml_set_input(rpb); - auto* tab = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); - ggml_set_name(tab, "text_attn_bias"); - ggml_set_input(tab); - auto* tvm = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); - ggml_set_name(tvm, "text_valid_mask"); - ggml_set_input(tvm); - - auto dout = sam3_build_ddec_graph(ctx, model, enc, pe, txt, sdt, rpb, tab, tvm); - ggml_set_output(dout.class_scores); - ggml_set_output(dout.pred_boxes); - ggml_set_output(dout.presence_score); - ggml_set_output(dout.queries); - - auto* graph = ggml_new_graph_custom(ctx, 65536, false); - ggml_build_forward_expand(graph, dout.class_scores); - ggml_build_forward_expand(graph, dout.pred_boxes); - ggml_build_forward_expand(graph, dout.presence_score); - ggml_build_forward_expand(graph, dout.queries); - - auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { - fprintf(stderr, "%s: DETR decoder alloc failed\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(enc, fenc_output_cpu.data(), 0, D * N_spatial * sizeof(float)); - ggml_backend_tensor_set(pe, img_pe_cpu.data(), 0, D * N_spatial * sizeof(float)); - ggml_backend_tensor_set(txt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); - ggml_backend_tensor_set(sdt, sine_dim_t_cpu.data(), 0, 64 * sizeof(float)); - ggml_backend_tensor_set(rpb, rpb_coords_cpu.data(), 0, H * sizeof(float)); - ggml_backend_tensor_set(tab, combined_bias_cpu.data(), 0, T * sizeof(float)); - ggml_backend_tensor_set(tvm, text_valid_mask_cpu.data(), 0, T * sizeof(float)); - - // Presence token position encoding: zeros - auto* qpos = ggml_get_tensor(ctx, "ddec_query_pos_pres"); - if (qpos) { - std::vector z(D, 0.0f); - ggml_backend_tensor_set(qpos, z.data(), 0, D * sizeof(float)); - } - auto* rpbz = ggml_get_tensor(ctx, "rpb_pres_zeros"); - if (rpbz) { - int n = (int)(rpbz->ne[0] * rpbz->ne[1] * rpbz->ne[2] * rpbz->ne[3]); - std::vector z(n, 0.0f); - ggml_backend_tensor_set(rpbz, z.data(), 0, n * sizeof(float)); - } - - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_get(dout.class_scores, scores_data.data(), 0, NQ * sizeof(float)); - ggml_backend_tensor_get(dout.pred_boxes, boxes_data.data(), 0, 4 * NQ * sizeof(float)); - ggml_backend_tensor_get(dout.presence_score, &presence_logit, 0, sizeof(float)); - ggml_backend_tensor_get(dout.queries, queries_data.data(), 0, - D * (NQ + 1) * sizeof(float)); - - ggml_gallocr_free(alloc); - ggml_free(ctx); - } - - float presence_prob = 1.0f / (1.0f + expf(-presence_logit)); - - SAM3_LOG(2, "%s: DETR decoder done\n", __func__); - /* - ** ── SUB-GRAPH 5: Segmentation Head ─────────────────────────────── - */ - const int mask_hw = H * 4; // 288 for SAM3 - std::vector all_masks(NQ * mask_hw * mask_hw); - { - const size_t sz = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gp = {sz, nullptr, true}; - auto* ctx = ggml_init(gp); - - auto* enc_h = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_spatial, 1); - ggml_set_name(enc_h, "seg_enc"); - ggml_set_input(enc_h); - - // FPN features at 3 scales — create fresh inputs - const int H0 = H * 4, H1 = H * 2; - auto* fpn0 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H0, H0, 1); - ggml_set_name(fpn0, "seg_fpn0"); - ggml_set_input(fpn0); - auto* fpn1 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H1, H1, 1); - ggml_set_name(fpn1, "seg_fpn1"); - ggml_set_input(fpn1); - auto* fpn2 = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(fpn2, "seg_fpn2"); - ggml_set_input(fpn2); - struct ggml_tensor* fpn_feats[3] = {fpn0, fpn1, fpn2}; - - // Object queries (skip presence token at index 0 → start at index 1) - auto* oq = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, NQ, 1); - ggml_set_name(oq, "seg_queries"); - ggml_set_input(oq); - - auto* txt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, T, 1); - ggml_set_name(txt, "seg_text"); - ggml_set_input(txt); - auto* tab = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, T, 1, 1); - ggml_set_name(tab, "seg_bias"); - ggml_set_input(tab); - - auto* out = sam3_build_seg_head_graph(ctx, model, enc_h, fpn_feats, - oq, txt, tab); - ggml_set_output(out); - - auto* graph = ggml_new_graph_custom(ctx, 32768, false); - ggml_build_forward_expand(graph, out); - - SAM3_LOG(2, "%s: seg head graph: %d nodes\n", __func__, ggml_graph_n_nodes(graph)); - - auto* alloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(alloc, graph) || !ggml_gallocr_alloc_graph(alloc, graph)) { - fprintf(stderr, "%s: segmentation head alloc failed\n", __func__); - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(enc_h, fenc_output_cpu.data(), 0, - D * N_spatial * sizeof(float)); - - // Copy FPN features from state - { - size_t s0 = (size_t)D * H0 * H0 * sizeof(float); - size_t s1 = (size_t)D * H1 * H1 * sizeof(float); - size_t s2 = (size_t)D * H * H * sizeof(float); - std::vector b0(D * H0 * H0), b1(D * H1 * H1), b2(D * H * H); - ggml_backend_tensor_get(state.neck_det[0], b0.data(), 0, s0); - if (fpn0->buffer) ggml_backend_tensor_set(fpn0, b0.data(), 0, s0); - ggml_backend_tensor_get(state.neck_det[1], b1.data(), 0, s1); - if (fpn1->buffer) ggml_backend_tensor_set(fpn1, b1.data(), 0, s1); - ggml_backend_tensor_get(state.neck_det[2], b2.data(), 0, s2); - if (fpn2->buffer) ggml_backend_tensor_set(fpn2, b2.data(), 0, s2); - } - - // Object queries: extract from DETR queries (skip presence token at slot 0) - // queries_data is flat [D * 201], presence token is at positions [0..D-1] - // Object queries start at position [D..D*(NQ+1)-1] - ggml_backend_tensor_set(oq, queries_data.data() + D, 0, D * NQ * sizeof(float)); - - ggml_backend_tensor_set(txt, combined_prompt_cpu.data(), 0, D * T * sizeof(float)); - ggml_backend_tensor_set(tab, combined_bias_cpu.data(), 0, T * sizeof(float)); - - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(alloc); - ggml_free(ctx); - return result; - } - ggml_backend_tensor_get(out, all_masks.data(), 0, all_masks.size() * sizeof(float)); - - ggml_gallocr_free(alloc); - ggml_free(ctx); - } - - /* - ** ── Post-processing: thresholding + NMS + mask resize ──────────── - */ - std::vector dets; - for (int q = 0; q < NQ; ++q) { - float class_prob = 1.0f / (1.0f + expf(-scores_data[q])); - float score = class_prob * presence_prob; - if (score < params.score_threshold) continue; - - sam3_detection det; - float cx = boxes_data[0 + q * 4]; - float cy = boxes_data[1 + q * 4]; - float bw = boxes_data[2 + q * 4]; - float bh = boxes_data[3 + q * 4]; - - det.box = sam3_cxcywh_to_xyxy(cx, cy, bw, bh, state.orig_width, state.orig_height); - det.score = score; - - const float* mask_ptr = all_masks.data() + q * mask_hw * mask_hw; - auto mask_resized = sam3_bilinear_interpolate(mask_ptr, mask_hw, mask_hw, - state.orig_width, state.orig_height); - det.mask.width = state.orig_width; - det.mask.height = state.orig_height; - det.mask.data.resize(state.orig_width * state.orig_height); - for (int i = 0; i < (int)mask_resized.size(); ++i) - det.mask.data[i] = (mask_resized[i] > 0.0f) ? 255 : 0; - det.mask.iou_score = score; - - dets.push_back(std::move(det)); - } - - SAM3_LOG(2, "%s: %zu detections above threshold %.2f (presence=%.3f, logit=%.3f)\n", - __func__, dets.size(), params.score_threshold, presence_prob, presence_logit); - - auto keep = sam3_nms(dets, params.nms_threshold); - for (int i = 0; i < (int)keep.size(); ++i) { - dets[keep[i]].instance_id = i + 1; - result.detections.push_back(std::move(dets[keep[i]])); - } - - SAM3_LOG(2, "%s: %zu detections after NMS\n", __func__, result.detections.size()); - -#if SAM3_LOG_LEVEL >= 1 - auto t_end = std::chrono::high_resolution_clock::now(); - double total_ms = std::chrono::duration(t_end - t_start).count(); - SAM3_LOG(1, "%s: completed in %.1f ms\n", __func__, total_ms); -#endif - - return result; -} - -/***************************************************************************** -** SAM attention helper (separate Q, K, V weight/bias) -*****************************************************************************/ - -static struct ggml_tensor* sam3_sam_attention( - struct ggml_context* ctx, - struct ggml_tensor* q_in, // [D, N_q, B] - struct ggml_tensor* k_in, // [D, N_kv, B] - struct ggml_tensor* v_in, // [D, N_kv, B] - const sam3_sam_attn& attn, - int n_heads) { - const int64_t N_q = q_in->ne[1]; - const int64_t B = q_in->ne[2]; - const int64_t N_kv = k_in->ne[1]; - - // Project - auto* Q = ggml_add(ctx, ggml_mul_mat(ctx, attn.q_w, q_in), attn.q_b); - auto* K = ggml_add(ctx, ggml_mul_mat(ctx, attn.k_w, k_in), attn.k_b); - auto* V = ggml_add(ctx, ggml_mul_mat(ctx, attn.v_w, v_in), attn.v_b); - - // Debug: mark projections for the first SA call (N_q=8 tokens, block 0) - static int _sa_call_count = 0; - if (_sa_call_count == 0 && N_q <= 16) { - ggml_set_name(Q, "dbg_sa0_Q_proj"); - ggml_set_output(Q); - ggml_set_name(V, "dbg_sa0_V_proj"); - ggml_set_output(V); - } - _sa_call_count++; - - // internal_dim = out_proj cols = attn.q_w->ne[1] - const int64_t ID = attn.q_w->ne[1]; - const int64_t HD = ID / n_heads; - - // Reshape to multi-head: [HD, N, NH, B] - Q = ggml_reshape_4d(ctx, Q, HD, n_heads, N_q, B); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); // [HD, N_q, NH, B] - - K = ggml_reshape_4d(ctx, K, HD, n_heads, N_kv, B); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); // [HD, N_kv, NH, B] - - V = ggml_reshape_4d(ctx, V, HD, n_heads, N_kv, B); - V = ggml_cont(ctx, ggml_permute(ctx, V, 0, 2, 1, 3)); // [HD, N_kv, NH, B] contiguous - - // Attention - float scale = 1.0f / sqrtf((float)HD); - auto* out = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, scale, 0.0f, 0.0f); - // out: [HD, NH, N_q, B] (flash_attn_ext swaps dims 1,2 vs input) - -#if 0 // Manual SDPA (for debugging only) - auto* Q3 = ggml_reshape_3d(ctx, Q, HD, N_q, n_heads * B); - auto* K3 = ggml_reshape_3d(ctx, K, HD, N_kv, n_heads * B); - auto* V3 = ggml_reshape_3d(ctx, V, HD, N_kv, n_heads * B); - // QK^T: ggml_mul_mat(K, Q) → K^T @ Q → [N_kv, N_q, NH*B] - auto* attn_scores = ggml_mul_mat(ctx, K3, Q3); - attn_scores = ggml_scale(ctx, attn_scores, scale); - attn_scores = ggml_soft_max(ctx, attn_scores); - - // attn @ V: need attn^T [N_q, N_kv] and V^T [HD, N_kv] - // ggml_mul_mat(attn^T, V) = (attn^T)^T @ V = attn @ V = [N_q, HD]... no. - // ggml_mul_mat(A, B) = A^T @ B where A=[K, M], B=[K, N] → [M, N] - // Want: output[q, d] = sum_k attn[q, k] * V[k, d] - // = (V^T @ attn^T)^T... let me think differently. - // attn_scores is [N_kv, N_q, NH*B]. For each head: - // attn[k, q] = attn_scores[k, q] (col q has the weights for query q) - // V3 is [HD, N_kv, NH*B]. - // Want: out[d, q] = sum_k V[d, k] * attn[k, q] = V @ attn - // = ggml_mul_mat? mul_mat(A, B) = A^T B with A=[K, M], B=[K, N] → [M, N] - // V has ne=[HD, N_kv, ...]. attn has ne=[N_kv, N_q, ...]. - // If A=V3 (ne0=HD, ne1=N_kv) and B=attn_scores (ne0=N_kv, ne1=N_q): - // Shared dim ne0: V3 ne0=HD ≠ attn ne0=N_kv. Mismatch! - // - // Need to transpose V: V^T is [N_kv, HD]. Then A=V^T, B=attn_scores. - // A ne0=N_kv, B ne0=N_kv → shared. A^T B = V @ attn → [HD, N_q]. ✓ - auto* VT = ggml_permute(ctx, V3, 1, 0, 2, 3); // [N_kv, HD, NH*B] - VT = ggml_cont(ctx, VT); - auto* out3 = ggml_mul_mat(ctx, VT, attn_scores); // [HD, N_q, NH*B] - - // Reshape back to 4D: [HD, N_q, NH, B] - auto* out = ggml_reshape_4d(ctx, out3, HD, N_q, n_heads, B); - // Permute to [HD, NH, N_q, B] to match flash_attn_ext output convention - out = ggml_cont(ctx, ggml_permute(ctx, out, 0, 2, 1, 3)); -#endif - - // Merge heads: [ID=HD*NH, N_q, B] - auto* merged = ggml_reshape_3d(ctx, out, ID, N_q, B); - - // Debug: mark merged attention output for first SA call - static int _sa_merge_count = 0; - if (_sa_merge_count == 0 && N_q <= 16) { - ggml_set_name(merged, "dbg_sa0_merged"); - ggml_set_output(merged); - } - _sa_merge_count++; - - // Output projection - out = ggml_mul_mat(ctx, attn.out_w, merged); - out = ggml_add(ctx, out, attn.out_b); - - return out; -} - -/***************************************************************************** -** SAM prompt encoder — graph building (Phase 6, Step 6.1) -*****************************************************************************/ - -// Random Fourier positional encoding for a single (x, y) coordinate -// coords_norm: normalized to [0, 1], pe_gaussian: PyTorch row-major [2, 128] -// Output: [256] = [sin(128); cos(128)] -static void sam3_pe_encode_coord(float* out, float x_norm, float y_norm, - const float* pe_gauss, int num_pos_feats) { - // Map [0,1] → [-1,1] - float coords[2] = {2.0f * x_norm - 1.0f, 2.0f * y_norm - 1.0f}; - - // coords @ pe_gaussian → [128] - for (int i = 0; i < num_pos_feats; ++i) { - float dot = coords[0] * pe_gauss[i] + - coords[1] * pe_gauss[num_pos_feats + i]; - dot *= 2.0f * (float)M_PI; - out[i] = sinf(dot); - out[i + num_pos_feats] = cosf(dot); - } -} - -// Read SAM prompt encoder weights from GPU and cache them in state. -// Also pre-computes the dense PE grid and no-mask tiled embedding. -// These never change between PVS calls for the same model. -static void sam3_populate_pe_cache(sam3_state& state, const sam3_model& model) { - if (state.pe_cache_valid) return; - - const int D = model.hparams.sam_embed_dim; // 256 - const int H = sam3_eff_feat_size(state, model.hparams); - const int num_pos_feats = D / 2; // 128 - const int pe_nel = 2 * num_pos_feats; // 256 - const auto& pe = model.sam_pe; - - state.pe_gauss_cache.resize(pe_nel); - if (pe.pe_gaussian->type == GGML_TYPE_F16) { - std::vector tmp(pe_nel); - ggml_backend_tensor_get(pe.pe_gaussian, tmp.data(), 0, pe_nel * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), state.pe_gauss_cache.data(), pe_nel); - } else { - ggml_backend_tensor_get(pe.pe_gaussian, state.pe_gauss_cache.data(), 0, pe_nel * sizeof(float)); - } - - for (int i = 0; i < 4; ++i) { - if (pe.point_embed[i]->type == GGML_TYPE_F16) { - std::vector tmp(D); - ggml_backend_tensor_get(pe.point_embed[i], tmp.data(), 0, D * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), state.point_emb_cache[i], D); - } else { - ggml_backend_tensor_get(pe.point_embed[i], state.point_emb_cache[i], 0, D * sizeof(float)); - } - } - - if (pe.not_a_point_embed->type == GGML_TYPE_F16) { - std::vector tmp(D); - ggml_backend_tensor_get(pe.not_a_point_embed, tmp.data(), 0, D * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), state.not_a_point_cache, D); - } else { - ggml_backend_tensor_get(pe.not_a_point_embed, state.not_a_point_cache, 0, D * sizeof(float)); - } - - if (pe.no_mask_embed->type == GGML_TYPE_F16) { - std::vector tmp(D); - ggml_backend_tensor_get(pe.no_mask_embed, tmp.data(), 0, D * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), state.no_mask_emb_cache, D); - } else { - ggml_backend_tensor_get(pe.no_mask_embed, state.no_mask_emb_cache, 0, D * sizeof(float)); - } - - state.dense_pe_cache.resize(D * H * H); - for (int row = 0; row < H; ++row) { - for (int col = 0; col < H; ++col) { - float x_norm = ((float)col + 0.5f) / (float)H; - float y_norm = ((float)row + 0.5f) / (float)H; - float pe_vec[256]; - sam3_pe_encode_coord(pe_vec, x_norm, y_norm, - state.pe_gauss_cache.data(), num_pos_feats); - for (int d = 0; d < D; ++d) - state.dense_pe_cache[d + col * D + row * D * H] = pe_vec[d]; - } - } - - state.dense_nomask_cache.resize(D * H * H); - for (int row = 0; row < H; ++row) { - for (int col = 0; col < H; ++col) { - for (int d = 0; d < D; ++d) - state.dense_nomask_cache[d + col * D + row * D * H] = state.no_mask_emb_cache[d]; - } - } - - state.pe_cache_valid = true; - fprintf(stderr, "%s: PE cache populated (%d embeddings, %.1f KB dense grids)\n", - __func__, pe_nel, 2.0f * D * H * H * sizeof(float) / 1024.0f); -} - -// Match the official image predictor prompt ordering: -// box corners (if any) → user points → trailing padding point. -// Even when a box is present, SAM keeps the final padding token because boxes -// are merged into the point stream before calling PromptEncoder(points=..., boxes=None). -static void sam3_collect_pvs_prompt_tokens(const sam3_pvs_params& params, - std::vector& all_coords, - std::vector& all_labels) { - all_coords.clear(); - all_labels.clear(); - - if (params.use_box) { - all_coords.push_back(params.box.x0); - all_coords.push_back(params.box.y0); - all_labels.push_back(2); - - all_coords.push_back(params.box.x1); - all_coords.push_back(params.box.y1); - all_labels.push_back(3); - } - - for (const auto& pt : params.pos_points) { - all_coords.push_back(pt.x); - all_coords.push_back(pt.y); - all_labels.push_back(1); - } - for (const auto& pt : params.neg_points) { - all_coords.push_back(pt.x); - all_coords.push_back(pt.y); - all_labels.push_back(0); - } - - all_coords.push_back(0.0f); - all_coords.push_back(0.0f); - all_labels.push_back(-1); -} - -// Build sparse and dense embeddings from point/box prompts -// sparse_out: [D, N_pts, 1] where N_pts = n_pos + n_neg + (use_box ? 2 : 0) + pad -// dense_out: [D, H, H, 1] (no-mask default or mask downsample) -struct sam3_pe_result { - struct ggml_tensor* sparse; // [D, N_pts, 1] - struct ggml_tensor* dense; // [D, H, H, 1] - struct ggml_tensor* image_pe; // [D, H, H, 1] — dense positional encoding grid - int n_tokens; -}; - -static sam3_pe_result sam3_build_sam_pe( - struct ggml_context* ctx, - const sam3_pvs_params& params, - int embed_dim, int feat_size) { - const int D = embed_dim; // 256 - const int H = feat_size; // 72 - - int N_pts = (int)(params.pos_points.size() + params.neg_points.size()) + 1; // pad - if (params.use_box) N_pts += 2; - - auto* sparse = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_pts, 1); - ggml_set_name(sparse, "sam_pe_sparse"); - ggml_set_input(sparse); - - auto* image_pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(image_pe, "sam_pe_image_pe"); - ggml_set_input(image_pe); - - auto* dense = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(dense, "sam_pe_dense"); - ggml_set_input(dense); - - sam3_pe_result result; - result.sparse = sparse; - result.dense = dense; - result.image_pe = image_pe; - result.n_tokens = N_pts; - return result; -} - -/***************************************************************************** -** SAM mask decoder — graph building (Phase 6, Step 6.2) -*****************************************************************************/ - -// TwoWayAttentionBlock forward -static void sam3_twoway_block_forward( - struct ggml_context* ctx, - struct ggml_tensor*& queries, // [D, N_q, B] — modified in place - struct ggml_tensor*& keys, // [D, N_kv, B] — modified in place - struct ggml_tensor* query_pe, // [D, N_q, B] - struct ggml_tensor* key_pe, // [D, N_kv, B] - const sam3_twoway_block& blk, - int n_heads, - bool skip_first_layer_pe) { - // 1. Self-attention on queries - if (skip_first_layer_pe) { - // Python: queries = self.self_attn(q=queries, k=queries, v=queries) - // No residual connection when skipping first layer PE - queries = sam3_sam_attention(ctx, queries, queries, queries, blk.self_attn, n_heads); - } else { - auto* q = ggml_add(ctx, queries, query_pe); - auto* attn_out = sam3_sam_attention(ctx, q, q, queries, blk.self_attn, n_heads); - queries = ggml_add(ctx, queries, attn_out); - } - queries = sam3_layer_norm(ctx, queries, blk.norm1_w, blk.norm1_b); - if (skip_first_layer_pe) { - ggml_set_name(queries, "dbg_twoway_skip_sa_norm"); - ggml_set_output(queries); - } - - // 2. Cross-attention: tokens attending to image - { - auto* q = ggml_add(ctx, queries, query_pe); - auto* k = ggml_add(ctx, keys, key_pe); - auto* attn_out = sam3_sam_attention(ctx, q, k, keys, blk.ca_tok2img, n_heads); - queries = ggml_add(ctx, queries, attn_out); - queries = sam3_layer_norm(ctx, queries, blk.norm2_w, blk.norm2_b); - } - if (skip_first_layer_pe) { - ggml_set_name(queries, "dbg_twoway_skip_ca_tok2img"); - ggml_set_output(queries); - } - - // 3. MLP on queries (ReLU activation) - { - auto* mlp = ggml_mul_mat(ctx, blk.mlp_fc1_w, queries); - mlp = ggml_add(ctx, mlp, blk.mlp_fc1_b); - mlp = ggml_relu(ctx, mlp); - mlp = ggml_mul_mat(ctx, blk.mlp_fc2_w, mlp); - mlp = ggml_add(ctx, mlp, blk.mlp_fc2_b); - queries = ggml_add(ctx, queries, mlp); - queries = sam3_layer_norm(ctx, queries, blk.norm3_w, blk.norm3_b); - } - if (skip_first_layer_pe) { - ggml_set_name(queries, "dbg_twoway_skip_mlp"); - ggml_set_output(queries); - } - - // 4. Cross-attention: image attending to tokens - { - auto* q = ggml_add(ctx, queries, query_pe); - auto* k = ggml_add(ctx, keys, key_pe); - // Note: q and k are swapped — image (k) attends to tokens (q) - auto* attn_out = sam3_sam_attention(ctx, k, q, queries, blk.ca_img2tok, n_heads); - keys = ggml_add(ctx, keys, attn_out); - keys = sam3_layer_norm(ctx, keys, blk.norm4_w, blk.norm4_b); - } - if (skip_first_layer_pe) { - ggml_set_name(keys, "dbg_twoway_skip_img2tok"); - ggml_set_output(keys); - } -} - -// MLP forward: N layers with ReLU (except last), optional sigmoid on last -static struct ggml_tensor* sam3_mlp_forward( - struct ggml_context* ctx, - struct ggml_tensor* x, - struct ggml_tensor* const* weights, - struct ggml_tensor* const* biases, - int n_layers, - bool sigmoid_output = false) { - for (int i = 0; i < n_layers; ++i) { - x = ggml_mul_mat(ctx, weights[i], x); - x = ggml_add(ctx, x, biases[i]); - if (i < n_layers - 1) { - x = ggml_relu(ctx, x); - } - } - if (sigmoid_output) { - x = ggml_sigmoid(ctx, x); - } - return x; -} - -// Full SAM mask decoder graph -// Inputs: -// image_feats: [D, H, H, 1] — tracker neck features (scale 2 = 72×72) -// image_pe: [D, H, H, 1] — dense positional encoding -// sparse_emb: [D, N_pts, 1] — sparse prompt embeddings -// dense_emb: [D, H, H, 1] — dense prompt embeddings (no_mask default) -// feat_s0: [D, H0, H0, 1] — high-res features (scale 0 = 288×288) -// feat_s1: [D, H1, H1, 1] — mid-res features (scale 1 = 144×144) -// Outputs: sam3_dec_result with masks, iou_pred, obj_score, sam_token_out -struct sam3_dec_result { - struct ggml_tensor* masks; // [288*288, N_masks, 1] - struct ggml_tensor* iou_pred; // [N_masks, 1] - struct ggml_tensor* obj_score; // [1, 1] - struct ggml_tensor* sam_token; // [D, 1] — for object pointer - struct ggml_tensor* mask_tokens; // [D, N_masks, 1] — raw SAM mask tokens -}; - -static sam3_dec_result sam3_build_sam_dec_graph( - struct ggml_context* ctx, - const sam3_model& model, - struct ggml_tensor* image_feats, // [D, H, H, 1] - struct ggml_tensor* image_pe, // [D, H, H, 1] - struct ggml_tensor* sparse_emb, // [D, N_pts, 1] - struct ggml_tensor* dense_emb, // [D, H, H, 1] - struct ggml_tensor* feat_s0, // [D, H*4, H*4, 1] high-res - struct ggml_tensor* feat_s1, // [D, H*2, H*2, 1] mid-res - int eff_feat_size = 0) -{ - const auto& dec = model.sam_dec; - const auto& hp = model.hparams; - const int D = hp.sam_embed_dim; // 256 - const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); - const int N_pts = (int)sparse_emb->ne[1]; - const int n_heads = 8; // SAM uses 8 heads - const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 - - // ── Concatenate output tokens ──────────────────────────────────────── - // When pred_obj_scores=True: [obj_score(1,D), iou(1,D), masks(4,D)] = 6 tokens - // When pred_obj_scores=False: [iou(1,D), masks(4,D)] = 5 tokens (older SAM2) - const bool has_obj_score = (dec.obj_score_token != nullptr); - const int n_special = (has_obj_score ? 6 : 5); - - struct ggml_tensor* output_tokens; - if (has_obj_score) { - output_tokens = ggml_concat(ctx, dec.obj_score_token, dec.iou_token, 1); - } else { - output_tokens = ggml_reshape_2d(ctx, dec.iou_token, D, 1); - } - output_tokens = ggml_concat(ctx, output_tokens, dec.mask_tokens, 1); - output_tokens = ggml_reshape_3d(ctx, output_tokens, D, n_special, 1); - auto* tokens = ggml_concat(ctx, output_tokens, sparse_emb, 1); - ggml_set_name(tokens, "sam_dec_tokens_initial"); - ggml_set_output(tokens); - - const int N_tok = 6 + N_pts; - - auto* src = ggml_add(ctx, image_feats, dense_emb); - src = ggml_reshape_3d(ctx, src, D, H * H, 1); - auto* pos_src = ggml_reshape_3d(ctx, image_pe, D, H * H, 1); - - auto* queries = tokens; - auto* keys = src; - auto* query_pe = tokens; // query PE = initial point embedding - auto* key_pe = pos_src; - - for (int i = 0; i < hp.sam_dec_depth; ++i) { - sam3_twoway_block_forward(ctx, queries, keys, query_pe, key_pe, - dec.twoway_blocks[i], n_heads, - /*skip_first_layer_pe=*/(i == 0)); - sam3_name_tensorf(queries, "sam_dec_block%d_queries", i); - ggml_set_output(queries); - sam3_name_tensorf(keys, "sam_dec_block%d_keys", i); - ggml_set_output(keys); - } - - // Final attention: tokens → image - { - auto* q = ggml_add(ctx, queries, query_pe); - auto* k = ggml_add(ctx, keys, key_pe); - auto* attn_out = sam3_sam_attention(ctx, q, k, keys, dec.final_attn, n_heads); - queries = ggml_add(ctx, queries, attn_out); - queries = sam3_layer_norm(ctx, queries, dec.final_norm_w, dec.final_norm_b); - ggml_set_name(queries, "sam_dec_final_queries"); - } - - // Debug: mark transformer outputs - ggml_set_name(queries, "dbg_dec_queries_out"); - ggml_set_output(queries); - ggml_set_name(keys, "dbg_dec_keys_out"); - ggml_set_output(keys); - - // ── Extract output tokens ──────────────────────────────────────────── - // With pred_obj_scores=True (6 tokens): obj(0), iou(1), masks(2..5) - // With pred_obj_scores=False (5 tokens): iou(0), masks(1..4) - const int s = has_obj_score ? 1 : 0; - auto* iou_token_out = ggml_view_3d(ctx, queries, D, 1, 1, - queries->nb[1], queries->nb[2], - s * queries->nb[1]); - iou_token_out = ggml_cont(ctx, iou_token_out); // [D, 1, 1] - - auto* mask_tokens_out = ggml_view_3d(ctx, queries, D, num_mask_tokens, 1, - queries->nb[1], queries->nb[2], - (s + 1) * queries->nb[1]); - mask_tokens_out = ggml_cont(ctx, mask_tokens_out); // [D, 4, 1] - ggml_set_name(mask_tokens_out, "sam_dec_mask_tokens"); - - struct ggml_tensor* obj_in = nullptr; - if (has_obj_score) { - obj_in = ggml_view_3d(ctx, queries, D, 1, 1, - queries->nb[1], queries->nb[2], 0); - obj_in = ggml_cont(ctx, obj_in); // [D, 1, 1] - } - - // SAM output token = first mask token, used for object pointer - auto* sam_token = ggml_view_2d(ctx, queries, D, 1, - queries->nb[1], (s + 1) * queries->nb[1]); - sam_token = ggml_cont(ctx, sam_token); // [D, 1] - ggml_set_name(sam_token, "sam_dec_sam_token"); - - // Upscale: [D, H*H, 1] → ConvTranspose → high-res masks - auto* src_img = ggml_reshape_4d(ctx, keys, D, H, H, 1); - src_img = ggml_cont(ctx, ggml_permute(ctx, src_img, 2, 0, 1, 3)); - - auto* up1 = ggml_conv_transpose_2d_p0(ctx, sam3_conv_transpose_weight(ctx, dec.up1_w), src_img, 2); - up1 = ggml_add(ctx, up1, ggml_reshape_4d(ctx, dec.up1_b, 1, 1, ggml_nelements(dec.up1_b), 1)); - - auto* fs1 = ggml_cont(ctx, ggml_permute(ctx, feat_s1, 2, 0, 1, 3)); - auto* hs1 = ggml_conv_2d_sk_p0(ctx, dec.conv_s1_w, fs1); // [W, H, 64, B] - hs1 = ggml_add(ctx, hs1, ggml_reshape_4d(ctx, dec.conv_s1_b, 1, 1, 64, 1)); - ggml_set_name(hs1, "sam_dec_feat_s1_proj"); - - // Python: act1(ln1(dc1(src) + feat_s1)) — add before LayerNorm - up1 = ggml_add(ctx, up1, hs1); - up1 = ggml_cont(ctx, ggml_permute(ctx, up1, 1, 2, 0, 3)); - up1 = sam3_layer_norm_2d(ctx, up1, dec.up1_norm_w, dec.up1_norm_b); - - up1 = ggml_gelu_erf(ctx, up1); - - // Permute back to [W, H, C, B] for next deconv - up1 = ggml_cont(ctx, ggml_permute(ctx, up1, 2, 0, 1, 3)); // [144, 144, 64, 1] - - // dc2: ConvTranspose2d(64, 32, k=2, s=2) → [288, 288, 32, 1] - auto* up2 = ggml_conv_transpose_2d_p0(ctx, sam3_conv_transpose_weight(ctx, dec.up2_w), up1, 2); - up2 = ggml_add(ctx, up2, ggml_reshape_4d(ctx, dec.up2_b, 1, 1, ggml_nelements(dec.up2_b), 1)); - - // conv_s0: 1x1 conv on feat_s0 (256→32). feat_s0 is [C, W, H, B] — permute for conv. - auto* fs0 = ggml_cont(ctx, ggml_permute(ctx, feat_s0, 2, 0, 1, 3)); // [W, H, C, B] - auto* hs0 = ggml_conv_2d_sk_p0(ctx, dec.conv_s0_w, fs0); // [W, H, 32, B] - hs0 = ggml_add(ctx, hs0, ggml_reshape_4d(ctx, dec.conv_s0_b, 1, 1, 32, 1)); - ggml_set_name(hs0, "sam_dec_feat_s0_proj"); - - // Python: act2(dc2(upscaled_embedding) + feat_s0) — no LayerNorm here - up2 = ggml_add(ctx, up2, hs0); // both [W, H, 32, B] - - // Permute to [C, W, H, B] for subsequent operations - up2 = ggml_cont(ctx, ggml_permute(ctx, up2, 1, 2, 0, 3)); // [32, 288, 288, 1] - - // GELU activation (exact, matching Python nn.GELU) - up2 = ggml_gelu_erf(ctx, up2); - - // up2: [32, 288, 288, 1] — this is our upscaled_embedding - ggml_set_name(up2, "sam_dec_upscaled"); - - // ── Hypernetwork: predict masks ────────────────────────────────────── - // For each mask token i, pass through 3-layer MLP to get [32] vector - // Then dot product with upscaled_embedding [32, (H*4)^2] to get mask - const int H4 = H * 4; - auto* up_flat = ggml_reshape_3d(ctx, up2, 32, H4 * H4, 1); - - // Process each mask token through its hypernetwork MLP - // mask_tokens_out: [D, 4, 1] - struct ggml_tensor* mask_list[4]; - for (int m = 0; m < num_mask_tokens; ++m) { - // Extract token m: [D, 1, 1] - auto* tok = ggml_view_3d(ctx, mask_tokens_out, D, 1, 1, - mask_tokens_out->nb[1], mask_tokens_out->nb[2], - m * mask_tokens_out->nb[1]); - tok = ggml_cont(ctx, tok); // [D, 1, 1] - - // MLP: 3 layers, 256→256→256→32, ReLU on first two - auto* hyper = sam3_mlp_forward(ctx, tok, - dec.hyper_w[m], dec.hyper_b[m], 3); - // hyper: [32, 1, 1] - - // Dot product: hyper^T @ up_flat → [1, 288*288, 1] - // Use mul_mat: up_flat^T [288*288, 32] @ hyper [32, 1] → [288*288, 1, 1] - auto* mask = ggml_mul_mat(ctx, up_flat, hyper); // [288*288, 1, 1] - mask_list[m] = mask; - } - - // Stack masks: [288*288, 4, 1] - auto* masks = mask_list[0]; - for (int m = 1; m < num_mask_tokens; ++m) { - masks = ggml_concat(ctx, masks, mask_list[m], 1); - } - ggml_set_name(masks, "sam_dec_masks"); - - // ── IoU prediction ─────────────────────────────────────────────────── - // iou_token_out: [D, 1, 1] - auto* iou_pred = sam3_mlp_forward(ctx, iou_token_out, - dec.iou_head_w, dec.iou_head_b, 3, - /*sigmoid_output=*/true); - // iou_pred: [4, 1, 1] → reshape to [4, 1] - iou_pred = ggml_reshape_2d(ctx, iou_pred, num_mask_tokens, 1); - ggml_set_name(iou_pred, "sam_dec_iou"); - - // ── Object score ───────────────────────────────────────────────────── - struct ggml_tensor* obj_score; - if (has_obj_score) { - // obj_in: [D, 1, 1] → MLP → [1, 1, 1] - obj_score = sam3_mlp_forward(ctx, obj_in, - dec.obj_head_w, dec.obj_head_b, 3); - obj_score = ggml_reshape_2d(ctx, obj_score, 1, 1); - } else { - // No obj_score prediction — return raw logit 10.0 (sigmoid ≈ 1.0, object always present) - obj_score = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, 1); - ggml_set_name(obj_score, "sam_dec_obj_score"); - ggml_set_input(obj_score); - // Mark that callers must set this to 10.0f before compute - } - - sam3_dec_result res; - res.masks = masks; - res.iou_pred = iou_pred; - res.obj_score = obj_score; - res.sam_token = sam_token; - res.mask_tokens = mask_tokens_out; - return res; -} - -/***************************************************************************** -** Image segmentation — PVS (visual-prompted) (Phase 6, Step 6.3) -*****************************************************************************/ - -sam3_result sam3_segment_pvs(sam3_state& state, - const sam3_model& model, - const sam3_pvs_params& params) { -#if SAM3_LOG_LEVEL >= 1 - auto t_start = std::chrono::high_resolution_clock::now(); -#endif - const auto& hp = model.hparams; - const int D = hp.sam_embed_dim; // 256 - const int H = sam3_eff_feat_size(state, hp); - const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 - const int eff_img_size = sam3_eff_img_size(state, hp); - sam3_result result; - - // ── Validate ───────────────────────────────────────────────────────── - if (!state.neck_trk[0]) { - fprintf(stderr, "%s: image not encoded — call sam3_encode_image first\n", __func__); - return result; - } - if (params.pos_points.empty() && !params.use_box) { - fprintf(stderr, "%s: no prompts provided (need at least one point or box)\n", __func__); - return result; - } - - SAM3_LOG(2, "%s: %zu pos points, %zu neg points, box=%s, multimask=%s\n", - __func__, params.pos_points.size(), params.neg_points.size(), - params.use_box ? "yes" : "no", params.multimask ? "yes" : "no"); - - // ── Build computation graph ────────────────────────────────────────── - const size_t buf_size = ggml_tensor_overhead() * 8192 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return result; - } - - // ── SAM prompt encoder (CPU pre-compute + input tensors) ───────────── - auto pe_out = sam3_build_sam_pe(ctx0, params, D, H); - - // ── Create fresh input tensors for tracker features ────────────────── - // CRITICAL: Do NOT use state.neck_trk[*] directly in graph operations - // (like ggml_add). They are tensors from a previous graph, and - // ggml_build_forward_expand traces all ancestors — which would pull in - // the ENTIRE ViT + neck recomputation (2500+ nodes, ~37 seconds). - // Instead: create fresh input tensors and copy data from state on CPU. - const int H0 = H * 4; // 288 - const int H1 = H * 2; // 144 - - auto* image_feats = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(image_feats, "sam_dec_image_feats"); - ggml_set_input(image_feats); - - auto* feat_s0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); - ggml_set_name(feat_s0, "pvs_feat_s0"); - ggml_set_input(feat_s0); - - auto* feat_s1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); - ggml_set_name(feat_s1, "pvs_feat_s1"); - ggml_set_input(feat_s1); - - // ── SAM mask decoder graph ─────────────────────────────────────────── - auto dec_out = sam3_build_sam_dec_graph(ctx0, model, - image_feats, - pe_out.image_pe, - pe_out.sparse, - pe_out.dense, - feat_s0, - feat_s1, H); - - // Mark outputs - ggml_set_output(dec_out.masks); - ggml_set_output(dec_out.iou_pred); - ggml_set_output(dec_out.obj_score); - ggml_set_output(dec_out.sam_token); - - // ── Build and allocate graph ───────────────────────────────────────── - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 32768, false); - ggml_build_forward_expand(graph, dec_out.masks); - ggml_build_forward_expand(graph, dec_out.iou_pred); - ggml_build_forward_expand(graph, dec_out.obj_score); - ggml_build_forward_expand(graph, dec_out.sam_token); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph)) { - fprintf(stderr, "%s: failed to reserve graph memory\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return result; - } - if (!ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return result; - } - - SAM3_LOG(2, "%s: graph allocated, %d nodes\n", __func__, ggml_graph_n_nodes(graph)); - - // Set default obj_score when pred_obj_scores=False (older SAM2 models) - if (!model.sam_dec.obj_score_token) { - auto* t = ggml_graph_get_tensor(graph, "sam_dec_obj_score"); - if (t) { float v = 10.0f; ggml_backend_tensor_set(t, &v, 0, sizeof(float)); } - } - - // ── Upload input data (using cached embeddings) ──────────────────── - // Populate PE cache on first call (reads model weights from GPU once) - sam3_populate_pe_cache(state, model); - - { - const int N_pts = pe_out.n_tokens; - const int num_pos_feats = D / 2; - - std::vector all_coords; - std::vector all_labels; - sam3_collect_pvs_prompt_tokens(params, all_coords, all_labels); - - // Sparse embeddings — only this changes per call - std::vector sparse_data(N_pts * D, 0.0f); - for (int p = 0; p < N_pts; ++p) { - // Scale from original image space to model input space, then shift to pixel center - float px = all_coords[p * 2 + 0] / (float)state.orig_width * (float)eff_img_size + 0.5f; - float py = all_coords[p * 2 + 1] / (float)state.orig_height * (float)eff_img_size + 0.5f; - float x_norm = px / (float)eff_img_size; - float y_norm = py / (float)eff_img_size; - float pe_vec[256]; - sam3_pe_encode_coord(pe_vec, x_norm, y_norm, - state.pe_gauss_cache.data(), num_pos_feats); - int label = all_labels[p]; - if (label == -1) { - for (int d = 0; d < D; ++d) - sparse_data[p * D + d] = state.not_a_point_cache[d]; - } else { - for (int d = 0; d < D; ++d) - sparse_data[p * D + d] = pe_vec[d] + state.point_emb_cache[label][d]; - } - } - ggml_backend_tensor_set(pe_out.sparse, sparse_data.data(), 0, N_pts * D * sizeof(float)); - - // Dump sparse embeddings if requested - { - const char* dd = getenv("SAM2_DUMP_DIR"); - if (dd) { - char p[512]; snprintf(p, sizeof(p), "%s/cpp_sparse_emb.bin", dd); - FILE* f = fopen(p, "wb"); - if (f) { fwrite(sparse_data.data(), sizeof(float), N_pts * D, f); fclose(f); } - fprintf(stderr, " [DUMP] cpp_sparse_emb: %d tokens x %d dims\n", N_pts, D); - } - } - - // Dense PE grid and no-mask embedding — use pre-computed caches - ggml_backend_tensor_set(pe_out.image_pe, state.dense_pe_cache.data(), - 0, D * H * H * sizeof(float)); - ggml_backend_tensor_set(pe_out.dense, state.dense_nomask_cache.data(), - 0, D * H * H * sizeof(float)); - } - - // ── Copy tracker features from state to fresh input tensors ───────── - // image_feats = neck_trk[2] + no_mem_embed (computed on CPU) - { - const int n2 = D * H * H; - std::vector trk2(n2), no_mem_data(D); - ggml_backend_tensor_get(state.neck_trk[2], trk2.data(), 0, n2 * sizeof(float)); - ggml_backend_tensor_get(model.tensors.at("no_mem_embed"), no_mem_data.data(), 0, D * sizeof(float)); - // Add no_mem_embed (broadcast [D] to [D, H, H]) - for (int s = 0; s < H * H; ++s) - for (int d = 0; d < D; ++d) - trk2[d + s * D] += no_mem_data[d]; - ggml_backend_tensor_set(image_feats, trk2.data(), 0, n2 * sizeof(float)); - - // Dump image_feats (with no_mem_embed) if requested - { - const char* dd = getenv("SAM2_DUMP_DIR"); - if (dd) { - char p[512]; snprintf(p, sizeof(p), "%s/cpp_image_feats.bin", dd); - FILE* f = fopen(p, "wb"); - if (f) { fwrite(trk2.data(), sizeof(float), n2, f); fclose(f); } - fprintf(stderr, " [DUMP] cpp_image_feats: [%d, %d, %d]\n", D, H, H); - } - } - - // feat_s0 = neck_trk[0], feat_s1 = neck_trk[1] - const int n0 = D * H0 * H0; - const int n1 = D * H1 * H1; - std::vector trk0(n0), trk1(n1); - ggml_backend_tensor_get(state.neck_trk[0], trk0.data(), 0, n0 * sizeof(float)); - ggml_backend_tensor_set(feat_s0, trk0.data(), 0, n0 * sizeof(float)); - ggml_backend_tensor_get(state.neck_trk[1], trk1.data(), 0, n1 * sizeof(float)); - ggml_backend_tensor_set(feat_s1, trk1.data(), 0, n1 * sizeof(float)); - } - - // ── Compute ────────────────────────────────────────────────────────── - { -#if SAM3_LOG_LEVEL >= 1 - auto t0 = std::chrono::high_resolution_clock::now(); -#endif - if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return result; - } -#if SAM3_LOG_LEVEL >= 1 - auto t1 = std::chrono::high_resolution_clock::now(); - double ms = std::chrono::duration(t1 - t0).count(); - SAM3_LOG(1, "%s: graph computed in %.1f ms (%d threads)\n", - __func__, ms, state.n_threads); -#endif - } - - // ── Dump decoder outputs if SAM2_DUMP_DIR set ────────────────────── - { - const char* dump_dir = getenv("SAM2_DUMP_DIR"); - if (dump_dir) { - auto dump_t = [&](const char* name, struct ggml_tensor* t) { - if (!t) return; - int64_t nb = ggml_nbytes(t); - std::vector buf(nb); - ggml_backend_tensor_get(t, buf.data(), 0, nb); - char path[512]; - snprintf(path, sizeof(path), "%s/%s.bin", dump_dir, name); - FILE* f = fopen(path, "wb"); - if (f) { fwrite(buf.data(), 1, nb, f); fclose(f); } - snprintf(path, sizeof(path), "%s/%s.shape", dump_dir, name); - f = fopen(path, "w"); - if (f) { - fprintf(f, "%lld,%lld,%lld,%lld", - (long long)t->ne[0], (long long)t->ne[1], - (long long)t->ne[2], (long long)t->ne[3]); - fclose(f); - } - fprintf(stderr, " [DUMP] %s: [%lld,%lld,%lld,%lld]\n", name, - (long long)t->ne[0], (long long)t->ne[1], - (long long)t->ne[2], (long long)t->ne[3]); - }; - dump_t("cpp_pvs_masks", dec_out.masks); - dump_t("cpp_pvs_iou", dec_out.iou_pred); - dump_t("cpp_pvs_obj_score", dec_out.obj_score); - // Decoder transformer intermediates - dump_t("cpp_dec_queries", ggml_graph_get_tensor(graph, "dbg_dec_queries_out")); - dump_t("cpp_dec_keys", ggml_graph_get_tensor(graph, "dbg_dec_keys_out")); - // Block 0 internals - const char* b0_names[] = {"sam_dec_tokens_initial", - "dbg_twoway_skip_sa_norm", "dbg_twoway_skip_ca_tok2img", - "dbg_twoway_skip_mlp", "dbg_twoway_skip_img2tok", - "dbg_sa0_Q_proj", "dbg_sa0_merged"}; - for (auto* bn : b0_names) dump_t(bn, ggml_graph_get_tensor(graph, bn)); - // Per-block outputs - for (int bi = 0; bi < 2; bi++) { - char bn[64]; - snprintf(bn, sizeof(bn), "sam_dec_block%d_queries", bi); - dump_t(bn, ggml_graph_get_tensor(graph, bn)); - snprintf(bn, sizeof(bn), "sam_dec_block%d_keys", bi); - dump_t(bn, ggml_graph_get_tensor(graph, bn)); - } - } - } - - // ── Read outputs ───────────────────────────────────────────────────── - // masks: [H*4×H*4, 4, 1] - const int mask_hw = H * 4; - std::vector masks_data(mask_hw * mask_hw * num_mask_tokens); - ggml_backend_tensor_get(dec_out.masks, masks_data.data(), 0, masks_data.size() * sizeof(float)); - - // IoU predictions: [4, 1] - std::vector iou_data(num_mask_tokens); - ggml_backend_tensor_get(dec_out.iou_pred, iou_data.data(), 0, num_mask_tokens * sizeof(float)); - - // Object score: [1, 1] - float obj_logit = 0.0f; - ggml_backend_tensor_get(dec_out.obj_score, &obj_logit, 0, sizeof(float)); - float obj_score = 1.0f / (1.0f + expf(-obj_logit)); - - // SAM output token: [D, 1] — needed for object pointer extraction in tracking - std::vector sam_token_data(D); - ggml_backend_tensor_get(dec_out.sam_token, sam_token_data.data(), 0, D * sizeof(float)); - - SAM3_LOG(2, "%s: obj_score=%.4f (logit=%.4f), iou=[%.3f, %.3f, %.3f, %.3f]\n", - __func__, obj_score, obj_logit, - iou_data[0], iou_data[1], iou_data[2], iou_data[3]); - - // ── Select masks based on multimask mode ───────────────────────────── - // Python: if multimask_output → masks[:, 1:, :, :], iou_pred[:, 1:] - // else → masks[:, 0:1, :, :], iou_pred[:, 0:1] - int start_idx, end_idx; - if (params.multimask) { - start_idx = 1; - end_idx = num_mask_tokens; - } else { - start_idx = 0; - end_idx = 1; - } - - for (int m = start_idx; m < end_idx; ++m) { - sam3_detection det; - det.sam_token = sam_token_data; - - // Resize mask from 288×288 to original image size - const float* mask_ptr = masks_data.data() + m * mask_hw * mask_hw; - auto mask_resized = sam3_bilinear_interpolate(mask_ptr, mask_hw, mask_hw, - state.orig_width, state.orig_height); - - // Binarize at threshold 0.0 (sigmoid(logit) > 0.5 ↔ logit > 0.0) - det.mask.width = state.orig_width; - det.mask.height = state.orig_height; - det.mask.data.resize(state.orig_width * state.orig_height); - for (int i = 0; i < (int)mask_resized.size(); ++i) { - det.mask.data[i] = (mask_resized[i] > 0.0f) ? 255 : 0; - } - - det.mask.iou_score = iou_data[m]; - det.mask.obj_score = obj_score; - det.mask.instance_id = m; - det.score = iou_data[m]; - det.iou_score = iou_data[m]; - det.instance_id = m; - - // Compute bounding box from mask - int min_x = state.orig_width, min_y = state.orig_height; - int max_x = 0, max_y = 0; - for (int y = 0; y < state.orig_height; ++y) { - for (int x = 0; x < state.orig_width; ++x) { - if (det.mask.data[y * state.orig_width + x] > 0) { - min_x = std::min(min_x, x); - min_y = std::min(min_y, y); - max_x = std::max(max_x, x); - max_y = std::max(max_y, y); - } - } - } - det.box = {(float)min_x, (float)min_y, (float)max_x, (float)max_y}; - - result.detections.push_back(std::move(det)); - } - - SAM3_LOG(2, "%s: %zu masks returned\n", __func__, result.detections.size()); - - // ── Cleanup ────────────────────────────────────────────────────────── - ggml_gallocr_free(galloc); - ggml_free(ctx0); - -#if SAM3_LOG_LEVEL >= 1 - auto t_end = std::chrono::high_resolution_clock::now(); - double total_ms = std::chrono::duration(t_end - t_start).count(); - SAM3_LOG(1, "%s: completed in %.1f ms\n", __func__, total_ms); -#endif - - return result; -} - -/***************************************************************************** -** Video tracking (Phase 7) -*****************************************************************************/ - -struct sam3_prop_output { - std::vector mask_logits; - std::vector iou_scores; - float obj_score; - std::vector sam_token; - int n_masks, mask_h, mask_w; -}; - -// Lazily compute and cache PE/RoPE data that is identical across all propagation calls. -static void sam3_ensure_tracker_pe_caches(sam3_tracker& tracker, const sam3_hparams& hp, - int eff_feat_size = 0) { - const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); - if (tracker.pe_caches_valid) return; - - const int D = hp.neck_dim; // 256 - const int MD = hp.mem_out_dim; // 64 - const int N = H * H; - const int half_d = D / 2; // 128 - - tracker.cached_sinpe_256 = sam3_sinusoidal_pe_2d(H, H, D); - tracker.cached_sinpe_64 = sam3_sinusoidal_pe_2d(H, H, MD); - - // Compute axial CIS and reorder to [2, half_d, N] layout - std::vector rope_raw(N * D); - sam3_compute_axial_cis(rope_raw.data(), D, H, H, 10000.0f, 1.0f); - tracker.cached_axial_cis_reord.resize(2 * half_d * N); - for (int n = 0; n < N; ++n) - for (int i = 0; i < half_d; ++i) { - tracker.cached_axial_cis_reord[0 + i * 2 + n * D] = rope_raw[n * D + i * 2 + 0]; - tracker.cached_axial_cis_reord[1 + i * 2 + n * D] = rope_raw[n * D + i * 2 + 1]; - } - - // EdgeTAM: compute 16x16 RoPE for cross-attention K (perceiver 2D latents) - if (hp.has_perceiver) { - const int K16 = 16; // perceiver 2D grid size - const int NK = K16 * K16; // 256 tokens - std::vector rope_k16_raw(NK * D); - sam3_compute_axial_cis(rope_k16_raw.data(), D, K16, K16, 10000.0f, 1.0f); - tracker.cached_axial_cis_k16_reord.resize(2 * half_d * NK); - for (int n = 0; n < NK; ++n) - for (int i = 0; i < half_d; ++i) { - tracker.cached_axial_cis_k16_reord[0 + i * 2 + n * D] = rope_k16_raw[n * D + i * 2 + 0]; - tracker.cached_axial_cis_k16_reord[1 + i * 2 + n * D] = rope_k16_raw[n * D + i * 2 + 1]; - } - SAM3_LOG(2, "%s: EdgeTAM K16 RoPE cache: %zu floats\n", __func__, - tracker.cached_axial_cis_k16_reord.size()); - } - - tracker.pe_caches_valid = true; - SAM3_LOG(2, "%s: tracker PE caches populated (%.1f KB)\n", __func__, - (tracker.cached_sinpe_256.size() + tracker.cached_sinpe_64.size() + - tracker.cached_axial_cis_reord.size()) * sizeof(float) / 1024.0f); -} - -static sam3_prop_output sam3_propagate_single( - sam3_tracker& tracker, sam3_state& state, const sam3_model& model, - const sam3_masklet& masklet, - const std::vector& mem_bank, - const std::vector>& ptr_bank) { - sam3_prop_output output = {}; - const auto& hp = model.hparams; - const int D = hp.neck_dim, MD = hp.mem_out_dim; - const int H = sam3_eff_feat_size(state, hp); - const int N = H * H; - - auto sel = sam3_select_memory_frames(mem_bank, hp.num_maskmem); - if (sel.empty()) return output; - - // ── Build prompt and prompt_pos via sam3_build_prompt_and_pos ───────── - // EdgeTAM with perceiver: each slot has 512 tokens (not H*H). - // Standard SAM2/SAM3: each slot has H*H tokens. - const bool use_perceiver = hp.has_perceiver != 0; - const int N_per_slot = use_perceiver ? (hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d) : N; - - int n_sel = (int)sel.size(); - std::vector> slot_feats(n_sel), slot_pes(n_sel); - std::vector spatial_tpos(n_sel, 1); // default t_pos=1 for non-cond - for (int s = 0; s < n_sel; ++s) { - slot_feats[s].resize(MD * N_per_slot); - ggml_backend_tensor_get(mem_bank[sel[s]].spatial_feats, - slot_feats[s].data(), 0, MD * N_per_slot * sizeof(float)); - slot_pes[s].resize(MD * N_per_slot); - if (mem_bank[sel[s]].spatial_pe) { - ggml_backend_tensor_get(mem_bank[sel[s]].spatial_pe, - slot_pes[s].data(), 0, MD * N_per_slot * sizeof(float)); - } else { - sam3_ensure_tracker_pe_caches(tracker, hp, H); - if (use_perceiver) { - // For perceiver: zeros for 1D tokens, sinusoidal for 2D tokens - const int N_1d = hp.perceiver_n_latents_1d; - const int N_2d = hp.perceiver_n_latents_2d; - memset(slot_pes[s].data(), 0, MD * N_1d * sizeof(float)); - auto pe_2d = sam3_sinusoidal_pe_2d(16, 16, MD); - memcpy(slot_pes[s].data() + MD * N_1d, pe_2d.data(), MD * N_2d * sizeof(float)); - } else { - slot_pes[s] = tracker.cached_sinpe_64; - } - } - spatial_tpos[s] = mem_bank[sel[s]].is_cond_frame ? 0 : (n_sel - s); - } - - int P = std::min((int)ptr_bank.size(), hp.max_obj_ptrs); - std::vector> obj_ptrs(P); - std::vector ptr_tpos(P); - int cur_frame = tracker.frame_index; - for (int p = 0; p < P; ++p) { - obj_ptrs[p].resize(D); - ggml_backend_tensor_get(ptr_bank[p].second, obj_ptrs[p].data(), 0, D * sizeof(float)); - // Use actual frame distance (matches Python: abs(frame_idx - t)) - ptr_tpos[p] = std::abs(cur_frame - ptr_bank[p].first); - if (ptr_tpos[p] < 1) ptr_tpos[p] = 1; // minimum distance of 1 - } - - auto pd = sam3_build_prompt_and_pos(model, slot_feats, slot_pes, spatial_tpos, obj_ptrs, ptr_tpos, H); - - // ── RoPE frequencies (cached) ────────────────────────────────────── - sam3_ensure_tracker_pe_caches(tracker, hp, H); - const int half_d = D / 2; // 128 - const auto& rope_q_reord = tracker.cached_axial_cis_reord; - // For cross-attn K: build rope_k_data for all M_spatial tokens - std::vector rope_k_data; - if (pd.M_spatial > 0) { - rope_k_data.resize(2 * half_d * pd.M_spatial); - if (use_perceiver) { - // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. - // First 256 (1D latents): identity RoPE (cos=1, sin=0). - // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. - const int N_1d = hp.perceiver_n_latents_1d; // 256 - const int N_2d = hp.perceiver_n_latents_2d; // 256 - const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] - for (int s = 0; s < n_sel; ++s) { - float* dst = rope_k_data.data() + s * D * N_per_slot; - // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout - // Layout: for token n, dim i: cos at [0 + i*2 + n*D], sin at [1 + i*2 + n*D] - for (int n = 0; n < N_1d; ++n) - for (int i = 0; i < half_d; ++i) { - dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 - dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 - } - // 2D tokens: copy 16x16 RoPE - float* dst_2d = dst + D * N_1d; - memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); - } - } else { - // Standard: repeat HxH axial CIS for each memory frame - for (int s = 0; s < pd.M_spatial / N; ++s) - memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); - } - } - - // ── Build graph ───────────────────────────────────────────────────── - const size_t buf_size = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = {buf_size, nullptr, true}; - auto* ctx0 = ggml_init(gparams); - if (!ctx0) return output; - - // CRITICAL: create fresh input tensors for state features. - // Using state.neck_trk[*] directly as ggml_reshape operands pulls in - // the entire ViT+neck recomputation from the image encoder graph. - auto* curr = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); - ggml_set_name(curr, "prop_curr"); - ggml_set_input(curr); - - // src_pos (sinusoidal PE 256-dim for 72×72) - auto* src_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); - ggml_set_name(src_pos_t, "src_pos"); - ggml_set_input(src_pos_t); - - // Prompt and prompt_pos - auto* prompt_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); - ggml_set_name(prompt_t, "prompt"); - ggml_set_input(prompt_t); - auto* prompt_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); - ggml_set_name(prompt_pos_t, "prompt_pos"); - ggml_set_input(prompt_pos_t); - - // RoPE frequencies - auto* rope_q_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, N); - ggml_set_name(rope_q_t, "rope_q"); - ggml_set_input(rope_q_t); - struct ggml_tensor* rope_k_t = nullptr; - if (pd.M_spatial > 0) { - rope_k_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, pd.M_spatial); - ggml_set_name(rope_k_t, "rope_k"); - ggml_set_input(rope_k_t); - } - - auto* conditioned = sam3_build_mem_attn_graph(ctx0, model, curr, src_pos_t, - prompt_t, prompt_pos_t, - rope_q_t, rope_k_t, - pd.num_obj_ptr_tokens); - auto* cond_spatial = ggml_reshape_4d(ctx0, conditioned, D, H, H, 1); - - // Bug 3 fix: single not_a_point_embed token instead of empty sparse - auto* sparse_in = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, 1, 1); - ggml_set_name(sparse_in, "prop_sparse"); - ggml_set_input(sparse_in); - - auto* image_pe = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(image_pe, "prop_pe"); - ggml_set_input(image_pe); - auto* dense_emb = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(dense_emb, "prop_dense"); - ggml_set_input(dense_emb); - - const int H0 = H * 4, H1 = H * 2; - auto* trk_s0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); - ggml_set_name(trk_s0, "prop_trk_s0"); - ggml_set_input(trk_s0); - auto* trk_s1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); - ggml_set_name(trk_s1, "prop_trk_s1"); - ggml_set_input(trk_s1); - - auto dec = sam3_build_sam_dec_graph(ctx0, model, cond_spatial, image_pe, - sparse_in, dense_emb, - trk_s0, trk_s1, H); - ggml_set_output(dec.masks); - ggml_set_output(dec.iou_pred); - ggml_set_output(dec.obj_score); - ggml_set_output(dec.sam_token); - if (dec.mask_tokens) ggml_set_output(dec.mask_tokens); - - auto* graph = ggml_new_graph_custom(ctx0, 32768, false); - ggml_build_forward_expand(graph, dec.masks); - ggml_build_forward_expand(graph, dec.iou_pred); - ggml_build_forward_expand(graph, dec.obj_score); - ggml_build_forward_expand(graph, dec.sam_token); - if (dec.mask_tokens) ggml_build_forward_expand(graph, dec.mask_tokens); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return output; - } - - // Upload prompt data - ggml_backend_tensor_set(prompt_t, pd.prompt.data(), 0, pd.prompt.size() * sizeof(float)); - ggml_backend_tensor_set(prompt_pos_t, pd.prompt_pos.data(), 0, pd.prompt_pos.size() * sizeof(float)); - ggml_backend_tensor_set(rope_q_t, rope_q_reord.data(), 0, rope_q_reord.size() * sizeof(float)); - if (rope_k_t && !rope_k_data.empty()) - ggml_backend_tensor_set(rope_k_t, rope_k_data.data(), 0, rope_k_data.size() * sizeof(float)); - - // Set default obj_score when pred_obj_scores=False (older SAM2 models) - if (!model.sam_dec.obj_score_token) { - auto* t = ggml_graph_get_tensor(graph, "sam_dec_obj_score"); - if (t) { float v = 10.0f; ggml_backend_tensor_set(t, &v, 0, sizeof(float)); } - } - - // Upload src_pos (sinusoidal PE 256-dim) - ggml_backend_tensor_set(src_pos_t, tracker.cached_sinpe_256.data(), 0, - tracker.cached_sinpe_256.size() * sizeof(float)); - - // Upload not_a_point_embed, image_pe, dense_emb from state PE cache - sam3_populate_pe_cache(state, model); - ggml_backend_tensor_set(sparse_in, state.not_a_point_cache, 0, D * sizeof(float)); - ggml_backend_tensor_set(image_pe, state.dense_pe_cache.data(), 0, - state.dense_pe_cache.size() * sizeof(float)); - ggml_backend_tensor_set(dense_emb, state.dense_nomask_cache.data(), 0, - state.dense_nomask_cache.size() * sizeof(float)); - - // Copy tracker features from state to fresh input tensors - { - std::vector c2(D * N); - ggml_backend_tensor_get(state.neck_trk[2], c2.data(), 0, D * N * sizeof(float)); - ggml_backend_tensor_set(curr, c2.data(), 0, D * N * sizeof(float)); - - std::vector s0(D * H0 * H0); - ggml_backend_tensor_get(state.neck_trk[0], s0.data(), 0, D * H0 * H0 * sizeof(float)); - ggml_backend_tensor_set(trk_s0, s0.data(), 0, D * H0 * H0 * sizeof(float)); - - std::vector s1(D * H1 * H1); - ggml_backend_tensor_get(state.neck_trk[1], s1.data(), 0, D * H1 * H1 * sizeof(float)); - ggml_backend_tensor_set(trk_s1, s1.data(), 0, D * H1 * H1 * sizeof(float)); - } - - if (!sam3_graph_compute(model.backend, graph, 4)) { - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return output; - } - - const int mhw = H * 4; - const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 - - // SAM2/EdgeTAM with multimask_output_in_sam: read all masks, select best by IoU. - // SAM3 / SAM2 without multimask: use first mask only. - bool use_multimask = (hp.is_sam2() || hp.is_edgetam()) && hp.multimask_output_in_sam; - - if (use_multimask) { - // Read all 4 IoU predictions - std::vector all_ious(num_mask_tokens); - ggml_backend_tensor_get(dec.iou_pred, all_ious.data(), 0, num_mask_tokens * sizeof(float)); - - // Multimask uses mask tokens 1-3 (skip token 0 which is the single-mask output) - int best_idx = 1; - float best_iou = all_ious[1]; - for (int m = 2; m < num_mask_tokens; ++m) { - if (all_ious[m] > best_iou) { best_iou = all_ious[m]; best_idx = m; } - } - - output.n_masks = 1; - output.mask_h = mhw; - output.mask_w = mhw; - output.mask_logits.resize(mhw * mhw); - // Read best mask (offset by best_idx * mhw * mhw) - ggml_backend_tensor_get(dec.masks, output.mask_logits.data(), - best_idx * mhw * mhw * sizeof(float), mhw * mhw * sizeof(float)); - output.iou_scores.resize(1); - output.iou_scores[0] = best_iou; - ggml_backend_tensor_get(dec.obj_score, &output.obj_score, 0, sizeof(float)); - - // Object pointer token: use best multimask token if use_multimask_token_for_obj_ptr - output.sam_token.resize(D); - if (hp.use_multimask_token_for_obj_ptr && dec.mask_tokens) { - ggml_backend_tensor_get(dec.mask_tokens, output.sam_token.data(), - best_idx * D * sizeof(float), D * sizeof(float)); - } else { - ggml_backend_tensor_get(dec.sam_token, output.sam_token.data(), 0, D * sizeof(float)); - } - } else { - output.n_masks = 1; - output.mask_h = mhw; - output.mask_w = mhw; - output.mask_logits.resize(mhw * mhw); - ggml_backend_tensor_get(dec.masks, output.mask_logits.data(), 0, mhw * mhw * sizeof(float)); - output.iou_scores.resize(1); - ggml_backend_tensor_get(dec.iou_pred, output.iou_scores.data(), 0, sizeof(float)); - ggml_backend_tensor_get(dec.obj_score, &output.obj_score, 0, sizeof(float)); - output.sam_token.resize(D); - ggml_backend_tensor_get(dec.sam_token, output.sam_token.data(), 0, D * sizeof(float)); - } - - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return output; -} - -static std::vector> sam3_match_detections( - const std::vector& masklets, const std::vector& dets, - const std::vector& prop_masks, float iou_threshold) { - std::vector> matches; - if (masklets.empty() || dets.empty()) return matches; - int n_m = (int)masklets.size(), n_d = (int)dets.size(); - std::vector dm(n_d, false); - for (int i = 0; i < n_m; ++i) { - if (i >= (int)prop_masks.size() || prop_masks[i].data.empty()) continue; - int bj = -1; - float bi = iou_threshold; - for (int j = 0; j < n_d; ++j) { - if (dm[j] || dets[j].mask.data.empty()) continue; - int w = prop_masks[i].width, h = prop_masks[i].height; - if (w != dets[j].mask.width || h != dets[j].mask.height) continue; - float iou = sam3_mask_iou(prop_masks[i].data.data(), dets[j].mask.data.data(), w * h); - if (iou > bi) { - bi = iou; - bj = j; - } - } - if (bj >= 0) { - matches.push_back({i, bj}); - dm[bj] = true; - } - } - return matches; -} - -static void sam3_update_tracker(sam3_tracker& tracker, int frame_idx) { - for (auto it = tracker.pending.begin(); it != tracker.pending.end();) { - int age = frame_idx - it->first_frame; - if (age >= tracker.params.hotstart_delay && it->mds_sum > 0) { - it->confirmed = true; - tracker.masklets.push_back(std::move(*it)); - it = tracker.pending.erase(it); - } else if (age >= tracker.params.hotstart_delay) { - it = tracker.pending.erase(it); - } else - ++it; - } - for (auto it = tracker.masklets.begin(); it != tracker.masklets.end();) { - if (frame_idx - it->last_seen > tracker.params.max_keep_alive) { - tracker.mem_banks.erase(it->instance_id); - tracker.ptr_banks.erase(it->instance_id); - it = tracker.masklets.erase(it); - } else - ++it; - } -} - -static bool sam3_encode_memory( - sam3_tracker& tracker, sam3_state& state, const sam3_model& model, - int inst_id, const float* mask_logits, int mask_h, int mask_w, - int frame_idx, bool is_cond, float obj_score) { - const auto& hp = model.hparams; - const int D = hp.neck_dim, MD = hp.mem_out_dim; - const int H = sam3_eff_feat_size(state, hp); - const int HIGH_RES = sam3_eff_img_size(state, hp); - const int INTERPOL = H * 16; - - // Mask preprocessing: mask_logits → HIGH_RES → sigmoid → scale/bias → INTERPOL - auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); - const float sig_scale = hp.sigmoid_scale(), sig_bias = hp.sigmoid_bias(); - for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_scale + sig_bias; } - auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL); - - const size_t bs = ggml_tensor_overhead() * 16384 + ggml_graph_overhead(); - struct ggml_init_params gp = {bs, nullptr, true}; - auto* ctx0 = ggml_init(gp); - if (!ctx0) return false; - - auto* mask_in = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, INTERPOL, INTERPOL, 1, 1); - ggml_set_name(mask_in, "mem_mask"); - ggml_set_input(mask_in); - - // Learned mask downsampler: 4 stages conv+LN+GELU + final 1×1 - // Conv output is in ggml "conv" layout. permute(1,2,0,3) converts to "internal" layout - // where ne[0]=channel for LN2d. permute(2,0,1,3) converts back to "conv" layout. - auto* ds = mask_in; - for (int s = 0; s < 4; ++s) { - int out_ch = (int)model.mem_enc.ds_conv_w[s]->ne[3]; - ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[s], ds, 2, 2, 1, 1, 1, 1); - ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[s], 1, 1, out_ch, 1)); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal (ne[0]=channel) - ds = sam3_layer_norm_2d(ctx0, ds, model.mem_enc.ds_norm_w[s], model.mem_enc.ds_norm_b[s]); - ds = ggml_gelu(ctx0, ds); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 2, 0, 1, 3)); // internal→conv - } - ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[4], ds, 1, 1, 0, 0, 1, 1); - ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[4], 1, 1, D, 1)); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal [D, ...] - - // Pixel projection — use fresh input tensor to avoid pulling in the - // entire ViT+neck recomputation from state.neck_trk[2]'s dependency tree - auto* pix_in_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(pix_in_raw, "mem_pix_feat"); - ggml_set_input(pix_in_raw); - auto* pix_in = ggml_cont(ctx0, ggml_permute(ctx0, pix_in_raw, 2, 0, 1, 3)); - auto* pix = ggml_conv_2d(ctx0, model.mem_enc.pix_proj_w, pix_in, 1, 1, 0, 0, 1, 1); - pix = ggml_add(ctx0, pix, ggml_reshape_4d(ctx0, model.mem_enc.pix_proj_b, 1, 1, D, 1)); - pix = ggml_cont(ctx0, ggml_permute(ctx0, pix, 1, 2, 0, 3)); - - // Fusion: ADD (not multiply) - auto* fused = ggml_add(ctx0, pix, ds); - for (int i = 0; i < 2; ++i) - fused = sam3_cxblock_forward(ctx0, fused, - model.mem_enc.fuser_dw_w[i], model.mem_enc.fuser_dw_b[i], - model.mem_enc.fuser_norm_w[i], model.mem_enc.fuser_norm_b[i], - model.mem_enc.fuser_fc1_w[i], model.mem_enc.fuser_fc1_b[i], - model.mem_enc.fuser_fc2_w[i], model.mem_enc.fuser_fc2_b[i], - model.mem_enc.fuser_gamma[i]); - auto* fused_out = ggml_cont(ctx0, ggml_permute(ctx0, fused, 2, 0, 1, 3)); - auto* mo = ggml_conv_2d(ctx0, model.mem_enc.out_proj_w, fused_out, 1, 1, 0, 0, 1, 1); - mo = ggml_add(ctx0, mo, ggml_reshape_4d(ctx0, model.mem_enc.out_proj_b, 1, 1, MD, 1)); - mo = ggml_cont(ctx0, ggml_permute(ctx0, mo, 1, 2, 0, 3)); - ggml_set_name(mo, "mem_out"); - ggml_set_output(mo); - - auto* g = ggml_new_graph_custom(ctx0, 16384, false); - ggml_build_forward_expand(g, mo); - auto* ga = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(ga, g) || !ggml_gallocr_alloc_graph(ga, g)) { - ggml_gallocr_free(ga); - ggml_free(ctx0); - return false; - } - ggml_backend_tensor_set(mask_in, m_interp.data(), 0, m_interp.size() * sizeof(float)); - // Copy pixel features from state tensor to fresh input - { - std::vector pix_data(D * H * H); - ggml_backend_tensor_get(state.neck_trk[2], pix_data.data(), 0, D * H * H * sizeof(float)); - ggml_backend_tensor_set(pix_in_raw, pix_data.data(), 0, D * H * H * sizeof(float)); - } - if (!sam3_graph_compute(model.backend, g, 4)) { - ggml_gallocr_free(ga); - ggml_free(ctx0); - return false; - } - - std::vector md(MD * H * H); - ggml_backend_tensor_get(mo, md.data(), 0, md.size() * sizeof(float)); - - // Apply no_obj_embed_spatial if occluded (SAM2.1 only — EdgeTAM does not have this) - if (obj_score <= 0.0f && model.no_obj_embed_spatial) { - std::vector no_obj_emb(MD); - auto* noe = model.no_obj_embed_spatial; - if (noe->type == GGML_TYPE_F16) { - std::vector tmp(MD); - ggml_backend_tensor_get(noe, tmp.data(), 0, MD * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), no_obj_emb.data(), MD); - } else { - ggml_backend_tensor_get(noe, no_obj_emb.data(), 0, MD * sizeof(float)); - } - for (int i = 0; i < MD * H * H; ++i) - md[i] += no_obj_emb[i % MD]; - } - - if (!tracker.ctx) { - struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; - tracker.ctx = ggml_init(tp); - } - - // ── EdgeTAM perceiver: compress memory features before storage ─────── - if (hp.has_perceiver) { - // Compute sinusoidal PE for memory features (needed as perceiver input) - sam3_ensure_tracker_pe_caches(tracker, hp, H); - const auto& mem_pos = tracker.cached_sinpe_64; - - std::vector perc_latents, perc_pos; - if (!edgetam_perceiver_forward(model, md, mem_pos, H, H, - perc_latents, perc_pos)) { - fprintf(stderr, "%s: perceiver forward failed\n", __func__); - ggml_gallocr_free(ga); - ggml_free(ctx0); - return false; - } - - // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) - const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; - auto* st = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); - auto* sb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); - struct ggml_tallocr ta_perc = ggml_tallocr_new(sb); - ggml_tallocr_alloc(&ta_perc, st); - tracker.owned_buffers.push_back(sb); - ggml_backend_tensor_set(st, perc_latents.data(), 0, MD * N_perc * sizeof(float)); - - // Store perceiver PE: [MD=64, 512] (first 256 zeros, last 256 sinusoidal) - auto* spe = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); - auto* speb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); - struct ggml_tallocr ta_perc2 = ggml_tallocr_new(speb); - ggml_tallocr_alloc(&ta_perc2, spe); - tracker.owned_buffers.push_back(speb); - ggml_backend_tensor_set(spe, perc_pos.data(), 0, MD * N_perc * sizeof(float)); - - sam3_memory_slot slot; - slot.spatial_feats = st; - slot.spatial_pe = spe; - slot.frame_index = frame_idx; - slot.is_cond_frame = is_cond; - auto& bk = tracker.mem_banks[inst_id]; - bk.push_back(slot); - while ((int)bk.size() > hp.num_maskmem) { - bool removed = false; - for (auto it = bk.begin(); it != bk.end(); ++it) - if (!it->is_cond_frame) { - bk.erase(it); - removed = true; - break; - } - if (!removed) bk.erase(bk.begin() + 1); - } - ggml_gallocr_free(ga); - ggml_free(ctx0); - return true; - } - - // ── Standard path (SAM2/SAM3): store raw [MD, H, H] features ──────── - // Store spatial features - auto* st = ggml_new_tensor_4d(tracker.ctx, GGML_TYPE_F32, MD, H, H, 1); - auto* sb = ggml_backend_alloc_buffer(model.backend, MD * H * H * sizeof(float)); - struct ggml_tallocr ta = ggml_tallocr_new(sb); - ggml_tallocr_alloc(&ta, st); - tracker.owned_buffers.push_back(sb); - ggml_backend_tensor_set(st, md.data(), 0, md.size() * sizeof(float)); - - // Compute and store sinusoidal spatial PE - sam3_ensure_tracker_pe_caches(tracker, hp, H); - const auto& pe_data = tracker.cached_sinpe_64; - auto* spe = ggml_new_tensor_4d(tracker.ctx, GGML_TYPE_F32, MD, H, H, 1); - auto* speb = ggml_backend_alloc_buffer(model.backend, MD * H * H * sizeof(float)); - struct ggml_tallocr ta2 = ggml_tallocr_new(speb); - ggml_tallocr_alloc(&ta2, spe); - tracker.owned_buffers.push_back(speb); - ggml_backend_tensor_set(spe, pe_data.data(), 0, pe_data.size() * sizeof(float)); - - sam3_memory_slot slot; - slot.spatial_feats = st; - slot.spatial_pe = spe; - slot.frame_index = frame_idx; - slot.is_cond_frame = is_cond; - auto& bk = tracker.mem_banks[inst_id]; - bk.push_back(slot); - while ((int)bk.size() > hp.num_maskmem) { - bool removed = false; - for (auto it = bk.begin(); it != bk.end(); ++it) - if (!it->is_cond_frame) { - bk.erase(it); - removed = true; - break; - } - if (!removed) bk.erase(bk.begin() + 1); - } - ggml_gallocr_free(ga); - ggml_free(ctx0); - return true; -} - -static void sam3_store_obj_ptr( - sam3_tracker& tracker, const sam3_model& model, - int inst_id, const float* pd, int frame_idx) { - const int D = model.hparams.neck_dim; - if (!tracker.ctx) { - struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; - tracker.ctx = ggml_init(tp); - } - auto* pt = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, D, 1); - auto* pb = ggml_backend_alloc_buffer(model.backend, D * sizeof(float)); - struct ggml_tallocr ta = ggml_tallocr_new(pb); - ggml_tallocr_alloc(&ta, pt); - tracker.owned_buffers.push_back(pb); - ggml_backend_tensor_set(pt, pd, 0, D * sizeof(float)); - auto& bk = tracker.ptr_banks[inst_id]; - bk.push_back({frame_idx, pt}); - while ((int)bk.size() > model.hparams.max_obj_ptrs) bk.erase(bk.begin()); -} - -sam3_tracker_ptr sam3_create_tracker(const sam3_model& model, - const sam3_video_params& params) { - if (model.hparams.is_sam2()) { - fprintf(stderr, "%s: ERROR: text-prompted tracker not available for SAM2 " - "(use sam3_create_visual_tracker instead)\n", __func__); - return nullptr; - } - sam3_tracker_ptr tracker(new sam3_tracker()); - tracker->params = params; - fprintf(stderr, "%s: tracker created (hotstart=%d, max_keep_alive=%d)\n", - __func__, params.hotstart_delay, params.max_keep_alive); - return tracker; -} - -sam3_result sam3_track_frame(sam3_tracker& tracker, sam3_state& state, - const sam3_model& model, const sam3_image& frame) { - if (model.hparams.visual_only) { - fprintf(stderr, "%s: ERROR: track_frame not available on visual-only model " - "(use sam3_propagate_frame instead)\n", __func__); - return sam3_result{}; - } - - sam3_result result; - const int D = model.hparams.neck_dim; - if (!sam3_encode_image(state, model, frame)) return result; - int fi = tracker.frame_index; - fprintf(stderr, "%s: frame %d (%zu active + %zu pending)\n", - __func__, fi, tracker.masklets.size(), tracker.pending.size()); - - std::map pm; - std::map po; - for (auto& ml : tracker.masklets) { - int id = ml.instance_id; - auto im = tracker.mem_banks.find(id); - if (im == tracker.mem_banks.end() || im->second.empty()) continue; - po[id] = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); - if (po[id].mask_logits.empty()) continue; - auto rs = sam3_bilinear_interpolate(po[id].mask_logits.data(), - po[id].mask_w, po[id].mask_h, state.orig_width, state.orig_height); - pm[id].width = state.orig_width; - pm[id].height = state.orig_height; - pm[id].data.resize(state.orig_width * state.orig_height); - int fg = 0; - for (int p = 0; p < (int)rs.size(); ++p) { - bool f = rs[p] > 0.0f; - pm[id].data[p] = f ? 255 : 0; - if (f) fg++; - } - ml.last_score = po[id].iou_scores[0]; - ml.last_seen = fi; - float cov = (float)fg / (state.orig_width * state.orig_height); - ml.mds_sum += (cov > 0.001f && po[id].obj_score > 0.0f) ? 1 : -1; - } - for (auto& ml : tracker.pending) { - int id = ml.instance_id; - auto im = tracker.mem_banks.find(id); - if (im == tracker.mem_banks.end() || im->second.empty()) continue; - auto p2 = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); - if (!p2.mask_logits.empty()) { - ml.last_score = p2.iou_scores[0]; - ml.last_seen = fi; - auto r2 = sam3_bilinear_interpolate(p2.mask_logits.data(), - p2.mask_w, p2.mask_h, state.orig_width, state.orig_height); - int fg2 = 0; - for (auto v : r2) - if (v > 0.0f) fg2++; - float c2 = (float)fg2 / (state.orig_width * state.orig_height); - ml.mds_sum += (c2 > 0.001f && p2.obj_score > 0.0f) ? 1 : -1; - sam3_encode_memory(tracker, state, model, id, - p2.mask_logits.data(), p2.mask_h, p2.mask_w, fi, false, p2.obj_score); - std::vector op(D); - sam3_extract_obj_ptr_cpu(model, p2.sam_token.data(), p2.obj_score, op.data()); - sam3_store_obj_ptr(tracker, model, id, op.data(), fi); - } - } - sam3_result nd; - if (!tracker.params.text_prompt.empty()) { - sam3_pcs_params pcs; - pcs.text_prompt = tracker.params.text_prompt; - pcs.score_threshold = tracker.params.score_threshold; - pcs.nms_threshold = tracker.params.nms_threshold; - nd = sam3_segment_pcs(state, model, pcs); - } - // Match new PCS detections against ALL tracked objects (both active AND pending) - // so that the same object isn't detected as a new instance every frame. - std::vector all_tracked; - std::vector all_tracked_masks; - for (auto& ml : tracker.masklets) { - all_tracked.push_back(ml); - auto it = pm.find(ml.instance_id); - all_tracked_masks.push_back(it != pm.end() ? it->second : sam3_mask{}); - } - for (auto& ml : tracker.pending) { - all_tracked.push_back(ml); - auto it = pm.find(ml.instance_id); - all_tracked_masks.push_back(it != pm.end() ? it->second : sam3_mask{}); - } - auto mat = sam3_match_detections(all_tracked, nd.detections, - all_tracked_masks, tracker.params.assoc_iou_threshold); - std::vector dmat(nd.detections.size(), false); - for (auto& m : mat) { - dmat[m.second] = true; - // Update last_seen on the matched masklet (in active or pending) - int mid = all_tracked[m.first].instance_id; - for (auto& ml : tracker.masklets) - if (ml.instance_id == mid) { - ml.last_seen = fi; - break; - } - for (auto& ml : tracker.pending) - if (ml.instance_id == mid) { - ml.last_seen = fi; - ml.mds_sum++; - break; - } - } - // Map: pending instance_id → PCS detection index (for display) - std::map pending_det_idx; - - for (int j = 0; j < (int)nd.detections.size(); ++j) { - if (dmat[j]) continue; - sam3_masklet ml; - ml.instance_id = tracker.next_inst_id++; - ml.first_frame = fi; - ml.last_seen = fi; - ml.last_score = nd.detections[j].score; - ml.mds_sum = 1; - pending_det_idx[ml.instance_id] = j; - - // Encode the detection's mask into memory so subsequent frames can - // propagate from it. Without this, pending masklets have no memory - // and propagation returns empty on all future frames. - const auto& det = nd.detections[j]; - if (!det.mask.data.empty()) { - // Build mask logits from the PCS detection's binary mask. - const int mh = sam3_eff_feat_size(state, model.hparams) * 4, mw = mh; - // Resize to mh×mw logit space: inside mask → +5.0, outside → -5.0. - std::vector det_logits(mh * mw); - for (int r = 0; r < mh; ++r) { - for (int c = 0; c < mw; ++c) { - int sx = c * det.mask.width / mw; - int sy = r * det.mask.height / mh; - sx = std::min(sx, det.mask.width - 1); - sy = std::min(sy, det.mask.height - 1); - det_logits[r * mw + c] = (det.mask.data[sy * det.mask.width + sx] > 127) - ? 5.0f - : -5.0f; - } - } - sam3_encode_memory(tracker, state, model, ml.instance_id, - det_logits.data(), mh, mw, fi, true, 1.0f); - - // Store a dummy object pointer (from no_obj_ptr since we don't - // have a SAM token from PCS — PCS uses a different decoder) - std::vector no_ptr(D); - auto* nop = model.tensors.at("no_obj_ptr"); - if (nop->type == GGML_TYPE_F16) { - std::vector tmp(D); - ggml_backend_tensor_get(nop, tmp.data(), 0, D * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), no_ptr.data(), D); - } else { - ggml_backend_tensor_get(nop, no_ptr.data(), 0, D * sizeof(float)); - } - sam3_store_obj_ptr(tracker, model, ml.instance_id, no_ptr.data(), fi); - } - - tracker.pending.push_back(std::move(ml)); - } - for (auto& ml : tracker.masklets) { - int id = ml.instance_id; - auto it = po.find(id); - if (it == po.end() || it->second.mask_logits.empty()) continue; - sam3_encode_memory(tracker, state, model, id, - it->second.mask_logits.data(), it->second.mask_h, it->second.mask_w, fi, false, it->second.obj_score); - std::vector op(D); - sam3_extract_obj_ptr_cpu(model, it->second.sam_token.data(), it->second.obj_score, op.data()); - sam3_store_obj_ptr(tracker, model, id, op.data(), fi); - } - sam3_update_tracker(tracker, fi); - - // Helper: build detection from a mask and add to result - auto add_mask_to_result = [&](int inst_id, float score, const sam3_mask& mask) { - if (mask.data.empty()) return; - sam3_detection det; - det.instance_id = inst_id; - det.score = score; - det.mask = mask; - det.mask.instance_id = inst_id; - det.mask.iou_score = score; - float x0 = 1e9f, y0 = 1e9f, x1 = -1e9f, y1 = -1e9f; - for (int p = 0; p < (int)det.mask.data.size(); ++p) - if (det.mask.data[p] > 127) { - int x = p % det.mask.width, y = p / det.mask.width; - x0 = std::min(x0, (float)x); - y0 = std::min(y0, (float)y); - x1 = std::max(x1, (float)x); - y1 = std::max(y1, (float)y); - } - if (x0 <= x1) det.box = {x0, y0, x1, y1}; - result.detections.push_back(std::move(det)); - }; - - // Include confirmed (active) masklet propagation results - for (auto& ml : tracker.masklets) { - auto it = pm.find(ml.instance_id); - if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); - } - - // Include pending masklet results: on the detection frame, use the PCS - // detection mask; on subsequent frames, use the propagated mask. - for (auto& ml : tracker.pending) { - auto it = pm.find(ml.instance_id); - if (it != pm.end() && !it->second.data.empty()) { - // Propagated mask available (frame > detection frame) - add_mask_to_result(ml.instance_id, ml.last_score, it->second); - } else { - // First detection frame: use the PCS detection's mask+box directly - auto di = pending_det_idx.find(ml.instance_id); - if (di != pending_det_idx.end() && di->second < (int)nd.detections.size()) { - const auto& det = nd.detections[di->second]; - if (!det.mask.data.empty()) { - add_mask_to_result(ml.instance_id, ml.last_score, det.mask); - } - } - } - } - sam3_resolve_overlaps(result.detections); - for (auto& d : result.detections) { - if (d.mask.data.empty()) continue; - sam3_fill_holes(d.mask.data.data(), d.mask.width, d.mask.height, tracker.params.fill_hole_area); - sam3_remove_sprinkles(d.mask.data.data(), d.mask.width, d.mask.height, tracker.params.fill_hole_area); - } - tracker.frame_index++; - SAM3_LOG(2, "%s: frame %d done — %zu tracked\n", __func__, fi, result.detections.size()); - return result; -} - -bool sam3_refine_instance(sam3_tracker& tracker, sam3_state& state, - const sam3_model& model, int instance_id, - const std::vector& pos_points, - const std::vector& neg_points) { - const int D = model.hparams.neck_dim; - sam3_masklet* tgt = nullptr; - for (auto& ml : tracker.masklets) - if (ml.instance_id == instance_id) { - tgt = &ml; - break; - } - if (!tgt) - for (auto& ml : tracker.pending) - if (ml.instance_id == instance_id) { - tgt = &ml; - break; - } - if (!tgt) { - fprintf(stderr, "%s: instance %d not found\n", __func__, instance_id); - return false; - } - sam3_pvs_params pvs; - pvs.pos_points = pos_points; - pvs.neg_points = neg_points; - pvs.multimask = false; - auto r = sam3_segment_pvs(state, model, pvs); - if (r.detections.empty()) return false; - // tracker.frame_index points to the *next* frame; the refinement applies - // to the frame that was last tracked / encoded. - int fi = std::max(0, tracker.frame_index - 1); - const auto& rdet = r.detections[0]; - tgt->last_score = rdet.score; - tgt->last_seen = fi; - std::vector op(D); - if (!rdet.sam_token.empty()) { - sam3_extract_obj_ptr_cpu(model, rdet.sam_token.data(), rdet.mask.obj_score, op.data()); + state.pe_gauss_cache.resize(pe_nel); + if (pe.pe_gaussian->type == GGML_TYPE_F16) { + std::vector tmp(pe_nel); + ggml_backend_tensor_get(pe.pe_gaussian, tmp.data(), 0, pe_nel * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), state.pe_gauss_cache.data(), pe_nel); } else { - std::fill(op.begin(), op.end(), 0.0f); - } - sam3_store_obj_ptr(tracker, model, instance_id, op.data(), fi); - SAM3_LOG(2, "%s: refined instance %d\n", __func__, instance_id); - return true; -} - -int sam3_tracker_add_instance(sam3_tracker& tracker, sam3_state& state, - const sam3_model& model, - const sam3_pvs_params& pvs_params) { - const int D = model.hparams.neck_dim; - const int mask_hw = sam3_eff_feat_size(state, model.hparams) * 4; + ggml_backend_tensor_get(pe.pe_gaussian, state.pe_gauss_cache.data(), 0, pe_nel * sizeof(float)); + } - // Run PVS to get the segmentation mask - auto r = sam3_segment_pvs(state, model, pvs_params); - if (r.detections.empty()) { - fprintf(stderr, "%s: PVS returned no masks\n", __func__); - return -1; + for (int i = 0; i < 4; ++i) { + if (pe.point_embed[i]->type == GGML_TYPE_F16) { + std::vector tmp(D); + ggml_backend_tensor_get(pe.point_embed[i], tmp.data(), 0, D * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), state.point_emb_cache[i], D); + } else { + ggml_backend_tensor_get(pe.point_embed[i], state.point_emb_cache[i], 0, D * sizeof(float)); + } } - const auto& det = r.detections[0]; - if (det.mask.data.empty()) { - fprintf(stderr, "%s: PVS mask is empty\n", __func__); - return -1; + if (pe.not_a_point_embed->type == GGML_TYPE_F16) { + std::vector tmp(D); + ggml_backend_tensor_get(pe.not_a_point_embed, tmp.data(), 0, D * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), state.not_a_point_cache, D); + } else { + ggml_backend_tensor_get(pe.not_a_point_embed, state.not_a_point_cache, 0, D * sizeof(float)); } - int inst_id = tracker.next_inst_id++; - // tracker.frame_index points to the *next* frame to process; - // the instance is being added on the frame that was just tracked. - int fi = tracker.frame_index - 1; - if (fi < 0) fi = 0; + if (pe.no_mask_embed->type == GGML_TYPE_F16) { + std::vector tmp(D); + ggml_backend_tensor_get(pe.no_mask_embed, tmp.data(), 0, D * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), state.no_mask_emb_cache, D); + } else { + ggml_backend_tensor_get(pe.no_mask_embed, state.no_mask_emb_cache, 0, D * sizeof(float)); + } - // Create synthetic 288x288 logits from the binary mask. - // sam3_encode_memory applies sigmoid then scale/bias, so +6/-6 gives - // sigmoid values ~0.9975/~0.0025 — practically identical to real logits. - std::vector synth_logits(mask_hw * mask_hw); - { - int mw = det.mask.width, mh = det.mask.height; - for (int y = 0; y < mask_hw; ++y) { - int sy = y * mh / mask_hw; - for (int x = 0; x < mask_hw; ++x) { - int sx = x * mw / mask_hw; - synth_logits[y * mask_hw + x] = - (det.mask.data[sy * mw + sx] > 127) ? 6.0f : -6.0f; - } + state.dense_pe_cache.resize(D * H * H); + for (int row = 0; row < H; ++row) { + for (int col = 0; col < H; ++col) { + float x_norm = ((float)col + 0.5f) / (float)H; + float y_norm = ((float)row + 0.5f) / (float)H; + float pe_vec[256]; + sam3_pe_encode_coord(pe_vec, x_norm, y_norm, + state.pe_gauss_cache.data(), num_pos_feats); + for (int d = 0; d < D; ++d) + state.dense_pe_cache[d + col * D + row * D * H] = pe_vec[d]; } } - // Encode into memory bank - float obj_score = det.mask.obj_score; - if (!sam3_encode_memory(tracker, state, model, inst_id, - synth_logits.data(), mask_hw, mask_hw, - fi, true, obj_score)) { - fprintf(stderr, "%s: failed to encode memory for instance %d\n", __func__, inst_id); - return -1; + state.dense_nomask_cache.resize(D * H * H); + for (int row = 0; row < H; ++row) { + for (int col = 0; col < H; ++col) { + for (int d = 0; d < D; ++d) + state.dense_nomask_cache[d + col * D + row * D * H] = state.no_mask_emb_cache[d]; + } } - // Extract object pointer from the SAM decoder token - std::vector op(D); - if (!det.sam_token.empty()) { - sam3_extract_obj_ptr_cpu(model, det.sam_token.data(), obj_score, op.data()); - } else { - std::fill(op.begin(), op.end(), 0.0f); - } - sam3_store_obj_ptr(tracker, model, inst_id, op.data(), fi); + state.pe_cache_valid = true; + fprintf(stderr, "%s: PE cache populated (%d embeddings, %.1f KB dense grids)\n", + __func__, pe_nel, 2.0f * D * H * H * sizeof(float) / 1024.0f); +} - // Create confirmed masklet - sam3_masklet ml; - ml.instance_id = inst_id; - ml.first_frame = fi; - ml.last_seen = fi; - ml.last_score = det.score; - ml.confirmed = true; - ml.mds_sum = 1; - tracker.masklets.push_back(std::move(ml)); +// Match the official image predictor prompt ordering: +// box corners (if any) → user points → trailing padding point. +// Even when a box is present, SAM keeps the final padding token because boxes +// are merged into the point stream before calling PromptEncoder(points=..., boxes=None). +static void sam3_collect_pvs_prompt_tokens(const sam3_pvs_params& params, + std::vector& all_coords, + std::vector& all_labels) { + all_coords.clear(); + all_labels.clear(); - SAM3_LOG(2, "%s: added instance #%d (score=%.3f)\n", __func__, inst_id, det.score); - return inst_id; -} + if (params.use_box) { + all_coords.push_back(params.box.x0); + all_coords.push_back(params.box.y0); + all_labels.push_back(2); -int sam3_tracker_frame_index(const sam3_tracker& tracker) { return tracker.frame_index; } + all_coords.push_back(params.box.x1); + all_coords.push_back(params.box.y1); + all_labels.push_back(3); + } -void sam3_tracker_reset(sam3_tracker& tracker) { - tracker.frame_index = 0; - tracker.next_inst_id = 1; - tracker.masklets.clear(); - tracker.pending.clear(); - tracker.mem_banks.clear(); - tracker.ptr_banks.clear(); - for (auto* b : tracker.owned_buffers) - if (b) ggml_backend_buffer_free(b); - tracker.owned_buffers.clear(); - if (tracker.ctx) { - ggml_free(tracker.ctx); - tracker.ctx = nullptr; + for (const auto& pt : params.pos_points) { + all_coords.push_back(pt.x); + all_coords.push_back(pt.y); + all_labels.push_back(1); } - if (tracker.buffer) { - ggml_backend_buffer_free(tracker.buffer); - tracker.buffer = nullptr; + for (const auto& pt : params.neg_points) { + all_coords.push_back(pt.x); + all_coords.push_back(pt.y); + all_labels.push_back(0); } -} - -/***************************************************************************** -** Visual-only video tracking -*****************************************************************************/ -sam3_tracker_ptr sam3_create_visual_tracker( - const sam3_model& model, - const sam3_visual_track_params& params) { - sam3_video_params vp; - vp.text_prompt = ""; // no PCS detection - vp.assoc_iou_threshold = params.assoc_iou_threshold; - vp.max_keep_alive = params.max_keep_alive; - vp.recondition_every = params.recondition_every; - vp.fill_hole_area = params.fill_hole_area; - sam3_tracker_ptr tracker(new sam3_tracker()); - tracker->params = vp; - fprintf(stderr, "%s: visual-only tracker created (max_keep_alive=%d)\n", - __func__, params.max_keep_alive); - return tracker; + all_coords.push_back(0.0f); + all_coords.push_back(0.0f); + all_labels.push_back(-1); } -sam3_result sam3_propagate_frame( - sam3_tracker& tracker, sam3_state& state, - const sam3_model& model, const sam3_image& frame) { - sam3_result result; - const int D = model.hparams.neck_dim; - if (!sam3_encode_image(state, model, frame)) return result; - int fi = tracker.frame_index; - fprintf(stderr, "%s: frame %d (%zu active + %zu pending)\n", - __func__, fi, tracker.masklets.size(), tracker.pending.size()); - - // ── Propagate active masklets ──────────────────────────────────────── - std::map pm; - std::map po; - for (auto& ml : tracker.masklets) { - int id = ml.instance_id; - auto im = tracker.mem_banks.find(id); - if (im == tracker.mem_banks.end() || im->second.empty()) continue; - po[id] = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); - if (po[id].mask_logits.empty()) continue; - auto rs = sam3_bilinear_interpolate(po[id].mask_logits.data(), - po[id].mask_w, po[id].mask_h, - state.orig_width, state.orig_height); - pm[id].width = state.orig_width; - pm[id].height = state.orig_height; - pm[id].data.resize(state.orig_width * state.orig_height); - int fg = 0; - for (int p = 0; p < (int)rs.size(); ++p) { - bool f = rs[p] > 0.0f; - pm[id].data[p] = f ? 255 : 0; - if (f) fg++; - } - ml.last_score = po[id].iou_scores[0]; - ml.last_seen = fi; - float cov = (float)fg / (state.orig_width * state.orig_height); - ml.mds_sum += (cov > 0.001f && po[id].obj_score > 0.0f) ? 1 : -1; - } +// Build sparse and dense embeddings from point/box prompts +// sparse_out: [D, N_pts, 1] where N_pts = n_pos + n_neg + (use_box ? 2 : 0) + pad +// dense_out: [D, H, H, 1] (no-mask default or mask downsample) +struct sam3_pe_result { + struct ggml_tensor* sparse; // [D, N_pts, 1] + struct ggml_tensor* dense; // [D, H, H, 1] + struct ggml_tensor* image_pe; // [D, H, H, 1] — dense positional encoding grid + int n_tokens; +}; - // ── Propagate pending masklets ─────────────────────────────────────── - for (auto& ml : tracker.pending) { - int id = ml.instance_id; - auto im = tracker.mem_banks.find(id); - if (im == tracker.mem_banks.end() || im->second.empty()) continue; - auto p2 = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); - if (!p2.mask_logits.empty()) { - ml.last_score = p2.iou_scores[0]; - ml.last_seen = fi; - auto r2 = sam3_bilinear_interpolate(p2.mask_logits.data(), - p2.mask_w, p2.mask_h, - state.orig_width, state.orig_height); - int fg2 = 0; - for (auto v : r2) - if (v > 0.0f) fg2++; - float c2 = (float)fg2 / (state.orig_width * state.orig_height); - ml.mds_sum += (c2 > 0.001f && p2.obj_score > 0.0f) ? 1 : -1; - pm[id].width = state.orig_width; - pm[id].height = state.orig_height; - pm[id].data.resize(state.orig_width * state.orig_height); - for (int p = 0; p < (int)r2.size(); ++p) - pm[id].data[p] = r2[p] > 0.0f ? 255 : 0; - sam3_encode_memory(tracker, state, model, id, - p2.mask_logits.data(), p2.mask_h, p2.mask_w, - fi, false, p2.obj_score); - std::vector op(D); - sam3_extract_obj_ptr_cpu(model, p2.sam_token.data(), p2.obj_score, op.data()); - sam3_store_obj_ptr(tracker, model, id, op.data(), fi); - } - } +static sam3_pe_result sam3_build_sam_pe( + struct ggml_context* ctx, + const sam3_pvs_params& params, + int embed_dim, int feat_size) { + const int D = embed_dim; // 256 + const int H = feat_size; // 72 - // ── Encode memory for active masklets ──────────────────────────────── - for (auto& ml : tracker.masklets) { - int id = ml.instance_id; - auto it = po.find(id); - if (it == po.end() || it->second.mask_logits.empty()) continue; - sam3_encode_memory(tracker, state, model, id, - it->second.mask_logits.data(), it->second.mask_h, - it->second.mask_w, fi, false, it->second.obj_score); - std::vector op(D); - sam3_extract_obj_ptr_cpu(model, it->second.sam_token.data(), - it->second.obj_score, op.data()); - sam3_store_obj_ptr(tracker, model, id, op.data(), fi); - } + int N_pts = (int)(params.pos_points.size() + params.neg_points.size()) + 1; // pad + if (params.use_box) N_pts += 2; - // ── Update tracker state (confirmation / eviction) ─────────────────── - sam3_update_tracker(tracker, fi); + auto* sparse = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, D, N_pts, 1); + ggml_set_name(sparse, "sam_pe_sparse"); + ggml_set_input(sparse); - // ── Build result ───────────────────────────────────────────────────── - auto add_mask_to_result = [&](int inst_id, float score, const sam3_mask& mask) { - if (mask.data.empty()) return; - sam3_detection det; - det.instance_id = inst_id; - det.score = score; - det.mask = mask; - det.mask.instance_id = inst_id; - det.mask.iou_score = score; - float x0 = 1e9f, y0 = 1e9f, x1 = -1e9f, y1 = -1e9f; - for (int p = 0; p < (int)det.mask.data.size(); ++p) - if (det.mask.data[p] > 127) { - int x = p % det.mask.width, y = p / det.mask.width; - x0 = std::min(x0, (float)x); - y0 = std::min(y0, (float)y); - x1 = std::max(x1, (float)x); - y1 = std::max(y1, (float)y); - } - if (x0 <= x1) det.box = {x0, y0, x1, y1}; - result.detections.push_back(std::move(det)); - }; + auto* image_pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(image_pe, "sam_pe_image_pe"); + ggml_set_input(image_pe); - for (auto& ml : tracker.masklets) { - auto it = pm.find(ml.instance_id); - if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); - } - for (auto& ml : tracker.pending) { - auto it = pm.find(ml.instance_id); - if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); - } + auto* dense = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(dense, "sam_pe_dense"); + ggml_set_input(dense); - sam3_resolve_overlaps(result.detections); - for (auto& d : result.detections) { - if (d.mask.data.empty()) continue; - sam3_fill_holes(d.mask.data.data(), d.mask.width, d.mask.height, - tracker.params.fill_hole_area); - sam3_remove_sprinkles(d.mask.data.data(), d.mask.width, d.mask.height, - tracker.params.fill_hole_area); - } - tracker.frame_index++; - SAM3_LOG(2, "%s: frame %d done — %zu tracked\n", - __func__, fi, result.detections.size()); + sam3_pe_result result; + result.sparse = sparse; + result.dense = dense; + result.image_pe = image_pe; + result.n_tokens = N_pts; return result; } /***************************************************************************** -** Utility — image I/O +** SAM mask decoder — graph building (Phase 6, Step 6.2) *****************************************************************************/ -sam3_image sam3_load_image(const std::string& path) { - sam3_image img; - int w, h, c; - uint8_t* data = stbi_load(path.c_str(), &w, &h, &c, 3); - if (!data) { - fprintf(stderr, "%s: failed to load '%s'\n", __func__, path.c_str()); - return img; +// TwoWayAttentionBlock forward +static void sam3_twoway_block_forward( + struct ggml_context* ctx, + struct ggml_tensor*& queries, // [D, N_q, B] — modified in place + struct ggml_tensor*& keys, // [D, N_kv, B] — modified in place + struct ggml_tensor* query_pe, // [D, N_q, B] + struct ggml_tensor* key_pe, // [D, N_kv, B] + const sam3_twoway_block& blk, + int n_heads, + bool skip_first_layer_pe) { + // 1. Self-attention on queries + if (skip_first_layer_pe) { + // Python: queries = self.self_attn(q=queries, k=queries, v=queries) + // No residual connection when skipping first layer PE + queries = sam3_sam_attention(ctx, queries, queries, queries, blk.self_attn, n_heads); + } else { + auto* q = ggml_add(ctx, queries, query_pe); + auto* attn_out = sam3_sam_attention(ctx, q, q, queries, blk.self_attn, n_heads); + queries = ggml_add(ctx, queries, attn_out); + } + queries = sam3_layer_norm(ctx, queries, blk.norm1_w, blk.norm1_b); + if (skip_first_layer_pe) { + ggml_set_name(queries, "dbg_twoway_skip_sa_norm"); + ggml_set_output(queries); } - img.width = w; - img.height = h; - img.channels = 3; - img.data.assign(data, data + w * h * 3); - stbi_image_free(data); - return img; -} - -bool sam3_save_mask(const sam3_mask& mask, const std::string& path) { - if (mask.data.empty()) return false; - return stbi_write_png(path.c_str(), mask.width, mask.height, 1, - mask.data.data(), mask.width) != 0; -} - -sam3_image sam3_decode_video_frame(const std::string& video_path, int frame_index) { - sam3_image img; - - // Use ffmpeg to extract a single frame as raw RGB (frame-accurate) - char cmd[1024]; - snprintf(cmd, sizeof(cmd), - "ffmpeg -nostdin -loglevel error -i \"%s\" " - "-vf \"select=eq(n\\,%d)\" -vsync vfr -frames:v 1 " - "-f rawvideo -pix_fmt rgb24 pipe:1 2>%s", - video_path.c_str(), frame_index, SAM3_NULL_DEV); - // First, get dimensions - char info_cmd[1024]; - snprintf(info_cmd, sizeof(info_cmd), - "ffprobe -v error -select_streams v:0 " - "-show_entries stream=width,height -of csv=p=0 \"%s\" 2>%s", - video_path.c_str(), SAM3_NULL_DEV); - FILE* fp = popen(info_cmd, SAM3_POPEN_READ); - if (!fp) return img; - int w = 0, h = 0; - if (fscanf(fp, "%d,%d", &w, &h) != 2) { - pclose(fp); - return img; + // 2. Cross-attention: tokens attending to image + { + auto* q = ggml_add(ctx, queries, query_pe); + auto* k = ggml_add(ctx, keys, key_pe); + auto* attn_out = sam3_sam_attention(ctx, q, k, keys, blk.ca_tok2img, n_heads); + queries = ggml_add(ctx, queries, attn_out); + queries = sam3_layer_norm(ctx, queries, blk.norm2_w, blk.norm2_b); + } + if (skip_first_layer_pe) { + ggml_set_name(queries, "dbg_twoway_skip_ca_tok2img"); + ggml_set_output(queries); } - pclose(fp); - img.width = w; - img.height = h; - img.channels = 3; - img.data.resize(w * h * 3); + // 3. MLP on queries (ReLU activation) + { + auto* mlp = ggml_mul_mat(ctx, blk.mlp_fc1_w, queries); + mlp = ggml_add(ctx, mlp, blk.mlp_fc1_b); + mlp = ggml_relu(ctx, mlp); + mlp = ggml_mul_mat(ctx, blk.mlp_fc2_w, mlp); + mlp = ggml_add(ctx, mlp, blk.mlp_fc2_b); + queries = ggml_add(ctx, queries, mlp); + queries = sam3_layer_norm(ctx, queries, blk.norm3_w, blk.norm3_b); + } + if (skip_first_layer_pe) { + ggml_set_name(queries, "dbg_twoway_skip_mlp"); + ggml_set_output(queries); + } - fp = popen(cmd, SAM3_POPEN_READ); - if (!fp) { - img.data.clear(); - return img; + // 4. Cross-attention: image attending to tokens + { + auto* q = ggml_add(ctx, queries, query_pe); + auto* k = ggml_add(ctx, keys, key_pe); + // Note: q and k are swapped — image (k) attends to tokens (q) + auto* attn_out = sam3_sam_attention(ctx, k, q, queries, blk.ca_img2tok, n_heads); + keys = ggml_add(ctx, keys, attn_out); + keys = sam3_layer_norm(ctx, keys, blk.norm4_w, blk.norm4_b); } - size_t nread = fread(img.data.data(), 1, img.data.size(), fp); - pclose(fp); - if (nread != img.data.size()) { - img.data.clear(); + if (skip_first_layer_pe) { + ggml_set_name(keys, "dbg_twoway_skip_img2tok"); + ggml_set_output(keys); } +} - return img; +// MLP forward: N layers with ReLU (except last), optional sigmoid on last +static struct ggml_tensor* sam3_mlp_forward( + struct ggml_context* ctx, + struct ggml_tensor* x, + struct ggml_tensor* const* weights, + struct ggml_tensor* const* biases, + int n_layers, + bool sigmoid_output = false) { + for (int i = 0; i < n_layers; ++i) { + x = ggml_mul_mat(ctx, weights[i], x); + x = ggml_add(ctx, x, biases[i]); + if (i < n_layers - 1) { + x = ggml_relu(ctx, x); + } + } + if (sigmoid_output) { + x = ggml_sigmoid(ctx, x); + } + return x; } -sam3_video_info sam3_get_video_info(const std::string& video_path) { - sam3_video_info info; +// Full SAM mask decoder graph +// Inputs: +// image_feats: [D, H, H, 1] — tracker neck features (scale 2 = 72×72) +// image_pe: [D, H, H, 1] — dense positional encoding +// sparse_emb: [D, N_pts, 1] — sparse prompt embeddings +// dense_emb: [D, H, H, 1] — dense prompt embeddings (no_mask default) +// feat_s0: [D, H0, H0, 1] — high-res features (scale 0 = 288×288) +// feat_s1: [D, H1, H1, 1] — mid-res features (scale 1 = 144×144) +// Outputs: sam3_dec_result with masks, iou_pred, obj_score, sam_token_out +struct sam3_dec_result { + struct ggml_tensor* masks; // [288*288, N_masks, 1] + struct ggml_tensor* iou_pred; // [N_masks, 1] + struct ggml_tensor* obj_score; // [1, 1] + struct ggml_tensor* sam_token; // [D, 1] — for object pointer + struct ggml_tensor* mask_tokens; // [D, N_masks, 1] — raw SAM mask tokens +}; - char cmd[1024]; - snprintf(cmd, sizeof(cmd), - "ffprobe -v error -select_streams v:0 " - "-show_entries stream=width,height,r_frame_rate,nb_frames " - "-of csv=p=0 \"%s\" 2>%s", - video_path.c_str(), SAM3_NULL_DEV); - FILE* fp = popen(cmd, SAM3_POPEN_READ); - if (!fp) return info; +static sam3_dec_result sam3_build_sam_dec_graph( + struct ggml_context* ctx, + const sam3_model& model, + struct ggml_tensor* image_feats, // [D, H, H, 1] + struct ggml_tensor* image_pe, // [D, H, H, 1] + struct ggml_tensor* sparse_emb, // [D, N_pts, 1] + struct ggml_tensor* dense_emb, // [D, H, H, 1] + struct ggml_tensor* feat_s0, // [D, H*4, H*4, 1] high-res + struct ggml_tensor* feat_s1, // [D, H*2, H*2, 1] mid-res + int eff_feat_size = 0) +{ + const auto& dec = model.sam_dec; + const auto& hp = model.hparams; + const int D = hp.sam_embed_dim; // 256 + const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); + const int N_pts = (int)sparse_emb->ne[1]; + const int n_heads = 8; // SAM uses 8 heads + const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 - int w = 0, h = 0, num = 0, den = 1, nf = 0; - if (fscanf(fp, "%d,%d,%d/%d,%d", &w, &h, &num, &den, &nf) >= 4) { - info.width = w; - info.height = h; - info.fps = (den > 0) ? static_cast(num) / den : 0.0f; - info.n_frames = nf; + // ── Concatenate output tokens ──────────────────────────────────────── + // When pred_obj_scores=True: [obj_score(1,D), iou(1,D), masks(4,D)] = 6 tokens + // When pred_obj_scores=False: [iou(1,D), masks(4,D)] = 5 tokens (older SAM2) + const bool has_obj_score = (dec.obj_score_token != nullptr); + const int n_special = (has_obj_score ? 6 : 5); + + struct ggml_tensor* output_tokens; + if (has_obj_score) { + output_tokens = ggml_concat(ctx, dec.obj_score_token, dec.iou_token, 1); + } else { + output_tokens = ggml_reshape_2d(ctx, dec.iou_token, D, 1); } - pclose(fp); - return info; -} - -/***************************************************************************** -** Tokenizer — standalone test API (does not require model weights) -*****************************************************************************/ + output_tokens = ggml_concat(ctx, output_tokens, dec.mask_tokens, 1); + output_tokens = ggml_reshape_3d(ctx, output_tokens, D, n_special, 1); + auto* tokens = ggml_concat(ctx, output_tokens, sparse_emb, 1); + ggml_set_name(tokens, "sam_dec_tokens_initial"); + ggml_set_output(tokens); -// Global tokenizer instance for the test API. -static sam3_bpe_tokenizer g_test_tokenizer; -static bool g_test_tokenizer_loaded = false; + const int N_tok = 6 + N_pts; -bool sam3_test_load_tokenizer(const std::string& model_path) { - std::ifstream fin(model_path, std::ios::binary); - if (!fin) return false; + auto* src = ggml_add(ctx, image_feats, dense_emb); + src = ggml_reshape_3d(ctx, src, D, H * H, 1); + auto* pos_src = ggml_reshape_3d(ctx, image_pe, D, H * H, 1); - // Read header - uint32_t magic; - int32_t version, ftype, n_tensors; - fin.read(reinterpret_cast(&magic), 4); - fin.read(reinterpret_cast(&version), 4); - fin.read(reinterpret_cast(&ftype), 4); - fin.read(reinterpret_cast(&n_tensors), 4); - if (magic != SAM3_MAGIC || version != SAM3_FILE_VERSION) return false; + auto* queries = tokens; + auto* keys = src; + auto* query_pe = tokens; // query PE = initial point embedding + auto* key_pe = pos_src; - // Skip hparams - sam3_hparams hp; - if (!sam3_load_hparams(fin, hp)) return false; - if (hp.visual_only) return false; + for (int i = 0; i < hp.sam_dec_depth; ++i) { + sam3_twoway_block_forward(ctx, queries, keys, query_pe, key_pe, + dec.twoway_blocks[i], n_heads, + /*skip_first_layer_pe=*/(i == 0)); + sam3_name_tensorf(queries, "sam_dec_block%d_queries", i); + ggml_set_output(queries); + sam3_name_tensorf(keys, "sam_dec_block%d_keys", i); + ggml_set_output(keys); + } - // Skip tensors - for (int t = 0; t < n_tensors; ++t) { - int32_t n_dims, name_len, dtype; - fin.read(reinterpret_cast(&n_dims), 4); - fin.read(reinterpret_cast(&name_len), 4); - fin.read(reinterpret_cast(&dtype), 4); - if (fin.fail()) return false; + // Final attention: tokens → image + { + auto* q = ggml_add(ctx, queries, query_pe); + auto* k = ggml_add(ctx, keys, key_pe); + auto* attn_out = sam3_sam_attention(ctx, q, k, keys, dec.final_attn, n_heads); + queries = ggml_add(ctx, queries, attn_out); + queries = sam3_layer_norm(ctx, queries, dec.final_norm_w, dec.final_norm_b); + ggml_set_name(queries, "sam_dec_final_queries"); + } - // Read shape to compute data size - int64_t n_el = 1; - std::vector shape(n_dims); - for (int i = 0; i < n_dims; ++i) { - int32_t d; - fin.read(reinterpret_cast(&d), 4); - shape[i] = d; - n_el *= d; - } + // Debug: mark transformer outputs + ggml_set_name(queries, "dbg_dec_queries_out"); + ggml_set_output(queries); + ggml_set_name(keys, "dbg_dec_keys_out"); + ggml_set_output(keys); - // Skip name - fin.seekg(name_len, std::ios::cur); + // ── Extract output tokens ──────────────────────────────────────────── + // With pred_obj_scores=True (6 tokens): obj(0), iou(1), masks(2..5) + // With pred_obj_scores=False (5 tokens): iou(0), masks(1..4) + const int s = has_obj_score ? 1 : 0; + auto* iou_token_out = ggml_view_3d(ctx, queries, D, 1, 1, + queries->nb[1], queries->nb[2], + s * queries->nb[1]); + iou_token_out = ggml_cont(ctx, iou_token_out); // [D, 1, 1] - // Skip padding to 32-byte alignment - size_t pos = fin.tellg(); - size_t pad = (32 - pos % 32) % 32; - if (pad > 0) fin.seekg(pad, std::ios::cur); + auto* mask_tokens_out = ggml_view_3d(ctx, queries, D, num_mask_tokens, 1, + queries->nb[1], queries->nb[2], + (s + 1) * queries->nb[1]); + mask_tokens_out = ggml_cont(ctx, mask_tokens_out); // [D, 4, 1] + ggml_set_name(mask_tokens_out, "sam_dec_mask_tokens"); - // Compute data size and skip - const ggml_type file_type = static_cast(dtype); - size_t bytes; - if (ggml_is_quantized(file_type)) { - const int64_t n_rows = n_el / shape[0]; - bytes = ggml_row_size(file_type, shape[0]) * n_rows; - } else { - const size_t elem_size = (file_type == GGML_TYPE_F16) ? 2 : 4; - bytes = n_el * elem_size; - } - fin.seekg(bytes, std::ios::cur); - if (fin.fail()) return false; + struct ggml_tensor* obj_in = nullptr; + if (has_obj_score) { + obj_in = ggml_view_3d(ctx, queries, D, 1, 1, + queries->nb[1], queries->nb[2], 0); + obj_in = ggml_cont(ctx, obj_in); // [D, 1, 1] } - // Read embedded tokenizer - if (!sam3_load_bpe_vocab_from_stream(fin, g_test_tokenizer)) return false; - g_test_tokenizer_loaded = true; - return true; -} - -std::vector sam3_test_tokenize(const std::string& text) { - if (!g_test_tokenizer_loaded) return {}; - return sam3_tokenize(g_test_tokenizer, text, 32); -} + // SAM output token = first mask token, used for object pointer + auto* sam_token = ggml_view_2d(ctx, queries, D, 1, + queries->nb[1], (s + 1) * queries->nb[1]); + sam_token = ggml_cont(ctx, sam_token); // [D, 1] + ggml_set_name(sam_token, "sam_dec_sam_token"); -static bool sam3_dump_tensor_to_path(struct ggml_tensor* t, - const std::string& tensor_name, - const std::string& output_path) { - if (!t) { - fprintf(stderr, "%s: tensor '%s' is null\n", __func__, tensor_name.c_str()); - return false; - } + // Upscale: [D, H*H, 1] → ConvTranspose → high-res masks + auto* src_img = ggml_reshape_4d(ctx, keys, D, H, H, 1); + src_img = ggml_cont(ctx, ggml_permute(ctx, src_img, 2, 0, 1, 3)); - int64_t numel = 1; - for (int i = 0; i < GGML_MAX_DIMS; ++i) { - if (t->ne[i] > 0) { - numel *= t->ne[i]; - } - } + auto* up1 = ggml_conv_transpose_2d_p0(ctx, sam3_conv_transpose_weight(ctx, dec.up1_w), src_img, 2); + up1 = ggml_add(ctx, up1, ggml_reshape_4d(ctx, dec.up1_b, 1, 1, ggml_nelements(dec.up1_b), 1)); - std::vector data(numel); - if (t->type != GGML_TYPE_F16 && t->type != GGML_TYPE_F32) { - fprintf(stderr, "%s: unsupported tensor type %d for '%s'\n", - __func__, (int)t->type, tensor_name.c_str()); - return false; - } + auto* fs1 = ggml_cont(ctx, ggml_permute(ctx, feat_s1, 2, 0, 1, 3)); + auto* hs1 = ggml_conv_2d_sk_p0(ctx, dec.conv_s1_w, fs1); // [W, H, 64, B] + hs1 = ggml_add(ctx, hs1, ggml_reshape_4d(ctx, dec.conv_s1_b, 1, 1, 64, 1)); + ggml_set_name(hs1, "sam_dec_feat_s1_proj"); - const int64_t ne0 = t->ne[0]; - const int64_t ne1 = t->ne[1]; - const int64_t ne2 = t->ne[2]; - const int64_t ne3 = t->ne[3]; + // Python: act1(ln1(dc1(src) + feat_s1)) — add before LayerNorm + up1 = ggml_add(ctx, up1, hs1); + up1 = ggml_cont(ctx, ggml_permute(ctx, up1, 1, 2, 0, 3)); + up1 = sam3_layer_norm_2d(ctx, up1, dec.up1_norm_w, dec.up1_norm_b); - if (ggml_is_contiguous(t)) { - if (t->type == GGML_TYPE_F16) { - std::vector f16_data(numel); - ggml_backend_tensor_get(t, f16_data.data(), 0, numel * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(f16_data.data(), data.data(), numel); - } else { - ggml_backend_tensor_get(t, data.data(), 0, numel * sizeof(float)); - } - } else if (t->nb[0] == ggml_type_size(t->type)) { - // Serialize non-contiguous logical tensors in row-major ggml order. - if (t->type == GGML_TYPE_F16) { - std::vector row(ne0); - for (int64_t i3 = 0; i3 < ne3; ++i3) { - for (int64_t i2 = 0; i2 < ne2; ++i2) { - for (int64_t i1 = 0; i1 < ne1; ++i1) { - const size_t row_idx = ((size_t) i3 * ne2 * ne1 + (size_t) i2 * ne1 + (size_t) i1) * ne0; - const size_t offs = i3 * t->nb[3] + i2 * t->nb[2] + i1 * t->nb[1]; - ggml_backend_tensor_get(t, row.data(), offs, ne0 * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(row.data(), data.data() + row_idx, ne0); - } - } - } - } else { - for (int64_t i3 = 0; i3 < ne3; ++i3) { - for (int64_t i2 = 0; i2 < ne2; ++i2) { - for (int64_t i1 = 0; i1 < ne1; ++i1) { - const size_t row_idx = ((size_t) i3 * ne2 * ne1 + (size_t) i2 * ne1 + (size_t) i1) * ne0; - const size_t offs = i3 * t->nb[3] + i2 * t->nb[2] + i1 * t->nb[1]; - ggml_backend_tensor_get(t, data.data() + row_idx, offs, ne0 * sizeof(float)); - } - } - } - } - } else { - fprintf(stderr, "%s: unsupported non-contiguous layout for '%s' (nb0=%llu)\n", - __func__, tensor_name.c_str(), (unsigned long long) t->nb[0]); - return false; - } + up1 = ggml_gelu_erf(ctx, up1); - { - std::ofstream f(output_path + ".bin", std::ios::binary); - if (!f) return false; - f.write(reinterpret_cast(data.data()), numel * sizeof(float)); - } + // Permute back to [W, H, C, B] for next deconv + up1 = ggml_cont(ctx, ggml_permute(ctx, up1, 2, 0, 1, 3)); // [144, 144, 64, 1] - { - std::ofstream f(output_path + ".shape"); - if (!f) return false; - int ndims = ggml_n_dims(t); - for (int i = 0; i < ndims; ++i) { - if (i > 0) f << ","; - f << t->ne[i]; - } - f << "\n"; - } + // dc2: ConvTranspose2d(64, 32, k=2, s=2) → [288, 288, 32, 1] + auto* up2 = ggml_conv_transpose_2d_p0(ctx, sam3_conv_transpose_weight(ctx, dec.up2_w), up1, 2); + up2 = ggml_add(ctx, up2, ggml_reshape_4d(ctx, dec.up2_b, 1, 1, ggml_nelements(dec.up2_b), 1)); - fprintf(stderr, "%s: dumped '%s' [", __func__, tensor_name.c_str()); - for (int i = 0; i < ggml_n_dims(t); ++i) { - if (i > 0) fprintf(stderr, ", "); - fprintf(stderr, "%lld", (long long)t->ne[i]); - } - fprintf(stderr, "] to %s\n", output_path.c_str()); + // conv_s0: 1x1 conv on feat_s0 (256→32). feat_s0 is [C, W, H, B] — permute for conv. + auto* fs0 = ggml_cont(ctx, ggml_permute(ctx, feat_s0, 2, 0, 1, 3)); // [W, H, C, B] + auto* hs0 = ggml_conv_2d_sk_p0(ctx, dec.conv_s0_w, fs0); // [W, H, 32, B] + hs0 = ggml_add(ctx, hs0, ggml_reshape_4d(ctx, dec.conv_s0_b, 1, 1, 32, 1)); + ggml_set_name(hs0, "sam_dec_feat_s0_proj"); - return true; -} + // Python: act2(dc2(upscaled_embedding) + feat_s0) — no LayerNorm here + up2 = ggml_add(ctx, up2, hs0); // both [W, H, 32, B] -static bool sam3_dump_raw_f32_to_path(const float* data, - const std::vector& shape, - const std::string& output_path) { - int64_t numel = 1; - for (int64_t d : shape) { - numel *= d; - } + // Permute to [C, W, H, B] for subsequent operations + up2 = ggml_cont(ctx, ggml_permute(ctx, up2, 1, 2, 0, 3)); // [32, 288, 288, 1] - { - std::ofstream f(output_path + ".bin", std::ios::binary); - if (!f) { - return false; - } - f.write(reinterpret_cast(data), numel * sizeof(float)); - } + // GELU activation (exact, matching Python nn.GELU) + up2 = ggml_gelu_erf(ctx, up2); - { - std::ofstream f(output_path + ".shape"); - if (!f) { - return false; - } - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - f << ","; - } - f << shape[i]; - } - f << "\n"; - } + // up2: [32, 288, 288, 1] — this is our upscaled_embedding + ggml_set_name(up2, "sam_dec_upscaled"); - return true; -} + // ── Hypernetwork: predict masks ────────────────────────────────────── + // For each mask token i, pass through 3-layer MLP to get [32] vector + // Then dot product with upscaled_embedding [32, (H*4)^2] to get mask + const int H4 = H * 4; + auto* up_flat = ggml_reshape_3d(ctx, up2, 32, H4 * H4, 1); -static bool sam3_load_ref_f32_data(const std::string& path, - std::vector& data, - int expected_numel = -1) { - std::ifstream shape_f(path + ".shape"); - if (!shape_f) { - fprintf(stderr, "%s: missing %s.shape\n", __func__, path.c_str()); - return false; - } + // Process each mask token through its hypernetwork MLP + // mask_tokens_out: [D, 4, 1] + struct ggml_tensor* mask_list[4]; + for (int m = 0; m < num_mask_tokens; ++m) { + // Extract token m: [D, 1, 1] + auto* tok = ggml_view_3d(ctx, mask_tokens_out, D, 1, 1, + mask_tokens_out->nb[1], mask_tokens_out->nb[2], + m * mask_tokens_out->nb[1]); + tok = ggml_cont(ctx, tok); // [D, 1, 1] - int64_t numel = 1; - std::string line; - std::getline(shape_f, line); - size_t pos = 0; - while (pos < line.size()) { - size_t end = line.find(',', pos); - if (end == std::string::npos) { - end = line.size(); - } - if (end > pos) { - numel *= std::stoll(line.substr(pos, end - pos)); - } - pos = end + 1; - } + // MLP: 3 layers, 256→256→256→32, ReLU on first two + auto* hyper = sam3_mlp_forward(ctx, tok, + dec.hyper_w[m], dec.hyper_b[m], 3); + // hyper: [32, 1, 1] - if (expected_numel >= 0 && numel != expected_numel) { - fprintf(stderr, "%s: %s expected %d elements, got %lld\n", - __func__, path.c_str(), expected_numel, (long long)numel); - return false; + // Dot product: hyper^T @ up_flat → [1, 288*288, 1] + // Use mul_mat: up_flat^T [288*288, 32] @ hyper [32, 1] → [288*288, 1, 1] + auto* mask = ggml_mul_mat(ctx, up_flat, hyper); // [288*288, 1, 1] + mask_list[m] = mask; } - std::ifstream data_f(path + ".bin", std::ios::binary); - if (!data_f) { - fprintf(stderr, "%s: missing %s.bin\n", __func__, path.c_str()); - return false; + // Stack masks: [288*288, 4, 1] + auto* masks = mask_list[0]; + for (int m = 1; m < num_mask_tokens; ++m) { + masks = ggml_concat(ctx, masks, mask_list[m], 1); } + ggml_set_name(masks, "sam_dec_masks"); - data.resize((size_t)numel); - data_f.read(reinterpret_cast(data.data()), numel * sizeof(float)); - return data_f.good() || data_f.eof(); -} + // ── IoU prediction ─────────────────────────────────────────────────── + // iou_token_out: [D, 1, 1] + auto* iou_pred = sam3_mlp_forward(ctx, iou_token_out, + dec.iou_head_w, dec.iou_head_b, 3, + /*sigmoid_output=*/true); + // iou_pred: [4, 1, 1] → reshape to [4, 1] + iou_pred = ggml_reshape_2d(ctx, iou_pred, num_mask_tokens, 1); + ggml_set_name(iou_pred, "sam_dec_iou"); -static std::vector sam3_reorder_nchw_to_ggml_dwh(const std::vector& src, - int channels, - int height, - int width) { - std::vector dst((size_t)channels * height * width); - for (int c = 0; c < channels; ++c) { - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; ++x) { - const size_t src_idx = (size_t)c * height * width + (size_t)y * width + x; - const size_t dst_idx = (size_t)c + (size_t)x * channels + (size_t)y * channels * width; - dst[dst_idx] = src[src_idx]; - } - } + // ── Object score ───────────────────────────────────────────────────── + struct ggml_tensor* obj_score; + if (has_obj_score) { + // obj_in: [D, 1, 1] → MLP → [1, 1, 1] + obj_score = sam3_mlp_forward(ctx, obj_in, + dec.obj_head_w, dec.obj_head_b, 3); + obj_score = ggml_reshape_2d(ctx, obj_score, 1, 1); + } else { + // No obj_score prediction — return raw logit 10.0 (sigmoid ≈ 1.0, object always present) + obj_score = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, 1); + ggml_set_name(obj_score, "sam_dec_obj_score"); + ggml_set_input(obj_score); + // Mark that callers must set this to 10.0f before compute } - return dst; -} -static bool sam3_load_kv_text_file(const std::string& path, - std::map& kv) { - std::ifstream f(path); - if (!f) { - fprintf(stderr, "%s: missing %s\n", __func__, path.c_str()); - return false; - } + sam3_dec_result res; + res.masks = masks; + res.iou_pred = iou_pred; + res.obj_score = obj_score; + res.sam_token = sam_token; + res.mask_tokens = mask_tokens_out; + return res; +} - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - const size_t eq = line.find('='); - if (eq == std::string::npos) { - continue; - } - kv[line.substr(0, eq)] = line.substr(eq + 1); - } +/***************************************************************************** +** Image segmentation — PVS (visual-prompted) (Phase 6, Step 6.3) +*****************************************************************************/ - return true; -} +sam3_result sam3_segment_pvs(sam3_state& state, + const sam3_model& model, + const sam3_pvs_params& params) { +#if SAM3_LOG_LEVEL >= 1 + auto t_start = std::chrono::high_resolution_clock::now(); +#endif + const auto& hp = model.hparams; + const int D = hp.sam_embed_dim; // 256 + const int H = sam3_eff_feat_size(state, hp); + const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 + const int eff_img_size = sam3_eff_img_size(state, hp); + sam3_result result; -static int sam3_meta_get_int(const std::map& kv, - const std::string& key, - int default_value = 0) { - auto it = kv.find(key); - if (it == kv.end()) { - return default_value; + // ── Validate ───────────────────────────────────────────────────────── + if (!state.neck_trk[0]) { + fprintf(stderr, "%s: image not encoded — call sam3_encode_image first\n", __func__); + return result; } - return std::stoi(it->second); -} - -bool sam3_test_dump_text_encoder(const sam3_model& model, - const std::vector& token_ids, - const std::string& output_dir, - int n_threads) { - const int L = model.hparams.text_ctx_len; - if ((int)token_ids.size() != L) { - fprintf(stderr, "%s: expected %d token IDs, got %zu\n", - __func__, L, token_ids.size()); - return false; + if (params.pos_points.empty() && !params.use_box) { + fprintf(stderr, "%s: no prompts provided (need at least one point or box)\n", __func__); + return result; } + SAM3_LOG(2, "%s: %zu pos points, %zu neg points, box=%s, multimask=%s\n", + __func__, params.pos_points.size(), params.neg_points.size(), + params.use_box ? "yes" : "no", params.multimask ? "yes" : "no"); + + // ── Build computation graph ────────────────────────────────────────── const size_t buf_size = ggml_tensor_overhead() * 8192 + ggml_graph_overhead() * 2; struct ggml_init_params gparams = { /*.mem_size =*/buf_size, @@ -12667,1773 +9523,1556 @@ bool sam3_test_dump_text_encoder(const sam3_model& model, struct ggml_context* ctx0 = ggml_init(gparams); if (!ctx0) { fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; + return result; } - auto* inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, L); - ggml_set_name(inp_tokens, "text_token_ids"); - ggml_set_input(inp_tokens); - - auto* text_features = sam3_build_text_encoder_graph(ctx0, inp_tokens, model); - std::vector tensor_names = { - "causal_mask", - "text_token_embed", - "text_after_pos_embed", - "text_final_ln", - "text_features_2d", - }; - for (int i = 0; i < model.hparams.text_layers; ++i) { - char name[64]; - - snprintf(name, sizeof(name), "text_block_%02d_after_ln1", i); - tensor_names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_qkv", i); - tensor_names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_attn_out", i); - tensor_names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_after_attn_residual", i); - tensor_names.emplace_back(name); + // ── SAM prompt encoder (CPU pre-compute + input tensors) ───────────── + auto pe_out = sam3_build_sam_pe(ctx0, params, D, H); - snprintf(name, sizeof(name), "text_block_%02d_after_ln2", i); - tensor_names.emplace_back(name); + // ── Create fresh input tensors for tracker features ────────────────── + // CRITICAL: Do NOT use state.neck_trk[*] directly in graph operations + // (like ggml_add). They are tensors from a previous graph, and + // ggml_build_forward_expand traces all ancestors — which would pull in + // the ENTIRE ViT + neck recomputation (2500+ nodes, ~37 seconds). + // Instead: create fresh input tensors and copy data from state on CPU. + const int H0 = H * 4; // 288 + const int H1 = H * 2; // 144 - snprintf(name, sizeof(name), "text_block_%02d_mlp_fc1", i); - tensor_names.emplace_back(name); + auto* image_feats = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(image_feats, "sam_dec_image_feats"); + ggml_set_input(image_feats); - snprintf(name, sizeof(name), "text_block_%02d_mlp_gelu", i); - tensor_names.emplace_back(name); + auto* feat_s0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); + ggml_set_name(feat_s0, "pvs_feat_s0"); + ggml_set_input(feat_s0); - snprintf(name, sizeof(name), "text_block_%02d_mlp_out", i); - tensor_names.emplace_back(name); + auto* feat_s1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); + ggml_set_name(feat_s1, "pvs_feat_s1"); + ggml_set_input(feat_s1); - snprintf(name, sizeof(name), "text_block_%02d_out", i); - tensor_names.emplace_back(name); - } + // ── SAM mask decoder graph ─────────────────────────────────────────── + auto dec_out = sam3_build_sam_dec_graph(ctx0, model, + image_feats, + pe_out.image_pe, + pe_out.sparse, + pe_out.dense, + feat_s0, + feat_s1, H); - ggml_set_output(text_features); - for (const auto& name : tensor_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t && t != inp_tokens) { - ggml_set_output(t); - } - } + // Mark outputs + ggml_set_output(dec_out.masks); + ggml_set_output(dec_out.iou_pred); + ggml_set_output(dec_out.obj_score); + ggml_set_output(dec_out.sam_token); - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 8192, false); - ggml_build_forward_expand(graph, text_features); + // ── Build and allocate graph ───────────────────────────────────────── + struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 32768, false); + ggml_build_forward_expand(graph, dec_out.masks); + ggml_build_forward_expand(graph, dec_out.iou_pred); + ggml_build_forward_expand(graph, dec_out.obj_score); + ggml_build_forward_expand(graph, dec_out.sam_token); auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate text encoder graph\n", __func__); + if (!ggml_gallocr_reserve(galloc, graph)) { + fprintf(stderr, "%s: failed to reserve graph memory\n", __func__); ggml_gallocr_free(galloc); ggml_free(ctx0); - return false; + return result; } - - ggml_backend_tensor_set(inp_tokens, token_ids.data(), 0, L * sizeof(int32_t)); - - auto* causal_mask = ggml_get_tensor(ctx0, "causal_mask"); - if (!causal_mask) { - fprintf(stderr, "%s: causal_mask tensor not found\n", __func__); + if (!ggml_gallocr_alloc_graph(galloc, graph)) { + fprintf(stderr, "%s: failed to allocate graph\n", __func__); ggml_gallocr_free(galloc); ggml_free(ctx0); - return false; + return result; } - std::vector mask_data(L * L); - sam3_fill_causal_mask(mask_data.data(), L); - ggml_backend_tensor_set(causal_mask, mask_data.data(), 0, L * L * sizeof(ggml_fp16_t)); + SAM3_LOG(2, "%s: graph allocated, %d nodes\n", __func__, ggml_graph_n_nodes(graph)); - sam3_graph_compute(model.backend, graph, n_threads); + // Set default obj_score when pred_obj_scores=False (older SAM2 models) + if (!model.sam_dec.obj_score_token) { + auto* t = ggml_graph_get_tensor(graph, "sam_dec_obj_score"); + if (t) { float v = 10.0f; ggml_backend_tensor_set(t, &v, 0, sizeof(float)); } + } - bool ok = true; - for (const auto& name : tensor_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in graph\n", __func__, name.c_str()); - ok = false; - continue; + // ── Upload input data (using cached embeddings) ──────────────────── + // Populate PE cache on first call (reads model weights from GPU once) + sam3_populate_pe_cache(state, model); + + { + const int N_pts = pe_out.n_tokens; + const int num_pos_feats = D / 2; + + std::vector all_coords; + std::vector all_labels; + sam3_collect_pvs_prompt_tokens(params, all_coords, all_labels); + + // Sparse embeddings — only this changes per call + std::vector sparse_data(N_pts * D, 0.0f); + for (int p = 0; p < N_pts; ++p) { + // Scale from original image space to model input space, then shift to pixel center + float px = all_coords[p * 2 + 0] / (float)state.orig_width * (float)eff_img_size + 0.5f; + float py = all_coords[p * 2 + 1] / (float)state.orig_height * (float)eff_img_size + 0.5f; + float x_norm = px / (float)eff_img_size; + float y_norm = py / (float)eff_img_size; + float pe_vec[256]; + sam3_pe_encode_coord(pe_vec, x_norm, y_norm, + state.pe_gauss_cache.data(), num_pos_feats); + int label = all_labels[p]; + if (label == -1) { + for (int d = 0; d < D; ++d) + sparse_data[p * D + d] = state.not_a_point_cache[d]; + } else { + for (int d = 0; d < D; ++d) + sparse_data[p * D + d] = pe_vec[d] + state.point_emb_cache[label][d]; + } } - if (!sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; + ggml_backend_tensor_set(pe_out.sparse, sparse_data.data(), 0, N_pts * D * sizeof(float)); + + // Dump sparse embeddings if requested + { + const char* dd = getenv("SAM2_DUMP_DIR"); + if (dd) { + char p[512]; snprintf(p, sizeof(p), "%s/cpp_sparse_emb.bin", dd); + FILE* f = fopen(p, "wb"); + if (f) { fwrite(sparse_data.data(), sizeof(float), N_pts * D, f); fclose(f); } + fprintf(stderr, " [DUMP] cpp_sparse_emb: %d tokens x %d dims\n", N_pts, D); + } } + + // Dense PE grid and no-mask embedding — use pre-computed caches + ggml_backend_tensor_set(pe_out.image_pe, state.dense_pe_cache.data(), + 0, D * H * H * sizeof(float)); + ggml_backend_tensor_set(pe_out.dense, state.dense_nomask_cache.data(), + 0, D * H * H * sizeof(float)); } - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; -} + // ── Copy tracker features from state to fresh input tensors ───────── + // image_feats = neck_trk[2] + no_mem_embed (computed on CPU) + { + const int n2 = D * H * H; + std::vector trk2(n2), no_mem_data(D); + ggml_backend_tensor_get(state.neck_trk[2], trk2.data(), 0, n2 * sizeof(float)); + ggml_backend_tensor_get(model.tensors.at("no_mem_embed"), no_mem_data.data(), 0, D * sizeof(float)); + // Add no_mem_embed (broadcast [D] to [D, H, H]) + for (int s = 0; s < H * H; ++s) + for (int d = 0; d < D; ++d) + trk2[d + s * D] += no_mem_data[d]; + ggml_backend_tensor_set(image_feats, trk2.data(), 0, n2 * sizeof(float)); -bool sam3_test_dump_phase5(const sam3_model& model, - const sam3_state& state, - const std::vector& token_ids, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; - const int H = hp.n_img_embd(); - const int L = hp.text_ctx_len; - const int NQ = hp.ddec_num_queries; + // Dump image_feats (with no_mem_embed) if requested + { + const char* dd = getenv("SAM2_DUMP_DIR"); + if (dd) { + char p[512]; snprintf(p, sizeof(p), "%s/cpp_image_feats.bin", dd); + FILE* f = fopen(p, "wb"); + if (f) { fwrite(trk2.data(), sizeof(float), n2, f); fclose(f); } + fprintf(stderr, " [DUMP] cpp_image_feats: [%d, %d, %d]\n", D, H, H); + } + } - if ((int)token_ids.size() != L) { - fprintf(stderr, "%s: expected %d token IDs, got %zu\n", - __func__, L, token_ids.size()); - return false; + // feat_s0 = neck_trk[0], feat_s1 = neck_trk[1] + const int n0 = D * H0 * H0; + const int n1 = D * H1 * H1; + std::vector trk0(n0), trk1(n1); + ggml_backend_tensor_get(state.neck_trk[0], trk0.data(), 0, n0 * sizeof(float)); + ggml_backend_tensor_set(feat_s0, trk0.data(), 0, n0 * sizeof(float)); + ggml_backend_tensor_get(state.neck_trk[1], trk1.data(), 0, n1 * sizeof(float)); + ggml_backend_tensor_set(feat_s1, trk1.data(), 0, n1 * sizeof(float)); } - if (!state.neck_det[0] || !state.neck_det_pe[2]) { - fprintf(stderr, "%s: encoded detector features are missing\n", __func__); - return false; + + // ── Compute ────────────────────────────────────────────────────────── + { +#if SAM3_LOG_LEVEL >= 1 + auto t0 = std::chrono::high_resolution_clock::now(); +#endif + if (!sam3_graph_compute(model.backend, graph, state.n_threads)) { + ggml_gallocr_free(galloc); + ggml_free(ctx0); + return result; + } +#if SAM3_LOG_LEVEL >= 1 + auto t1 = std::chrono::high_resolution_clock::now(); + double ms = std::chrono::duration(t1 - t0).count(); + SAM3_LOG(1, "%s: graph computed in %.1f ms (%d threads)\n", + __func__, ms, state.n_threads); +#endif + } + + // ── Dump decoder outputs if SAM2_DUMP_DIR set ────────────────────── + { + const char* dump_dir = getenv("SAM2_DUMP_DIR"); + if (dump_dir) { + auto dump_t = [&](const char* name, struct ggml_tensor* t) { + if (!t) return; + int64_t nb = ggml_nbytes(t); + std::vector buf(nb); + ggml_backend_tensor_get(t, buf.data(), 0, nb); + char path[512]; + snprintf(path, sizeof(path), "%s/%s.bin", dump_dir, name); + FILE* f = fopen(path, "wb"); + if (f) { fwrite(buf.data(), 1, nb, f); fclose(f); } + snprintf(path, sizeof(path), "%s/%s.shape", dump_dir, name); + f = fopen(path, "w"); + if (f) { + fprintf(f, "%lld,%lld,%lld,%lld", + (long long)t->ne[0], (long long)t->ne[1], + (long long)t->ne[2], (long long)t->ne[3]); + fclose(f); + } + fprintf(stderr, " [DUMP] %s: [%lld,%lld,%lld,%lld]\n", name, + (long long)t->ne[0], (long long)t->ne[1], + (long long)t->ne[2], (long long)t->ne[3]); + }; + dump_t("cpp_pvs_masks", dec_out.masks); + dump_t("cpp_pvs_iou", dec_out.iou_pred); + dump_t("cpp_pvs_obj_score", dec_out.obj_score); + // Decoder transformer intermediates + dump_t("cpp_dec_queries", ggml_graph_get_tensor(graph, "dbg_dec_queries_out")); + dump_t("cpp_dec_keys", ggml_graph_get_tensor(graph, "dbg_dec_keys_out")); + // Block 0 internals + const char* b0_names[] = {"sam_dec_tokens_initial", + "dbg_twoway_skip_sa_norm", "dbg_twoway_skip_ca_tok2img", + "dbg_twoway_skip_mlp", "dbg_twoway_skip_img2tok", + "dbg_sa0_Q_proj", "dbg_sa0_merged"}; + for (auto* bn : b0_names) dump_t(bn, ggml_graph_get_tensor(graph, bn)); + // Per-block outputs + for (int bi = 0; bi < 2; bi++) { + char bn[64]; + snprintf(bn, sizeof(bn), "sam_dec_block%d_queries", bi); + dump_t(bn, ggml_graph_get_tensor(graph, bn)); + snprintf(bn, sizeof(bn), "sam_dec_block%d_keys", bi); + dump_t(bn, ggml_graph_get_tensor(graph, bn)); + } + } } - const size_t buf_size = ggml_tensor_overhead() * 65536 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; - } + // ── Read outputs ───────────────────────────────────────────────────── + // masks: [H*4×H*4, 4, 1] + const int mask_hw = H * 4; + std::vector masks_data(mask_hw * mask_hw * num_mask_tokens); + ggml_backend_tensor_get(dec_out.masks, masks_data.data(), 0, masks_data.size() * sizeof(float)); + + // IoU predictions: [4, 1] + std::vector iou_data(num_mask_tokens); + ggml_backend_tensor_get(dec_out.iou_pred, iou_data.data(), 0, num_mask_tokens * sizeof(float)); + + // Object score: [1, 1] + float obj_logit = 0.0f; + ggml_backend_tensor_get(dec_out.obj_score, &obj_logit, 0, sizeof(float)); + float obj_score = 1.0f / (1.0f + expf(-obj_logit)); + + // SAM output token: [D, 1] — needed for object pointer extraction in tracking + std::vector sam_token_data(D); + ggml_backend_tensor_get(dec_out.sam_token, sam_token_data.data(), 0, D * sizeof(float)); + + SAM3_LOG(2, "%s: obj_score=%.4f (logit=%.4f), iou=[%.3f, %.3f, %.3f, %.3f]\n", + __func__, obj_score, obj_logit, + iou_data[0], iou_data[1], iou_data[2], iou_data[3]); - auto* inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, L); - ggml_set_name(inp_tokens, "text_token_ids"); - ggml_set_input(inp_tokens); + // ── Select masks based on multimask mode ───────────────────────────── + // Python: if multimask_output → masks[:, 1:, :, :], iou_pred[:, 1:] + // else → masks[:, 0:1, :, :], iou_pred[:, 0:1] + int start_idx, end_idx; + if (params.multimask) { + start_idx = 1; + end_idx = num_mask_tokens; + } else { + start_idx = 0; + end_idx = 1; + } - auto* text_features_2d = sam3_build_text_encoder_graph(ctx0, inp_tokens, model); - auto* text_features = ggml_reshape_3d(ctx0, text_features_2d, D, L, 1); - ggml_set_name(text_features, "text_features"); + for (int m = start_idx; m < end_idx; ++m) { + sam3_detection det; + det.sam_token = sam_token_data; - // Make a snapshot copy of text_features for dumping — the graph allocator - // may reuse the view's underlying buffer for later ops. - auto* text_features_snap = ggml_cont(ctx0, ggml_reshape_2d(ctx0, text_features_2d, D, L)); - ggml_set_name(text_features_snap, "text_features_snap"); + // Resize mask from 288×288 to original image size + const float* mask_ptr = masks_data.data() + m * mask_hw * mask_hw; + auto mask_resized = sam3_bilinear_interpolate(mask_ptr, mask_hw, mask_hw, + state.orig_width, state.orig_height); - auto* img_feats = ggml_reshape_3d(ctx0, state.neck_det[2], D, H * H, 1); - auto* img_pe = ggml_reshape_3d(ctx0, state.neck_det_pe[2], D, H * H, 1); + // Binarize at threshold 0.0 (sigmoid(logit) > 0.5 ↔ logit > 0.0) + det.mask.width = state.orig_width; + det.mask.height = state.orig_height; + det.mask.data.resize(state.orig_width * state.orig_height); + for (int i = 0; i < (int)mask_resized.size(); ++i) { + det.mask.data[i] = (mask_resized[i] > 0.0f) ? 255 : 0; + } - auto* sine_dim_t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, 64); - ggml_set_name(sine_dim_t, "sine_dim_t"); - ggml_set_input(sine_dim_t); + det.mask.iou_score = iou_data[m]; + det.mask.obj_score = obj_score; + det.mask.instance_id = m; + det.score = iou_data[m]; + det.iou_score = iou_data[m]; + det.instance_id = m; - auto* rpb_coords = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, H); - ggml_set_name(rpb_coords, "rpb_coords"); - ggml_set_input(rpb_coords); + // Compute bounding box from mask + int min_x = state.orig_width, min_y = state.orig_height; + int max_x = 0, max_y = 0; + for (int y = 0; y < state.orig_height; ++y) { + for (int x = 0; x < state.orig_width; ++x) { + if (det.mask.data[y * state.orig_width + x] > 0) { + min_x = std::min(min_x, x); + min_y = std::min(min_y, y); + max_x = std::max(max_x, x); + max_y = std::max(max_y, y); + } + } + } + det.box = {(float)min_x, (float)min_y, (float)max_x, (float)max_y}; - auto* text_valid_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, L, 1, 1); - ggml_set_name(text_valid_mask, "text_valid_mask"); - ggml_set_input(text_valid_mask); + result.detections.push_back(std::move(det)); + } - auto* text_attn_bias = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, L, 1, 1); - ggml_set_name(text_attn_bias, "text_attn_bias"); - ggml_set_input(text_attn_bias); + SAM3_LOG(2, "%s: %zu masks returned\n", __func__, result.detections.size()); - auto* conditioned = sam3_build_fenc_graph(ctx0, model, img_feats, text_features, - img_pe, text_attn_bias); - auto* fenc_layer5_out = ggml_cont(ctx0, conditioned); - ggml_set_name(fenc_layer5_out, "fenc_layer5_out"); - ggml_set_name(conditioned, "fenc_output"); + // ── Cleanup ────────────────────────────────────────────────────────── + ggml_gallocr_free(galloc); + ggml_free(ctx0); - auto ddec_out = sam3_build_ddec_graph(ctx0, model, conditioned, img_pe, text_features, - sine_dim_t, rpb_coords, text_attn_bias, text_valid_mask); +#if SAM3_LOG_LEVEL >= 1 + auto t_end = std::chrono::high_resolution_clock::now(); + double total_ms = std::chrono::duration(t_end - t_start).count(); + SAM3_LOG(1, "%s: completed in %.1f ms\n", __func__, total_ms); +#endif - struct ggml_tensor* fpn_feats[3] = { - state.neck_det[0], - state.neck_det[1], - state.neck_det[2], - }; + return result; +} - auto* obj_queries = ggml_view_3d(ctx0, ddec_out.queries, D, NQ, 1, - ddec_out.queries->nb[1], ddec_out.queries->nb[2], - 1 * ddec_out.queries->nb[1]); - obj_queries = ggml_cont(ctx0, obj_queries); +/***************************************************************************** +** Video tracking (Phase 7) +*****************************************************************************/ - auto* mask_logits = sam3_build_seg_head_graph(ctx0, model, conditioned, fpn_feats, - obj_queries, text_features, text_attn_bias); - ggml_set_name(mask_logits, "seg_mask_logits"); +struct sam3_prop_output { + std::vector mask_logits; + std::vector iou_scores; + float obj_score; + std::vector sam_token; + int n_masks, mask_h, mask_w; +}; - struct named_tensor { - const char* name; - struct ggml_tensor* tensor; - }; +// Lazily compute and cache PE/RoPE data that is identical across all propagation calls. +static void sam3_ensure_tracker_pe_caches(sam3_tracker& tracker, const sam3_hparams& hp, + int eff_feat_size = 0) { + const int H = (eff_feat_size > 0) ? eff_feat_size : hp.feat_size(); + if (tracker.pe_caches_valid) return; - const std::vector direct_tensors = { - {"text_features", text_features_snap}, - {"text_valid_mask", text_valid_mask}, - {"fenc_img_input", img_feats}, - {"fenc_pos_embed", img_pe}, - {"fenc_prompt", text_features_snap}, - {"img_pe_72", state.neck_det_pe[2]}, - {"ddec_query_embed", model.ddec.query_embed}, - {"ddec_ref_pts_raw", model.tensors.at("ddec.reference_points.weight")}, - {"ddec_presence_token", model.ddec.presence_token}, - {"ddec_pred_boxes", ddec_out.pred_boxes}, - {"ddec_presence_logit", ddec_out.presence_score}, - }; + const int D = hp.neck_dim; // 256 + const int MD = hp.mem_out_dim; // 64 + const int N = H * H; + const int half_d = D / 2; // 128 - std::vector named_outputs = { - "fenc_output", - "ddec_ref_boxes_init", - "ddec_query_sine_0", - "ddec_query_pos_0", - "ddec_rpb_mask_0", - "ddec_layer0_after_sa", - "ddec_layer0_after_text_ca", - "ddec_layer0_after_img_ca", - "ddec_layer0_full_out", - "ddec_layer0_presence", - "ddec_normed_output", - "scoring_prompt_mlp_out", - "scoring_pooled", - "scoring_proj_pooled", - "scoring_proj_hs", - "scoring_class_scores", - "seg_enc_after_ca", - "seg_enc_visual", - "seg_pixel_dec_stage0", - "seg_pixel_dec_stage1", - "seg_pixel_decoder_out", - "seg_instance_embed", - "seg_mask_embed", - "seg_mask_logits", - }; + tracker.cached_sinpe_256 = sam3_sinusoidal_pe_2d(H, H, D); + tracker.cached_sinpe_64 = sam3_sinusoidal_pe_2d(H, H, MD); - for (int i = 0; i < hp.fenc_layers; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - named_outputs.emplace_back(name); - } - for (int i = 0; i < hp.ddec_layers; ++i) { - char out_name[64]; - char box_name[64]; - snprintf(out_name, sizeof(out_name), "ddec_layer%d_out", i); - snprintf(box_name, sizeof(box_name), "ddec_layer%d_refboxes", i); - named_outputs.emplace_back(out_name); - named_outputs.emplace_back(box_name); - } - - ggml_set_output(text_features_snap); - ggml_set_output(text_valid_mask); - ggml_set_output(img_feats); - ggml_set_output(img_pe); - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_set_output(t); - } - } - ggml_set_output(mask_logits); - ggml_set_output(ddec_out.class_scores); - ggml_set_output(ddec_out.pred_boxes); - ggml_set_output(ddec_out.presence_score); - - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 65536, false); - ggml_build_forward_expand(graph, ddec_out.class_scores); - ggml_build_forward_expand(graph, ddec_out.pred_boxes); - ggml_build_forward_expand(graph, ddec_out.presence_score); - ggml_build_forward_expand(graph, mask_logits); - ggml_build_forward_expand(graph, text_features_snap); - ggml_build_forward_expand(graph, text_valid_mask); - ggml_build_forward_expand(graph, img_feats); - ggml_build_forward_expand(graph, img_pe); - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_build_forward_expand(graph, t); + // Compute axial CIS and reorder to [2, half_d, N] layout + std::vector rope_raw(N * D); + sam3_compute_axial_cis(rope_raw.data(), D, H, H, 10000.0f, 1.0f); + tracker.cached_axial_cis_reord.resize(2 * half_d * N); + for (int n = 0; n < N; ++n) + for (int i = 0; i < half_d; ++i) { + tracker.cached_axial_cis_reord[0 + i * 2 + n * D] = rope_raw[n * D + i * 2 + 0]; + tracker.cached_axial_cis_reord[1 + i * 2 + n * D] = rope_raw[n * D + i * 2 + 1]; } - } - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate phase 5 graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; + // EdgeTAM: compute 16x16 RoPE for cross-attention K (perceiver 2D latents) + if (hp.has_perceiver) { + const int K16 = 16; // perceiver 2D grid size + const int NK = K16 * K16; // 256 tokens + std::vector rope_k16_raw(NK * D); + sam3_compute_axial_cis(rope_k16_raw.data(), D, K16, K16, 10000.0f, 1.0f); + tracker.cached_axial_cis_k16_reord.resize(2 * half_d * NK); + for (int n = 0; n < NK; ++n) + for (int i = 0; i < half_d; ++i) { + tracker.cached_axial_cis_k16_reord[0 + i * 2 + n * D] = rope_k16_raw[n * D + i * 2 + 0]; + tracker.cached_axial_cis_k16_reord[1 + i * 2 + n * D] = rope_k16_raw[n * D + i * 2 + 1]; + } + SAM3_LOG(2, "%s: EdgeTAM K16 RoPE cache: %zu floats\n", __func__, + tracker.cached_axial_cis_k16_reord.size()); } - ggml_backend_tensor_set(inp_tokens, token_ids.data(), 0, L * sizeof(int32_t)); + tracker.pe_caches_valid = true; + SAM3_LOG(2, "%s: tracker PE caches populated (%.1f KB)\n", __func__, + (tracker.cached_sinpe_256.size() + tracker.cached_sinpe_64.size() + + tracker.cached_axial_cis_reord.size()) * sizeof(float) / 1024.0f); +} - auto* causal_mask = ggml_get_tensor(ctx0, "causal_mask"); - if (causal_mask) { - std::vector mask_data(L * L); - sam3_fill_causal_mask(mask_data.data(), L); - ggml_backend_tensor_set(causal_mask, mask_data.data(), 0, L * L * sizeof(ggml_fp16_t)); - } +static sam3_prop_output sam3_propagate_single( + sam3_tracker& tracker, sam3_state& state, const sam3_model& model, + const sam3_masklet& masklet, + const std::vector& mem_bank, + const std::vector>& ptr_bank) { + sam3_prop_output output = {}; + const auto& hp = model.hparams; + const int D = hp.neck_dim, MD = hp.mem_out_dim; + const int H = sam3_eff_feat_size(state, hp); + const int N = H * H; - { - std::vector dim_t_data(64); - for (int i = 0; i < 64; ++i) { - dim_t_data[i] = 2.0f * (float)M_PI / powf(10000.0f, 2.0f * (float)i / 128.0f); - } - ggml_backend_tensor_set(sine_dim_t, dim_t_data.data(), 0, 64 * sizeof(float)); - } + auto sel = sam3_select_memory_frames(mem_bank, hp.num_maskmem); + if (sel.empty()) return output; - { - std::vector coords(H); - for (int i = 0; i < H; ++i) { - coords[i] = (float)i / (float)H; + // ── Build prompt and prompt_pos via sam3_build_prompt_and_pos ───────── + // EdgeTAM with perceiver: each slot has 512 tokens (not H*H). + // Standard SAM2/SAM3: each slot has H*H tokens. + const bool use_perceiver = hp.has_perceiver != 0; + const int N_per_slot = use_perceiver ? (hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d) : N; + + int n_sel = (int)sel.size(); + std::vector> slot_feats(n_sel), slot_pes(n_sel); + std::vector spatial_tpos(n_sel, 1); // default t_pos=1 for non-cond + for (int s = 0; s < n_sel; ++s) { + slot_feats[s].resize(MD * N_per_slot); + ggml_backend_tensor_get(mem_bank[sel[s]].spatial_feats, + slot_feats[s].data(), 0, MD * N_per_slot * sizeof(float)); + slot_pes[s].resize(MD * N_per_slot); + if (mem_bank[sel[s]].spatial_pe) { + ggml_backend_tensor_get(mem_bank[sel[s]].spatial_pe, + slot_pes[s].data(), 0, MD * N_per_slot * sizeof(float)); + } else { + sam3_ensure_tracker_pe_caches(tracker, hp, H); + if (use_perceiver) { + // For perceiver: zeros for 1D tokens, sinusoidal for 2D tokens + const int N_1d = hp.perceiver_n_latents_1d; + const int N_2d = hp.perceiver_n_latents_2d; + memset(slot_pes[s].data(), 0, MD * N_1d * sizeof(float)); + auto pe_2d = sam3_sinusoidal_pe_2d(16, 16, MD); + memcpy(slot_pes[s].data() + MD * N_1d, pe_2d.data(), MD * N_2d * sizeof(float)); + } else { + slot_pes[s] = tracker.cached_sinpe_64; + } } - ggml_backend_tensor_set(rpb_coords, coords.data(), 0, H * sizeof(float)); + spatial_tpos[s] = mem_bank[sel[s]].is_cond_frame ? 0 : (n_sel - s); } - auto* qpos_pres = ggml_get_tensor(ctx0, "ddec_query_pos_pres"); - if (qpos_pres) { - std::vector zeros(D, 0.0f); - ggml_backend_tensor_set(qpos_pres, zeros.data(), 0, D * sizeof(float)); + int P = std::min((int)ptr_bank.size(), hp.max_obj_ptrs); + std::vector> obj_ptrs(P); + std::vector ptr_tpos(P); + int cur_frame = tracker.frame_index; + for (int p = 0; p < P; ++p) { + obj_ptrs[p].resize(D); + ggml_backend_tensor_get(ptr_bank[p].second, obj_ptrs[p].data(), 0, D * sizeof(float)); + // Use actual frame distance (matches Python: abs(frame_idx - t)) + ptr_tpos[p] = std::abs(cur_frame - ptr_bank[p].first); + if (ptr_tpos[p] < 1) ptr_tpos[p] = 1; // minimum distance of 1 } - auto* rpb_pz = ggml_get_tensor(ctx0, "rpb_pres_zeros"); - if (rpb_pz) { - int n = (int)(rpb_pz->ne[0] * rpb_pz->ne[1] * rpb_pz->ne[2] * rpb_pz->ne[3]); - std::vector zeros(n, 0.0f); - ggml_backend_tensor_set(rpb_pz, zeros.data(), 0, n * sizeof(float)); - } + auto pd = sam3_build_prompt_and_pos(model, slot_feats, slot_pes, spatial_tpos, obj_ptrs, ptr_tpos, H); - { - int n_valid = 0; - for (int i = 0; i < L; ++i) { - if (token_ids[i] != 0) { - ++n_valid; + // ── RoPE frequencies (cached) ────────────────────────────────────── + sam3_ensure_tracker_pe_caches(tracker, hp, H); + const int half_d = D / 2; // 128 + const auto& rope_q_reord = tracker.cached_axial_cis_reord; + // For cross-attn K: build rope_k_data for all M_spatial tokens + std::vector rope_k_data; + if (pd.M_spatial > 0) { + rope_k_data.resize(2 * half_d * pd.M_spatial); + if (use_perceiver) { + // EdgeTAM perceiver: each frame has N_per_slot=512 tokens. + // First 256 (1D latents): identity RoPE (cos=1, sin=0). + // Last 256 (2D latents): 16x16 RoPE from cached_axial_cis_k16_reord. + const int N_1d = hp.perceiver_n_latents_1d; // 256 + const int N_2d = hp.perceiver_n_latents_2d; // 256 + const auto& rope_k16 = tracker.cached_axial_cis_k16_reord; // [2, 128, 256] + for (int s = 0; s < n_sel; ++s) { + float* dst = rope_k_data.data() + s * D * N_per_slot; + // 1D tokens: identity RoPE (cos=1, sin=0) in [2, half_d, N_1d] layout + // Layout: for token n, dim i: cos at [0 + i*2 + n*D], sin at [1 + i*2 + n*D] + for (int n = 0; n < N_1d; ++n) + for (int i = 0; i < half_d; ++i) { + dst[0 + i * 2 + n * D] = 1.0f; // cos = 1 + dst[1 + i * 2 + n * D] = 0.0f; // sin = 0 + } + // 2D tokens: copy 16x16 RoPE + float* dst_2d = dst + D * N_1d; + memcpy(dst_2d, rope_k16.data(), D * N_2d * sizeof(float)); } + } else { + // Standard: repeat HxH axial CIS for each memory frame + for (int s = 0; s < pd.M_spatial / N; ++s) + memcpy(rope_k_data.data() + s * D * N, rope_q_reord.data(), D * N * sizeof(float)); } - if (n_valid == 0) { - n_valid = 1; - } - - const float scale = (float)L / (float)n_valid; - std::vector valid_mask(L); - std::vector attn_bias(L); - for (int i = 0; i < L; ++i) { - const bool is_valid = token_ids[i] != 0; - valid_mask[i] = is_valid ? scale : 0.0f; - attn_bias[i] = is_valid ? 0.0f : -1.0e9f; - } - ggml_backend_tensor_set(text_valid_mask, valid_mask.data(), 0, L * sizeof(float)); - ggml_backend_tensor_set(text_attn_bias, attn_bias.data(), 0, L * sizeof(float)); } - sam3_graph_compute(model.backend, graph, n_threads); + // ── Build graph ───────────────────────────────────────────────────── + const size_t buf_size = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; + struct ggml_init_params gparams = {buf_size, nullptr, true}; + auto* ctx0 = ggml_init(gparams); + if (!ctx0) return output; - bool ok = true; - for (const auto& item : direct_tensors) { - if (!sam3_dump_tensor_to_path(item.tensor, item.name, output_dir + "/" + item.name)) { - ok = false; - } - } - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in graph\n", __func__, name.c_str()); - ok = false; - continue; - } - if (!sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; - } - } + // CRITICAL: create fresh input tensors for state features. + // Using state.neck_trk[*] directly as ggml_reshape operands pulls in + // the entire ViT+neck recomputation from the image encoder graph. + auto* curr = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); + ggml_set_name(curr, "prop_curr"); + ggml_set_input(curr); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; -} + // src_pos (sinusoidal PE 256-dim for 72×72) + auto* src_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); + ggml_set_name(src_pos_t, "src_pos"); + ggml_set_input(src_pos_t); -bool sam3_test_dump_phase5_from_ref_inputs(const sam3_model& model, - const std::vector& token_ids, - const std::string& prephase_ref_dir, - const std::string& phase5_ref_dir, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; - const int H = hp.n_img_embd(); - const int L = hp.text_ctx_len; - const int NQ = hp.ddec_num_queries; - const int H1 = H * 2; - const int H0 = H * 4; - - if ((int)token_ids.size() != L) { - fprintf(stderr, "%s: expected %d token IDs, got %zu\n", - __func__, L, token_ids.size()); - return false; - } + // Prompt and prompt_pos + auto* prompt_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); + ggml_set_name(prompt_t, "prompt"); + ggml_set_input(prompt_t); + auto* prompt_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); + ggml_set_name(prompt_pos_t, "prompt_pos"); + ggml_set_input(prompt_pos_t); - std::vector neck_det_0; - std::vector neck_det_1; - std::vector fenc_img_input_data; - std::vector fenc_pos_embed_data; - std::vector img_pe_72_data; - std::vector text_features_data; - if (!sam3_load_ref_f32_data(prephase_ref_dir + "/neck_det_0", neck_det_0, D * H0 * H0) || - !sam3_load_ref_f32_data(prephase_ref_dir + "/neck_det_1", neck_det_1, D * H1 * H1) || - !sam3_load_ref_f32_data(phase5_ref_dir + "/fenc_img_input", fenc_img_input_data, D * H * H) || - !sam3_load_ref_f32_data(phase5_ref_dir + "/fenc_pos_embed", fenc_pos_embed_data, D * H * H) || - !sam3_load_ref_f32_data(phase5_ref_dir + "/img_pe_72", img_pe_72_data, D * H * H) || - !sam3_load_ref_f32_data(phase5_ref_dir + "/text_features", text_features_data, D * L)) { - return false; + // RoPE frequencies + auto* rope_q_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, N); + ggml_set_name(rope_q_t, "rope_q"); + ggml_set_input(rope_q_t); + struct ggml_tensor* rope_k_t = nullptr; + if (pd.M_spatial > 0) { + rope_k_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, pd.M_spatial); + ggml_set_name(rope_k_t, "rope_k"); + ggml_set_input(rope_k_t); } - neck_det_0 = sam3_reorder_nchw_to_ggml_dwh(neck_det_0, D, H0, H0); - neck_det_1 = sam3_reorder_nchw_to_ggml_dwh(neck_det_1, D, H1, H1); + auto* conditioned = sam3_build_mem_attn_graph(ctx0, model, curr, src_pos_t, + prompt_t, prompt_pos_t, + rope_q_t, rope_k_t, + pd.num_obj_ptr_tokens); + auto* cond_spatial = ggml_reshape_4d(ctx0, conditioned, D, H, H, 1); - const size_t buf_size = ggml_tensor_overhead() * 65536 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; - } + // Bug 3 fix: single not_a_point_embed token instead of empty sparse + auto* sparse_in = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, 1, 1); + ggml_set_name(sparse_in, "prop_sparse"); + ggml_set_input(sparse_in); - auto* text_features = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, L, 1); - ggml_set_name(text_features, "text_features"); - ggml_set_input(text_features); - - auto* neck0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); - ggml_set_name(neck0, "ref_neck_det_0"); - ggml_set_input(neck0); - auto* neck1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); - ggml_set_name(neck1, "ref_neck_det_1"); - ggml_set_input(neck1); - auto* img_feats = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, H * H, 1); - ggml_set_name(img_feats, "ref_fenc_img_input"); - ggml_set_input(img_feats); - auto* img_pe = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, H * H, 1); - ggml_set_name(img_pe, "ref_fenc_pos_embed"); - ggml_set_input(img_pe); - auto* img_pe_72_cmp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, D * H * H); - ggml_set_name(img_pe_72_cmp, "ref_img_pe_72_flat"); - ggml_set_input(img_pe_72_cmp); - - auto* sine_dim_t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, 64); - ggml_set_name(sine_dim_t, "sine_dim_t"); - ggml_set_input(sine_dim_t); - - auto* rpb_coords = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, H); - ggml_set_name(rpb_coords, "rpb_coords"); - ggml_set_input(rpb_coords); - - auto* text_valid_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, L, 1, 1); - ggml_set_name(text_valid_mask, "text_valid_mask"); - ggml_set_input(text_valid_mask); - - auto* text_attn_bias = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, L, 1, 1); - ggml_set_name(text_attn_bias, "text_attn_bias"); - ggml_set_input(text_attn_bias); - - auto* conditioned = sam3_build_fenc_graph(ctx0, model, img_feats, text_features, - img_pe, text_attn_bias); - auto* fenc_layer5_out = ggml_cont(ctx0, conditioned); - ggml_set_name(fenc_layer5_out, "fenc_layer5_out"); - ggml_set_name(conditioned, "fenc_output"); - - auto ddec_out = sam3_build_ddec_graph(ctx0, model, conditioned, img_pe, text_features, - sine_dim_t, rpb_coords, text_attn_bias, text_valid_mask); - - struct ggml_tensor* fpn_feats[3] = {neck0, neck1, nullptr}; - - auto* obj_queries = ggml_view_3d(ctx0, ddec_out.queries, D, NQ, 1, - ddec_out.queries->nb[1], ddec_out.queries->nb[2], - 1 * ddec_out.queries->nb[1]); - obj_queries = ggml_cont(ctx0, obj_queries); - - auto* mask_logits = sam3_build_seg_head_graph(ctx0, model, conditioned, fpn_feats, - obj_queries, text_features, text_attn_bias); - ggml_set_name(mask_logits, "seg_mask_logits"); - - struct named_tensor { - const char* name; - struct ggml_tensor* tensor; - }; + auto* image_pe = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(image_pe, "prop_pe"); + ggml_set_input(image_pe); + auto* dense_emb = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(dense_emb, "prop_dense"); + ggml_set_input(dense_emb); - const std::vector direct_tensors = { - {"text_features", text_features}, - {"text_valid_mask", text_valid_mask}, - {"fenc_img_input", img_feats}, - {"fenc_pos_embed", img_pe}, - {"fenc_prompt", text_features}, - {"img_pe_72", img_pe_72_cmp}, - {"ddec_query_embed", model.ddec.query_embed}, - {"ddec_ref_pts_raw", model.tensors.at("ddec.reference_points.weight")}, - {"ddec_presence_token", model.ddec.presence_token}, - {"ddec_pred_boxes", ddec_out.pred_boxes}, - {"ddec_presence_logit", ddec_out.presence_score}, - }; + const int H0 = H * 4, H1 = H * 2; + auto* trk_s0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); + ggml_set_name(trk_s0, "prop_trk_s0"); + ggml_set_input(trk_s0); + auto* trk_s1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); + ggml_set_name(trk_s1, "prop_trk_s1"); + ggml_set_input(trk_s1); - std::vector named_outputs = { - "fenc_output", - "ddec_ref_boxes_init", - "ddec_query_sine_0", - "ddec_query_pos_0", - "ddec_rpb_mask_0", - "ddec_layer0_after_sa", - "ddec_layer0_after_text_ca", - "ddec_layer0_after_img_ca", - "ddec_layer0_full_out", - "ddec_layer0_presence", - "ddec_normed_output", - "scoring_prompt_mlp_out", - "scoring_pooled", - "scoring_proj_pooled", - "scoring_proj_hs", - "scoring_class_scores", - "seg_enc_after_ca", - "seg_enc_visual", - "seg_pixel_dec_stage0", - "seg_pixel_dec_stage1", - "seg_pixel_decoder_out", - "seg_instance_embed", - "seg_mask_embed", - "seg_mask_logits", - }; + auto dec = sam3_build_sam_dec_graph(ctx0, model, cond_spatial, image_pe, + sparse_in, dense_emb, + trk_s0, trk_s1, H); + ggml_set_output(dec.masks); + ggml_set_output(dec.iou_pred); + ggml_set_output(dec.obj_score); + ggml_set_output(dec.sam_token); + if (dec.mask_tokens) ggml_set_output(dec.mask_tokens); - for (int i = 0; i < hp.fenc_layers; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - named_outputs.emplace_back(name); - } - for (int i = 0; i < hp.ddec_layers; ++i) { - char out_name[64]; - char box_name[64]; - snprintf(out_name, sizeof(out_name), "ddec_layer%d_out", i); - snprintf(box_name, sizeof(box_name), "ddec_layer%d_refboxes", i); - named_outputs.emplace_back(out_name); - named_outputs.emplace_back(box_name); - } - - ggml_set_output(text_features); - ggml_set_output(text_valid_mask); - ggml_set_output(img_feats); - ggml_set_output(img_pe); - ggml_set_output(img_pe_72_cmp); - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_set_output(t); - } - } - ggml_set_output(mask_logits); - ggml_set_output(ddec_out.class_scores); - ggml_set_output(ddec_out.pred_boxes); - ggml_set_output(ddec_out.presence_score); - - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 65536, false); - ggml_build_forward_expand(graph, ddec_out.class_scores); - ggml_build_forward_expand(graph, ddec_out.pred_boxes); - ggml_build_forward_expand(graph, ddec_out.presence_score); - ggml_build_forward_expand(graph, mask_logits); - ggml_build_forward_expand(graph, text_features); - ggml_build_forward_expand(graph, text_valid_mask); - ggml_build_forward_expand(graph, img_feats); - ggml_build_forward_expand(graph, img_pe); - ggml_build_forward_expand(graph, img_pe_72_cmp); - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_build_forward_expand(graph, t); - } - } + auto* graph = ggml_new_graph_custom(ctx0, 32768, false); + ggml_build_forward_expand(graph, dec.masks); + ggml_build_forward_expand(graph, dec.iou_pred); + ggml_build_forward_expand(graph, dec.obj_score); + ggml_build_forward_expand(graph, dec.sam_token); + if (dec.mask_tokens) ggml_build_forward_expand(graph, dec.mask_tokens); auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate phase 5 graph\n", __func__); ggml_gallocr_free(galloc); ggml_free(ctx0); - return false; + return output; } - ggml_backend_tensor_set(text_features, text_features_data.data(), 0, text_features_data.size() * sizeof(float)); - ggml_backend_tensor_set(neck0, neck_det_0.data(), 0, neck_det_0.size() * sizeof(float)); - ggml_backend_tensor_set(neck1, neck_det_1.data(), 0, neck_det_1.size() * sizeof(float)); - ggml_backend_tensor_set(img_feats, fenc_img_input_data.data(), 0, fenc_img_input_data.size() * sizeof(float)); - ggml_backend_tensor_set(img_pe, fenc_pos_embed_data.data(), 0, fenc_pos_embed_data.size() * sizeof(float)); - ggml_backend_tensor_set(img_pe_72_cmp, img_pe_72_data.data(), 0, img_pe_72_data.size() * sizeof(float)); - - { - std::vector dim_t_data(64); - for (int i = 0; i < 64; ++i) { - dim_t_data[i] = 2.0f * (float)M_PI / powf(10000.0f, 2.0f * (float)i / 128.0f); - } - ggml_backend_tensor_set(sine_dim_t, dim_t_data.data(), 0, 64 * sizeof(float)); - } + // Upload prompt data + ggml_backend_tensor_set(prompt_t, pd.prompt.data(), 0, pd.prompt.size() * sizeof(float)); + ggml_backend_tensor_set(prompt_pos_t, pd.prompt_pos.data(), 0, pd.prompt_pos.size() * sizeof(float)); + ggml_backend_tensor_set(rope_q_t, rope_q_reord.data(), 0, rope_q_reord.size() * sizeof(float)); + if (rope_k_t && !rope_k_data.empty()) + ggml_backend_tensor_set(rope_k_t, rope_k_data.data(), 0, rope_k_data.size() * sizeof(float)); - { - std::vector coords(H); - for (int i = 0; i < H; ++i) { - coords[i] = (float)i / (float)H; - } - ggml_backend_tensor_set(rpb_coords, coords.data(), 0, H * sizeof(float)); + // Set default obj_score when pred_obj_scores=False (older SAM2 models) + if (!model.sam_dec.obj_score_token) { + auto* t = ggml_graph_get_tensor(graph, "sam_dec_obj_score"); + if (t) { float v = 10.0f; ggml_backend_tensor_set(t, &v, 0, sizeof(float)); } } - auto* qpos_pres = ggml_get_tensor(ctx0, "ddec_query_pos_pres"); - if (qpos_pres) { - std::vector zeros(D, 0.0f); - ggml_backend_tensor_set(qpos_pres, zeros.data(), 0, D * sizeof(float)); - } + // Upload src_pos (sinusoidal PE 256-dim) + ggml_backend_tensor_set(src_pos_t, tracker.cached_sinpe_256.data(), 0, + tracker.cached_sinpe_256.size() * sizeof(float)); - auto* rpb_pz = ggml_get_tensor(ctx0, "rpb_pres_zeros"); - if (rpb_pz) { - int n = (int)(rpb_pz->ne[0] * rpb_pz->ne[1] * rpb_pz->ne[2] * rpb_pz->ne[3]); - std::vector zeros(n, 0.0f); - ggml_backend_tensor_set(rpb_pz, zeros.data(), 0, n * sizeof(float)); - } + // Upload not_a_point_embed, image_pe, dense_emb from state PE cache + sam3_populate_pe_cache(state, model); + ggml_backend_tensor_set(sparse_in, state.not_a_point_cache, 0, D * sizeof(float)); + ggml_backend_tensor_set(image_pe, state.dense_pe_cache.data(), 0, + state.dense_pe_cache.size() * sizeof(float)); + ggml_backend_tensor_set(dense_emb, state.dense_nomask_cache.data(), 0, + state.dense_nomask_cache.size() * sizeof(float)); + // Copy tracker features from state to fresh input tensors { - int n_valid = 0; - for (int i = 0; i < L; ++i) { - if (token_ids[i] != 0) { - ++n_valid; - } - } - if (n_valid == 0) { - n_valid = 1; - } - - const float scale = (float)L / (float)n_valid; - std::vector valid_mask(L); - std::vector attn_bias(L); - for (int i = 0; i < L; ++i) { - const bool is_valid = token_ids[i] != 0; - valid_mask[i] = is_valid ? scale : 0.0f; - attn_bias[i] = is_valid ? 0.0f : -1.0e9f; - } - ggml_backend_tensor_set(text_valid_mask, valid_mask.data(), 0, L * sizeof(float)); - ggml_backend_tensor_set(text_attn_bias, attn_bias.data(), 0, L * sizeof(float)); - } - - sam3_graph_compute(model.backend, graph, n_threads); - - bool ok = true; - for (const auto& item : direct_tensors) { - if (!sam3_dump_tensor_to_path(item.tensor, item.name, output_dir + "/" + item.name)) { - ok = false; - } - } - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in graph\n", __func__, name.c_str()); - ok = false; - continue; - } - if (!sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; - } - } - - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; -} - -bool sam3_test_fenc_only(const sam3_model& model, - const std::string& ref_dir, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; // 256 - const int H = hp.n_img_embd(); // 72 - const int N = H * H; // 5184 - - fprintf(stderr, "%s: D=%d H=%d N=%d fenc_layers=%d\n", - __func__, D, H, N, hp.fenc_layers); - - // ── Load reference tensors from Python dump ── - // The Python script saves batch-first [1, N, D] tensors. - // Memory layout: N blocks of D floats = same as ggml [D, N, 1]. - std::vector img_feat_data; - std::vector pos_data; - std::vector prompt_data; - std::vector attn_bias_data; - - if (!sam3_load_ref_f32_data(ref_dir + "/fenc_input_tgt", img_feat_data, D * N)) { - fprintf(stderr, "%s: failed to load fenc_input_tgt\n", __func__); - return false; - } - if (!sam3_load_ref_f32_data(ref_dir + "/fenc_input_pos", pos_data, D * N)) { - fprintf(stderr, "%s: failed to load fenc_input_pos\n", __func__); - return false; - } - if (!sam3_load_ref_f32_data(ref_dir + "/fenc_input_prompt", prompt_data)) { - fprintf(stderr, "%s: failed to load fenc_input_prompt\n", __func__); - return false; - } + std::vector c2(D * N); + ggml_backend_tensor_get(state.neck_trk[2], c2.data(), 0, D * N * sizeof(float)); + ggml_backend_tensor_set(curr, c2.data(), 0, D * N * sizeof(float)); - // Determine prompt length from loaded data - const int T = (int)prompt_data.size() / D; - fprintf(stderr, "%s: prompt tokens T=%d\n", __func__, T); + std::vector s0(D * H0 * H0); + ggml_backend_tensor_get(state.neck_trk[0], s0.data(), 0, D * H0 * H0 * sizeof(float)); + ggml_backend_tensor_set(trk_s0, s0.data(), 0, D * H0 * H0 * sizeof(float)); - // Attn bias: [1, T] float (0.0 valid, -1e9 padding) - // If not present, assume all valid (no bias) - bool have_attn_bias = sam3_load_ref_f32_data(ref_dir + "/fenc_attn_bias", attn_bias_data, T); - if (!have_attn_bias) { - fprintf(stderr, "%s: no fenc_attn_bias found, assuming all tokens valid\n", __func__); - attn_bias_data.assign(T, 0.0f); + std::vector s1(D * H1 * H1); + ggml_backend_tensor_get(state.neck_trk[1], s1.data(), 0, D * H1 * H1 * sizeof(float)); + ggml_backend_tensor_set(trk_s1, s1.data(), 0, D * H1 * H1 * sizeof(float)); } - // ── Build fenc-only graph ── - const size_t buf_size = ggml_tensor_overhead() * 16384 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; + if (!sam3_graph_compute(model.backend, graph, 4)) { + ggml_gallocr_free(galloc); + ggml_free(ctx0); + return output; } - // Create input tensors (ggml layout: [D, N, 1]) - auto* img_feats = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); - ggml_set_name(img_feats, "fenc_img_input"); - ggml_set_input(img_feats); - - auto* pos_enc = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); - ggml_set_name(pos_enc, "fenc_pos_input"); - ggml_set_input(pos_enc); - - auto* prompt_tokens = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, T, 1); - ggml_set_name(prompt_tokens, "fenc_prompt_input"); - ggml_set_input(prompt_tokens); - - auto* text_attn_bias = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, T, 1, 1); - ggml_set_name(text_attn_bias, "fenc_attn_bias"); - ggml_set_input(text_attn_bias); - - // Build fusion encoder graph - auto* conditioned = sam3_build_fenc_graph(ctx0, model, img_feats, prompt_tokens, - pos_enc, text_attn_bias); - // sam3_build_fenc_graph names the last layer fenc_layer5_out. ggml_set_name - // will overwrite it, so create a cont copy so both names resolve. - auto* fenc_last = ggml_cont(ctx0, conditioned); - ggml_set_name(fenc_last, "fenc_layer5_out"); - ggml_set_name(conditioned, "fenc_output"); + const int mhw = H * 4; + const int num_mask_tokens = hp.sam_n_multimask + 1; // 4 - // Mark all per-layer outputs as graph outputs for extraction - std::vector output_names = {"fenc_output"}; - for (int i = 0; i < hp.fenc_layers; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - output_names.emplace_back(name); - } + // SAM2/EdgeTAM with multimask_output_in_sam: read all masks, select best by IoU. + // SAM3 / SAM2 without multimask: use first mask only. + bool use_multimask = (hp.is_sam2() || hp.is_edgetam()) && hp.multimask_output_in_sam; - ggml_set_output(img_feats); - ggml_set_output(pos_enc); - ggml_set_output(prompt_tokens); - for (const auto& name : output_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_set_output(t); - } - } + if (use_multimask) { + // Read all 4 IoU predictions + std::vector all_ious(num_mask_tokens); + ggml_backend_tensor_get(dec.iou_pred, all_ious.data(), 0, num_mask_tokens * sizeof(float)); - // Build and allocate graph - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 16384, false); - ggml_build_forward_expand(graph, conditioned); - ggml_build_forward_expand(graph, fenc_last); - for (const auto& name : output_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) { - ggml_build_forward_expand(graph, t); + // Multimask uses mask tokens 1-3 (skip token 0 which is the single-mask output) + int best_idx = 1; + float best_iou = all_ious[1]; + for (int m = 2; m < num_mask_tokens; ++m) { + if (all_ious[m] > best_iou) { best_iou = all_ious[m]; best_idx = m; } } - } - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate fenc graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; - } + output.n_masks = 1; + output.mask_h = mhw; + output.mask_w = mhw; + output.mask_logits.resize(mhw * mhw); + // Read best mask (offset by best_idx * mhw * mhw) + ggml_backend_tensor_get(dec.masks, output.mask_logits.data(), + best_idx * mhw * mhw * sizeof(float), mhw * mhw * sizeof(float)); + output.iou_scores.resize(1); + output.iou_scores[0] = best_iou; + ggml_backend_tensor_get(dec.obj_score, &output.obj_score, 0, sizeof(float)); - // ── Set input data ── - ggml_backend_tensor_set(img_feats, img_feat_data.data(), 0, - img_feat_data.size() * sizeof(float)); - ggml_backend_tensor_set(pos_enc, pos_data.data(), 0, - pos_data.size() * sizeof(float)); - ggml_backend_tensor_set(prompt_tokens, prompt_data.data(), 0, - prompt_data.size() * sizeof(float)); - ggml_backend_tensor_set(text_attn_bias, attn_bias_data.data(), 0, - attn_bias_data.size() * sizeof(float)); - - // ── Compute ── - fprintf(stderr, "%s: computing fenc graph (%d nodes)...\n", __func__, ggml_graph_n_nodes(graph)); - sam3_graph_compute(model.backend, graph, n_threads); - - // ── Dump outputs ── - bool ok = true; - for (const auto& name : output_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found\n", __func__, name.c_str()); - ok = false; - continue; - } - if (!sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; + // Object pointer token: use best multimask token if use_multimask_token_for_obj_ptr + output.sam_token.resize(D); + if (hp.use_multimask_token_for_obj_ptr && dec.mask_tokens) { + ggml_backend_tensor_get(dec.mask_tokens, output.sam_token.data(), + best_idx * D * sizeof(float), D * sizeof(float)); + } else { + ggml_backend_tensor_get(dec.sam_token, output.sam_token.data(), 0, D * sizeof(float)); } + } else { + output.n_masks = 1; + output.mask_h = mhw; + output.mask_w = mhw; + output.mask_logits.resize(mhw * mhw); + ggml_backend_tensor_get(dec.masks, output.mask_logits.data(), 0, mhw * mhw * sizeof(float)); + output.iou_scores.resize(1); + ggml_backend_tensor_get(dec.iou_pred, output.iou_scores.data(), 0, sizeof(float)); + ggml_backend_tensor_get(dec.obj_score, &output.obj_score, 0, sizeof(float)); + output.sam_token.resize(D); + ggml_backend_tensor_get(dec.sam_token, output.sam_token.data(), 0, D * sizeof(float)); } - // Also dump inputs for verification - sam3_dump_tensor_to_path(img_feats, "fenc_img_input", output_dir + "/fenc_img_input"); - sam3_dump_tensor_to_path(pos_enc, "fenc_pos_input", output_dir + "/fenc_pos_input"); - sam3_dump_tensor_to_path(prompt_tokens, "fenc_prompt_input", output_dir + "/fenc_prompt_input"); - ggml_gallocr_free(galloc); ggml_free(ctx0); - return ok; -} - -static bool sam3_test_dump_phase6_impl(const sam3_model& model, - struct ggml_tensor* state_neck0, - struct ggml_tensor* state_neck1, - struct ggml_tensor* state_neck2, - const std::vector* neck0_data, - const std::vector* neck1_data, - const std::vector* neck2_data, - const sam3_pvs_params& params, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.sam_embed_dim; - const int H = hp.n_img_embd(); - const int H1 = H * 2; - const int H0 = H * 4; + return output; +} - const size_t buf_size = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init compute context\n", __func__); - return false; +static std::vector> sam3_match_detections( + const std::vector& masklets, const std::vector& dets, + const std::vector& prop_masks, float iou_threshold) { + std::vector> matches; + if (masklets.empty() || dets.empty()) return matches; + int n_m = (int)masklets.size(), n_d = (int)dets.size(); + std::vector dm(n_d, false); + for (int i = 0; i < n_m; ++i) { + if (i >= (int)prop_masks.size() || prop_masks[i].data.empty()) continue; + int bj = -1; + float bi = iou_threshold; + for (int j = 0; j < n_d; ++j) { + if (dm[j] || dets[j].mask.data.empty()) continue; + int w = prop_masks[i].width, h = prop_masks[i].height; + if (w != dets[j].mask.width || h != dets[j].mask.height) continue; + float iou = sam3_mask_iou(prop_masks[i].data.data(), dets[j].mask.data.data(), w * h); + if (iou > bi) { + bi = iou; + bj = j; + } + } + if (bj >= 0) { + matches.push_back({i, bj}); + dm[bj] = true; + } } + return matches; +} - struct ggml_tensor* neck0 = state_neck0; - struct ggml_tensor* neck1 = state_neck1; - struct ggml_tensor* neck2 = state_neck2; - if (neck0_data && neck1_data && neck2_data) { - neck0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); - ggml_set_name(neck0, "sam_dec_feat_s0_input"); - ggml_set_input(neck0); - - neck1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); - ggml_set_name(neck1, "sam_dec_feat_s1_input"); - ggml_set_input(neck1); - - neck2 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(neck2, "sam_dec_feat_s2_input"); - ggml_set_input(neck2); +static void sam3_update_tracker(sam3_tracker& tracker, int frame_idx) { + for (auto it = tracker.pending.begin(); it != tracker.pending.end();) { + int age = frame_idx - it->first_frame; + if (age >= tracker.params.hotstart_delay && it->mds_sum > 0) { + it->confirmed = true; + tracker.masklets.push_back(std::move(*it)); + it = tracker.pending.erase(it); + } else if (age >= tracker.params.hotstart_delay) { + it = tracker.pending.erase(it); + } else + ++it; } - - if (!neck0 || !neck1 || !neck2) { - fprintf(stderr, "%s: missing tracker feature inputs\n", __func__); - ggml_free(ctx0); - return false; + for (auto it = tracker.masklets.begin(); it != tracker.masklets.end();) { + if (frame_idx - it->last_seen > tracker.params.max_keep_alive) { + tracker.mem_banks.erase(it->instance_id); + tracker.ptr_banks.erase(it->instance_id); + it = tracker.masklets.erase(it); + } else + ++it; } +} - auto pe_out = sam3_build_sam_pe(ctx0, params, D, H); - - auto* no_mem = ggml_reshape_4d(ctx0, model.tensors.at("no_mem_embed"), D, 1, 1, 1); - auto* image_feats = ggml_add(ctx0, neck2, no_mem); - ggml_set_name(image_feats, "sam_dec_image_feats"); - - auto dec_out = sam3_build_sam_dec_graph(ctx0, model, image_feats, - pe_out.image_pe, pe_out.sparse, pe_out.dense, - neck0, neck1); +static bool sam3_encode_memory( + sam3_tracker& tracker, sam3_state& state, const sam3_model& model, + int inst_id, const float* mask_logits, int mask_h, int mask_w, + int frame_idx, bool is_cond, float obj_score) { + const auto& hp = model.hparams; + const int D = hp.neck_dim, MD = hp.mem_out_dim; + const int H = sam3_eff_feat_size(state, hp); + const int HIGH_RES = sam3_eff_img_size(state, hp); + const int INTERPOL = H * 16; - struct named_tensor { - const char* name; - struct ggml_tensor* tensor; - }; + // Mask preprocessing: mask_logits → HIGH_RES → sigmoid → scale/bias → INTERPOL + auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES); + const float sig_scale = hp.sigmoid_scale(), sig_bias = hp.sigmoid_bias(); + for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_scale + sig_bias; } + auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL); - const std::vector direct_tensors = { - {"sam_pe_sparse", pe_out.sparse}, - {"sam_pe_dense", pe_out.dense}, - {"sam_pe_image_pe", pe_out.image_pe}, - {"sam_dec_image_feats", image_feats}, - {"sam_dec_sam_token", dec_out.sam_token}, - }; + const size_t bs = ggml_tensor_overhead() * 16384 + ggml_graph_overhead(); + struct ggml_init_params gp = {bs, nullptr, true}; + auto* ctx0 = ggml_init(gp); + if (!ctx0) return false; - std::vector named_outputs = { - "sam_dec_tokens_initial", - "sam_dec_block0_queries", - "sam_dec_block0_keys", - "sam_dec_block1_queries", - "sam_dec_block1_keys", - "sam_dec_final_queries", - "sam_dec_feat_s1_proj", - "sam_dec_feat_s0_proj", - "sam_dec_upscaled", - "sam_dec_mask_tokens", - "sam_dec_masks", - "sam_dec_iou", - "sam_dec_obj_score", - }; + auto* mask_in = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, INTERPOL, INTERPOL, 1, 1); + ggml_set_name(mask_in, "mem_mask"); + ggml_set_input(mask_in); - ggml_set_output(dec_out.masks); - ggml_set_output(dec_out.iou_pred); - ggml_set_output(dec_out.obj_score); - ggml_set_output(dec_out.sam_token); - ggml_set_output(dec_out.mask_tokens); - for (const auto& item : direct_tensors) { - ggml_set_output(item.tensor); - } - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in graph\n", __func__, name.c_str()); - ggml_free(ctx0); - return false; - } - ggml_set_output(t); + // Learned mask downsampler: 4 stages conv+LN+GELU + final 1×1 + // Conv output is in ggml "conv" layout. permute(1,2,0,3) converts to "internal" layout + // where ne[0]=channel for LN2d. permute(2,0,1,3) converts back to "conv" layout. + auto* ds = mask_in; + for (int s = 0; s < 4; ++s) { + int out_ch = (int)model.mem_enc.ds_conv_w[s]->ne[3]; + ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[s], ds, 2, 2, 1, 1, 1, 1); + ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[s], 1, 1, out_ch, 1)); + ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal (ne[0]=channel) + ds = sam3_layer_norm_2d(ctx0, ds, model.mem_enc.ds_norm_w[s], model.mem_enc.ds_norm_b[s]); + ds = ggml_gelu(ctx0, ds); + ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 2, 0, 1, 3)); // internal→conv } + ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[4], ds, 1, 1, 0, 0, 1, 1); + ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[4], 1, 1, D, 1)); + ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal [D, ...] - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 32768, false); - ggml_build_forward_expand(graph, dec_out.masks); - ggml_build_forward_expand(graph, dec_out.iou_pred); - ggml_build_forward_expand(graph, dec_out.obj_score); - ggml_build_forward_expand(graph, dec_out.sam_token); - ggml_build_forward_expand(graph, dec_out.mask_tokens); - for (const auto& item : direct_tensors) { - ggml_build_forward_expand(graph, item.tensor); - } - for (const auto& name : named_outputs) { - ggml_build_forward_expand(graph, ggml_get_tensor(ctx0, name.c_str())); - } + // Pixel projection — use fresh input tensor to avoid pulling in the + // entire ViT+neck recomputation from state.neck_trk[2]'s dependency tree + auto* pix_in_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); + ggml_set_name(pix_in_raw, "mem_pix_feat"); + ggml_set_input(pix_in_raw); + auto* pix_in = ggml_cont(ctx0, ggml_permute(ctx0, pix_in_raw, 2, 0, 1, 3)); + auto* pix = ggml_conv_2d(ctx0, model.mem_enc.pix_proj_w, pix_in, 1, 1, 0, 0, 1, 1); + pix = ggml_add(ctx0, pix, ggml_reshape_4d(ctx0, model.mem_enc.pix_proj_b, 1, 1, D, 1)); + pix = ggml_cont(ctx0, ggml_permute(ctx0, pix, 1, 2, 0, 3)); - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate phase 6 graph\n", __func__); - ggml_gallocr_free(galloc); + // Fusion: ADD (not multiply) + auto* fused = ggml_add(ctx0, pix, ds); + for (int i = 0; i < 2; ++i) + fused = sam3_cxblock_forward(ctx0, fused, + model.mem_enc.fuser_dw_w[i], model.mem_enc.fuser_dw_b[i], + model.mem_enc.fuser_norm_w[i], model.mem_enc.fuser_norm_b[i], + model.mem_enc.fuser_fc1_w[i], model.mem_enc.fuser_fc1_b[i], + model.mem_enc.fuser_fc2_w[i], model.mem_enc.fuser_fc2_b[i], + model.mem_enc.fuser_gamma[i]); + auto* fused_out = ggml_cont(ctx0, ggml_permute(ctx0, fused, 2, 0, 1, 3)); + auto* mo = ggml_conv_2d(ctx0, model.mem_enc.out_proj_w, fused_out, 1, 1, 0, 0, 1, 1); + mo = ggml_add(ctx0, mo, ggml_reshape_4d(ctx0, model.mem_enc.out_proj_b, 1, 1, MD, 1)); + mo = ggml_cont(ctx0, ggml_permute(ctx0, mo, 1, 2, 0, 3)); + ggml_set_name(mo, "mem_out"); + ggml_set_output(mo); + + auto* g = ggml_new_graph_custom(ctx0, 16384, false); + ggml_build_forward_expand(g, mo); + auto* ga = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); + if (!ggml_gallocr_reserve(ga, g) || !ggml_gallocr_alloc_graph(ga, g)) { + ggml_gallocr_free(ga); ggml_free(ctx0); return false; } - - if (neck0_data && neck1_data && neck2_data) { - ggml_backend_tensor_set(neck0, neck0_data->data(), 0, neck0_data->size() * sizeof(float)); - ggml_backend_tensor_set(neck1, neck1_data->data(), 0, neck1_data->size() * sizeof(float)); - ggml_backend_tensor_set(neck2, neck2_data->data(), 0, neck2_data->size() * sizeof(float)); + ggml_backend_tensor_set(mask_in, m_interp.data(), 0, m_interp.size() * sizeof(float)); + // Copy pixel features from state tensor to fresh input + { + std::vector pix_data(D * H * H); + ggml_backend_tensor_get(state.neck_trk[2], pix_data.data(), 0, D * H * H * sizeof(float)); + ggml_backend_tensor_set(pix_in_raw, pix_data.data(), 0, D * H * H * sizeof(float)); } - - sam3_state pe_cache_state = {}; - sam3_populate_pe_cache(pe_cache_state, model); - - std::vector all_coords; - std::vector all_labels; - sam3_collect_pvs_prompt_tokens(params, all_coords, all_labels); - if ((int)all_labels.size() != pe_out.n_tokens) { - fprintf(stderr, "%s: prompt token count mismatch (%zu vs %d)\n", - __func__, all_labels.size(), pe_out.n_tokens); - ggml_gallocr_free(galloc); + if (!sam3_graph_compute(model.backend, g, 4)) { + ggml_gallocr_free(ga); ggml_free(ctx0); return false; } - { - const int num_pos_feats = D / 2; - std::vector sparse_data((size_t)pe_out.n_tokens * D, 0.0f); - for (int p = 0; p < pe_out.n_tokens; ++p) { - float px = all_coords[p * 2 + 0] + 0.5f; - float py = all_coords[p * 2 + 1] + 0.5f; - float x_norm = px / (float)hp.img_size; - float y_norm = py / (float)hp.img_size; - float pe_vec[256]; - sam3_pe_encode_coord(pe_vec, x_norm, y_norm, - pe_cache_state.pe_gauss_cache.data(), num_pos_feats); + std::vector md(MD * H * H); + ggml_backend_tensor_get(mo, md.data(), 0, md.size() * sizeof(float)); - const int label = all_labels[p]; - if (label == -1) { - for (int d = 0; d < D; ++d) { - sparse_data[(size_t)p * D + d] = pe_cache_state.not_a_point_cache[d]; - } - } else { - for (int d = 0; d < D; ++d) { - sparse_data[(size_t)p * D + d] = pe_vec[d] + pe_cache_state.point_emb_cache[label][d]; - } - } + // Apply no_obj_embed_spatial if occluded (SAM2.1 only — EdgeTAM does not have this) + if (obj_score <= 0.0f && model.no_obj_embed_spatial) { + std::vector no_obj_emb(MD); + auto* noe = model.no_obj_embed_spatial; + if (noe->type == GGML_TYPE_F16) { + std::vector tmp(MD); + ggml_backend_tensor_get(noe, tmp.data(), 0, MD * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), no_obj_emb.data(), MD); + } else { + ggml_backend_tensor_get(noe, no_obj_emb.data(), 0, MD * sizeof(float)); } + for (int i = 0; i < MD * H * H; ++i) + md[i] += no_obj_emb[i % MD]; + } - ggml_backend_tensor_set(pe_out.sparse, sparse_data.data(), 0, - sparse_data.size() * sizeof(float)); - ggml_backend_tensor_set(pe_out.image_pe, pe_cache_state.dense_pe_cache.data(), 0, - pe_cache_state.dense_pe_cache.size() * sizeof(float)); - ggml_backend_tensor_set(pe_out.dense, pe_cache_state.dense_nomask_cache.data(), 0, - pe_cache_state.dense_nomask_cache.size() * sizeof(float)); + if (!tracker.ctx) { + struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; + tracker.ctx = ggml_init(tp); } - sam3_graph_compute(model.backend, graph, n_threads); + // ── EdgeTAM perceiver: compress memory features before storage ─────── + if (hp.has_perceiver) { + // Compute sinusoidal PE for memory features (needed as perceiver input) + sam3_ensure_tracker_pe_caches(tracker, hp, H); + const auto& mem_pos = tracker.cached_sinpe_64; - bool ok = true; - for (const auto& item : direct_tensors) { - if (!sam3_dump_tensor_to_path(item.tensor, item.name, output_dir + "/" + item.name)) { - ok = false; - } - } - for (const auto& name : named_outputs) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found after compute\n", __func__, name.c_str()); - ok = false; - continue; - } - if (!sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; + std::vector perc_latents, perc_pos; + if (!edgetam_perceiver_forward(model, md, mem_pos, H, H, + perc_latents, perc_pos)) { + fprintf(stderr, "%s: perceiver forward failed\n", __func__); + ggml_gallocr_free(ga); + ggml_free(ctx0); + return false; } - } - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; -} + // Store perceiver output: [MD=64, N_perc] (N_1d + N_2d latents) + const int N_perc = hp.perceiver_n_latents_1d + hp.perceiver_n_latents_2d; + auto* st = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); + auto* sb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); + struct ggml_tallocr ta_perc = ggml_tallocr_new(sb); + ggml_tallocr_alloc(&ta_perc, st); + tracker.owned_buffers.push_back(sb); + ggml_backend_tensor_set(st, perc_latents.data(), 0, MD * N_perc * sizeof(float)); -bool sam3_test_dump_phase6(const sam3_model& model, - const sam3_state& state, - const sam3_pvs_params& params, - const std::string& output_dir, - int n_threads) { - return sam3_test_dump_phase6_impl(model, - state.neck_trk[0], state.neck_trk[1], state.neck_trk[2], - nullptr, nullptr, nullptr, - params, output_dir, n_threads); -} + // Store perceiver PE: [MD=64, 512] (first 256 zeros, last 256 sinusoidal) + auto* spe = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, MD, N_perc); + auto* speb = ggml_backend_alloc_buffer(model.backend, MD * N_perc * sizeof(float)); + struct ggml_tallocr ta_perc2 = ggml_tallocr_new(speb); + ggml_tallocr_alloc(&ta_perc2, spe); + tracker.owned_buffers.push_back(speb); + ggml_backend_tensor_set(spe, perc_pos.data(), 0, MD * N_perc * sizeof(float)); -bool sam3_test_dump_phase6_from_ref_inputs(const sam3_model& model, - const std::string& prephase_ref_dir, - const sam3_pvs_params& params, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.sam_embed_dim; - const int H = hp.n_img_embd(); - const int H1 = H * 2; - const int H0 = H * 4; - - std::vector neck_trk_0; - std::vector neck_trk_1; - std::vector neck_trk_2; - if (!sam3_load_ref_f32_data(prephase_ref_dir + "/neck_trk_0", neck_trk_0, D * H0 * H0) || - !sam3_load_ref_f32_data(prephase_ref_dir + "/neck_trk_1", neck_trk_1, D * H1 * H1) || - !sam3_load_ref_f32_data(prephase_ref_dir + "/neck_trk_2", neck_trk_2, D * H * H)) { - return false; + sam3_memory_slot slot; + slot.spatial_feats = st; + slot.spatial_pe = spe; + slot.frame_index = frame_idx; + slot.is_cond_frame = is_cond; + auto& bk = tracker.mem_banks[inst_id]; + bk.push_back(slot); + while ((int)bk.size() > hp.num_maskmem) { + bool removed = false; + for (auto it = bk.begin(); it != bk.end(); ++it) + if (!it->is_cond_frame) { + bk.erase(it); + removed = true; + break; + } + if (!removed) bk.erase(bk.begin() + 1); + } + ggml_gallocr_free(ga); + ggml_free(ctx0); + return true; } - neck_trk_0 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_0, D, H0, H0); - neck_trk_1 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_1, D, H1, H1); - neck_trk_2 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_2, D, H, H); - - return sam3_test_dump_phase6_impl(model, - nullptr, nullptr, nullptr, - &neck_trk_0, &neck_trk_1, &neck_trk_2, - params, output_dir, n_threads); -} - -/***************************************************************************** -** Test: Geometry encoder dump -*****************************************************************************/ - -bool sam3_test_dump_geom_enc(const sam3_model& model, - const std::string& prephase_ref_dir, - const sam3_pcs_params& params, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; // 256 - const int H = hp.n_img_embd(); // 72 + // ── Standard path (SAM2/SAM3): store raw [MD, H, H] features ──────── + // Store spatial features + auto* st = ggml_new_tensor_4d(tracker.ctx, GGML_TYPE_F32, MD, H, H, 1); + auto* sb = ggml_backend_alloc_buffer(model.backend, MD * H * H * sizeof(float)); + struct ggml_tallocr ta = ggml_tallocr_new(sb); + ggml_tallocr_alloc(&ta, st); + tracker.owned_buffers.push_back(sb); + ggml_backend_tensor_set(st, md.data(), 0, md.size() * sizeof(float)); - // Load backbone features from Phase 3 reference (NCHW format) - std::vector neck_det_2; - if (!sam3_load_ref_f32_data(prephase_ref_dir + "/neck_det_2", neck_det_2, D * H * H)) { - fprintf(stderr, "%s: failed to load neck_det_2\n", __func__); - return false; - } - // Reorder from NCHW to ggml [D, W, H] layout - auto neck_det_2_ggml = sam3_reorder_nchw_to_ggml_dwh(neck_det_2, D, H, H); + // Compute and store sinusoidal spatial PE + sam3_ensure_tracker_pe_caches(tracker, hp, H); + const auto& pe_data = tracker.cached_sinpe_64; + auto* spe = ggml_new_tensor_4d(tracker.ctx, GGML_TYPE_F32, MD, H, H, 1); + auto* speb = ggml_backend_alloc_buffer(model.backend, MD * H * H * sizeof(float)); + struct ggml_tallocr ta2 = ggml_tallocr_new(speb); + ggml_tallocr_alloc(&ta2, spe); + tracker.owned_buffers.push_back(speb); + ggml_backend_tensor_set(spe, pe_data.data(), 0, pe_data.size() * sizeof(float)); - // Compute sinusoidal PE for image features (matching Python PositionEmbeddingSine) - std::vector img_pe_nchw(D * H * H); - { - const float scale = 2.0f * (float)M_PI; - const float eps = 1e-6f; - const int num_pos_feats = D / 2; // 128 - - for (int row = 0; row < H; ++row) { - for (int col = 0; col < H; ++col) { - float y_embed = ((float)(row + 1) - 0.5f) / ((float)H + eps) * scale; - float x_embed = ((float)(col + 1) - 0.5f) / ((float)H + eps) * scale; - - for (int i = 0; i < num_pos_feats; ++i) { - int div_idx = 2 * (i / 2); - float dim_t = powf(10000.0f, (float)div_idx / (float)num_pos_feats); - - float px = x_embed / dim_t; - float py = y_embed / dim_t; - - float pe_y, pe_x; - if (i % 2 == 0) { - pe_y = sinf(py); - pe_x = sinf(px); - } else { - pe_y = cosf(py); - pe_x = cosf(px); - } - // Output: [1, D, H, W] NCHW — pos_y first half, pos_x second half - img_pe_nchw[i * H * H + row * H + col] = pe_y; - img_pe_nchw[(num_pos_feats + i) * H * H + row * H + col] = pe_x; - } + sam3_memory_slot slot; + slot.spatial_feats = st; + slot.spatial_pe = spe; + slot.frame_index = frame_idx; + slot.is_cond_frame = is_cond; + auto& bk = tracker.mem_banks[inst_id]; + bk.push_back(slot); + while ((int)bk.size() > hp.num_maskmem) { + bool removed = false; + for (auto it = bk.begin(); it != bk.end(); ++it) + if (!it->is_cond_frame) { + bk.erase(it); + removed = true; + break; } - } - } - auto img_pe_ggml = sam3_reorder_nchw_to_ggml_dwh(img_pe_nchw, D, H, H); - - // ── Build graph ───────────────────────────────────────────────────── - const size_t buf_size = ggml_tensor_overhead() * 8192 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = { - /*.mem_size =*/buf_size, - /*.mem_buffer =*/nullptr, - /*.no_alloc =*/true, - }; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init context\n", __func__); - return false; + if (!removed) bk.erase(bk.begin() + 1); } + ggml_gallocr_free(ga); + ggml_free(ctx0); + return true; +} - // Image features as input tensors (from reference data) - auto* img_feats_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, H * H, 1); - ggml_set_name(img_feats_t, "img_feats_input"); - ggml_set_input(img_feats_t); - - auto* img_pe_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, H * H, 1); - ggml_set_name(img_pe_t, "img_pe_input"); - ggml_set_input(img_pe_t); - - // Build geometry encoder graph - auto geom_out = sam3_build_geom_enc_graph(ctx0, model, params, img_feats_t, img_pe_t); - - // Build and allocate graph - struct ggml_cgraph* graph = ggml_new_graph_custom(ctx0, 16384, false); - ggml_build_forward_expand(graph, geom_out.geo_feats); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; +static void sam3_store_obj_ptr( + sam3_tracker& tracker, const sam3_model& model, + int inst_id, const float* pd, int frame_idx) { + const int D = model.hparams.neck_dim; + if (!tracker.ctx) { + struct ggml_init_params tp = {ggml_tensor_overhead() * 4096, nullptr, true}; + tracker.ctx = ggml_init(tp); } + auto* pt = ggml_new_tensor_2d(tracker.ctx, GGML_TYPE_F32, D, 1); + auto* pb = ggml_backend_alloc_buffer(model.backend, D * sizeof(float)); + struct ggml_tallocr ta = ggml_tallocr_new(pb); + ggml_tallocr_alloc(&ta, pt); + tracker.owned_buffers.push_back(pb); + ggml_backend_tensor_set(pt, pd, 0, D * sizeof(float)); + auto& bk = tracker.ptr_banks[inst_id]; + bk.push_back({frame_idx, pt}); + while ((int)bk.size() > model.hparams.max_obj_ptrs) bk.erase(bk.begin()); +} - // Upload image features - ggml_backend_tensor_set(img_feats_t, neck_det_2_ggml.data(), 0, D * H * H * sizeof(float)); - ggml_backend_tensor_set(img_pe_t, img_pe_ggml.data(), 0, D * H * H * sizeof(float)); - - // Pre-compute geometry input on CPU and upload - auto geom_data = sam3_precompute_geom_input(model, params, neck_det_2_ggml.data(), H, H); - auto* gi = ggml_get_tensor(ctx0, "geom_post_final_proj"); - if (gi) { - ggml_backend_tensor_set(gi, geom_data.data(), 0, geom_data.size() * sizeof(float)); +sam3_tracker_ptr sam3_create_tracker(const sam3_model& model, + const sam3_video_params& params) { + if (model.hparams.is_sam2()) { + fprintf(stderr, "%s: ERROR: text-prompted tracker not available for SAM2 " + "(use sam3_create_visual_tracker instead)\n", __func__); + return nullptr; } + sam3_tracker_ptr tracker(new sam3_tracker()); + tracker->params = params; + fprintf(stderr, "%s: tracker created (hotstart=%d, max_keep_alive=%d)\n", + __func__, params.hotstart_delay, params.max_keep_alive); + return tracker; +} - // Compute - sam3_graph_compute(model.backend, graph, n_threads); - - // Dump outputs - mkdir(output_dir.c_str(), 0755); - - // Dump the pre-computed input (after final_proj + norm, before transformer) - { - auto* pre_proj = ggml_get_tensor(ctx0, "geom_post_final_proj"); - if (pre_proj) sam3_dump_tensor_to_path(pre_proj, "post_final_proj", output_dir + "/post_final_proj"); +sam3_result sam3_track_frame(sam3_tracker& tracker, sam3_state& state, + const sam3_model& model, const sam3_image& frame) { + if (model.hparams.visual_only) { + fprintf(stderr, "%s: ERROR: track_frame not available on visual-only model " + "(use sam3_propagate_frame instead)\n", __func__); + return sam3_result{}; } - // Dump final output - { - auto* out = ggml_get_tensor(ctx0, "geom_output"); - if (out) sam3_dump_tensor_to_path(out, "geom_output", output_dir + "/geom_output"); - } + sam3_result result; + const int D = model.hparams.neck_dim; + if (!sam3_encode_image(state, model, frame)) return result; + int fi = tracker.frame_index; + fprintf(stderr, "%s: frame %d (%zu active + %zu pending)\n", + __func__, fi, tracker.masklets.size(), tracker.pending.size()); - // Also dump the pre-computed input data (before ggml processing) - { - const int N_geo = geom_out.n_tokens; - std::string path = output_dir + "/geom_input_precomputed"; - { - std::ofstream f(path + ".bin", std::ios::binary); - f.write(reinterpret_cast(geom_data.data()), geom_data.size() * sizeof(float)); + std::map pm; + std::map po; + for (auto& ml : tracker.masklets) { + int id = ml.instance_id; + auto im = tracker.mem_banks.find(id); + if (im == tracker.mem_banks.end() || im->second.empty()) continue; + po[id] = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); + if (po[id].mask_logits.empty()) continue; + auto rs = sam3_bilinear_interpolate(po[id].mask_logits.data(), + po[id].mask_w, po[id].mask_h, state.orig_width, state.orig_height); + pm[id].width = state.orig_width; + pm[id].height = state.orig_height; + pm[id].data.resize(state.orig_width * state.orig_height); + int fg = 0; + for (int p = 0; p < (int)rs.size(); ++p) { + bool f = rs[p] > 0.0f; + pm[id].data[p] = f ? 255 : 0; + if (f) fg++; } - { - std::ofstream f(path + ".shape"); - f << D << "," << N_geo; + ml.last_score = po[id].iou_scores[0]; + ml.last_seen = fi; + float cov = (float)fg / (state.orig_width * state.orig_height); + ml.mds_sum += (cov > 0.001f && po[id].obj_score > 0.0f) ? 1 : -1; + } + for (auto& ml : tracker.pending) { + int id = ml.instance_id; + auto im = tracker.mem_banks.find(id); + if (im == tracker.mem_banks.end() || im->second.empty()) continue; + auto p2 = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); + if (!p2.mask_logits.empty()) { + ml.last_score = p2.iou_scores[0]; + ml.last_seen = fi; + auto r2 = sam3_bilinear_interpolate(p2.mask_logits.data(), + p2.mask_w, p2.mask_h, state.orig_width, state.orig_height); + int fg2 = 0; + for (auto v : r2) + if (v > 0.0f) fg2++; + float c2 = (float)fg2 / (state.orig_width * state.orig_height); + ml.mds_sum += (c2 > 0.001f && p2.obj_score > 0.0f) ? 1 : -1; + sam3_encode_memory(tracker, state, model, id, + p2.mask_logits.data(), p2.mask_h, p2.mask_w, fi, false, p2.obj_score); + std::vector op(D); + sam3_extract_obj_ptr_cpu(model, p2.sam_token.data(), p2.obj_score, op.data()); + sam3_store_obj_ptr(tracker, model, id, op.data(), fi); } - fprintf(stderr, "%s: dumped geom_input_precomputed [%d, %d]\n", __func__, D, N_geo); } - - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return true; -} - -static bool sam3_test_dump_phase7_mem_slot_current( - const sam3_model& model, - const std::vector& neck2_data, - const std::vector& low_res_mask_logits, - float obj_score, - int slot_idx, - const std::string& output_dir, - std::vector& mem_out_data, - std::vector& mem_pe_data, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; - const int MD = hp.mem_out_dim; - const int H = hp.feat_size(); - const int MASK_HW = H * 4; - const int HIGH_RES = hp.img_size; - const int INTERPOL = H * 16; - - // Mask preprocessing: MASK_HW → HIGH_RES → sigmoid+scale+bias → INTERPOL - auto m_hires = sam3_bilinear_interpolate(low_res_mask_logits.data(), MASK_HW, MASK_HW, HIGH_RES, HIGH_RES); - const float sig_s = hp.sigmoid_scale(), sig_b = hp.sigmoid_bias(); - for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_s + sig_b; } - auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL); - - const size_t buf_size = ggml_tensor_overhead() * 16384 + ggml_graph_overhead(); - struct ggml_init_params gparams = {buf_size, nullptr, true}; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init memory encoder context\n", __func__); - return false; + sam3_result nd; + if (!tracker.params.text_prompt.empty()) { + sam3_pcs_params pcs; + pcs.text_prompt = tracker.params.text_prompt; + pcs.score_threshold = tracker.params.score_threshold; + pcs.nms_threshold = tracker.params.nms_threshold; + nd = sam3_segment_pcs(state, model, pcs); } - - auto* neck2 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(neck2, "phase7_curr_neck2"); - ggml_set_input(neck2); - - // Mask input at interpol resolution in WHCB layout for ggml_conv_2d - auto* mask_in = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, INTERPOL, INTERPOL, 1, 1); - ggml_set_name(mask_in, "phase7_mask_interp"); - ggml_set_input(mask_in); - - // ── Learned mask downsampler: 4 stages conv+LN+GELU, then 1×1 conv ── - // permute(1,2,0,3) converts conv→internal (ne[0]=channel for LN2d) - // permute(2,0,1,3) converts internal→conv - auto* ds = mask_in; - for (int s = 0; s < 4; ++s) { - int out_ch = (int)model.mem_enc.ds_conv_w[s]->ne[3]; - ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[s], ds, 2, 2, 1, 1, 1, 1); - ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[s], 1, 1, out_ch, 1)); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal - ds = sam3_layer_norm_2d(ctx0, ds, model.mem_enc.ds_norm_w[s], model.mem_enc.ds_norm_b[s]); - ds = ggml_gelu(ctx0, ds); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 2, 0, 1, 3)); // internal→conv + // Match new PCS detections against ALL tracked objects (both active AND pending) + // so that the same object isn't detected as a new instance every frame. + std::vector all_tracked; + std::vector all_tracked_masks; + for (auto& ml : tracker.masklets) { + all_tracked.push_back(ml); + auto it = pm.find(ml.instance_id); + all_tracked_masks.push_back(it != pm.end() ? it->second : sam3_mask{}); } - // Final 1×1 conv: channels → D=256 - ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[4], ds, 1, 1, 0, 0, 1, 1); - ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[4], 1, 1, D, 1)); - ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3)); // conv→internal [D, ...] + for (auto& ml : tracker.pending) { + all_tracked.push_back(ml); + auto it = pm.find(ml.instance_id); + all_tracked_masks.push_back(it != pm.end() ? it->second : sam3_mask{}); + } + auto mat = sam3_match_detections(all_tracked, nd.detections, + all_tracked_masks, tracker.params.assoc_iou_threshold); + std::vector dmat(nd.detections.size(), false); + for (auto& m : mat) { + dmat[m.second] = true; + // Update last_seen on the matched masklet (in active or pending) + int mid = all_tracked[m.first].instance_id; + for (auto& ml : tracker.masklets) + if (ml.instance_id == mid) { + ml.last_seen = fi; + break; + } + for (auto& ml : tracker.pending) + if (ml.instance_id == mid) { + ml.last_seen = fi; + ml.mds_sum++; + break; + } + } + // Map: pending instance_id → PCS detection index (for display) + std::map pending_det_idx; - // ── Pixel feature projection ── - auto* neck2_whcb = ggml_cont(ctx0, ggml_permute(ctx0, neck2, 2, 0, 1, 3)); - auto* pix = ggml_conv_2d(ctx0, model.mem_enc.pix_proj_w, neck2_whcb, 1, 1, 0, 0, 1, 1); - pix = ggml_add(ctx0, pix, ggml_reshape_4d(ctx0, model.mem_enc.pix_proj_b, 1, 1, D, 1)); - pix = ggml_cont(ctx0, ggml_permute(ctx0, pix, 1, 2, 0, 3)); - sam3_name_tensorf(pix, "phase7_mem%d_pix_proj", slot_idx); + for (int j = 0; j < (int)nd.detections.size(); ++j) { + if (dmat[j]) continue; + sam3_masklet ml; + ml.instance_id = tracker.next_inst_id++; + ml.first_frame = fi; + ml.last_seen = fi; + ml.last_score = nd.detections[j].score; + ml.mds_sum = 1; + pending_det_idx[ml.instance_id] = j; - // ── Fusion: ADD (not multiply) ── - auto* fused = ggml_add(ctx0, pix, ds); - sam3_name_tensorf(fused, "phase7_mem%d_fused_input", slot_idx); + // Encode the detection's mask into memory so subsequent frames can + // propagate from it. Without this, pending masklets have no memory + // and propagation returns empty on all future frames. + const auto& det = nd.detections[j]; + if (!det.mask.data.empty()) { + // Build mask logits from the PCS detection's binary mask. + const int mh = sam3_eff_feat_size(state, model.hparams) * 4, mw = mh; + // Resize to mh×mw logit space: inside mask → +5.0, outside → -5.0. + std::vector det_logits(mh * mw); + for (int r = 0; r < mh; ++r) { + for (int c = 0; c < mw; ++c) { + int sx = c * det.mask.width / mw; + int sy = r * det.mask.height / mh; + sx = std::min(sx, det.mask.width - 1); + sy = std::min(sy, det.mask.height - 1); + det_logits[r * mw + c] = (det.mask.data[sy * det.mask.width + sx] > 127) + ? 5.0f + : -5.0f; + } + } + sam3_encode_memory(tracker, state, model, ml.instance_id, + det_logits.data(), mh, mw, fi, true, 1.0f); - // ── CXBlocks ── - auto* h = fused; - struct ggml_tensor* fuser0 = nullptr; - struct ggml_tensor* fuser1 = nullptr; - for (int i = 0; i < 2; ++i) { - h = sam3_cxblock_forward(ctx0, h, - model.mem_enc.fuser_dw_w[i], model.mem_enc.fuser_dw_b[i], - model.mem_enc.fuser_norm_w[i], model.mem_enc.fuser_norm_b[i], - model.mem_enc.fuser_fc1_w[i], model.mem_enc.fuser_fc1_b[i], - model.mem_enc.fuser_fc2_w[i], model.mem_enc.fuser_fc2_b[i], - model.mem_enc.fuser_gamma[i]); - if (i == 0) { - sam3_name_tensorf(h, "phase7_mem%d_fuser0", slot_idx); - fuser0 = h; - } else { - sam3_name_tensorf(h, "phase7_mem%d_fuser1", slot_idx); - fuser1 = h; + // Store a dummy object pointer (from no_obj_ptr since we don't + // have a SAM token from PCS — PCS uses a different decoder) + std::vector no_ptr(D); + auto* nop = model.tensors.at("no_obj_ptr"); + if (nop->type == GGML_TYPE_F16) { + std::vector tmp(D); + ggml_backend_tensor_get(nop, tmp.data(), 0, D * sizeof(ggml_fp16_t)); + ggml_fp16_to_fp32_row(tmp.data(), no_ptr.data(), D); + } else { + ggml_backend_tensor_get(nop, no_ptr.data(), 0, D * sizeof(float)); + } + sam3_store_obj_ptr(tracker, model, ml.instance_id, no_ptr.data(), fi); } + + tracker.pending.push_back(std::move(ml)); + } + for (auto& ml : tracker.masklets) { + int id = ml.instance_id; + auto it = po.find(id); + if (it == po.end() || it->second.mask_logits.empty()) continue; + sam3_encode_memory(tracker, state, model, id, + it->second.mask_logits.data(), it->second.mask_h, it->second.mask_w, fi, false, it->second.obj_score); + std::vector op(D); + sam3_extract_obj_ptr_cpu(model, it->second.sam_token.data(), it->second.obj_score, op.data()); + sam3_store_obj_ptr(tracker, model, id, op.data(), fi); } + sam3_update_tracker(tracker, fi); - // ── Output projection ── - auto* h_whcb = ggml_cont(ctx0, ggml_permute(ctx0, h, 2, 0, 1, 3)); - auto* mo = ggml_conv_2d(ctx0, model.mem_enc.out_proj_w, h_whcb, 1, 1, 0, 0, 1, 1); - mo = ggml_add(ctx0, mo, ggml_reshape_4d(ctx0, model.mem_enc.out_proj_b, 1, 1, MD, 1)); - mo = ggml_cont(ctx0, ggml_permute(ctx0, mo, 1, 2, 0, 3)); - sam3_name_tensorf(mo, "phase7_mem%d_output", slot_idx); - - ggml_set_output(pix); - ggml_set_output(fused); - if (fuser0) ggml_set_output(fuser0); - if (fuser1) ggml_set_output(fuser1); - ggml_set_output(mo); - - auto* graph = ggml_new_graph_custom(ctx0, 16384, false); - ggml_build_forward_expand(graph, mo); - ggml_build_forward_expand(graph, pix); - ggml_build_forward_expand(graph, fused); - if (fuser0) ggml_build_forward_expand(graph, fuser0); - if (fuser1) ggml_build_forward_expand(graph, fuser1); + // Helper: build detection from a mask and add to result + auto add_mask_to_result = [&](int inst_id, float score, const sam3_mask& mask) { + if (mask.data.empty()) return; + sam3_detection det; + det.instance_id = inst_id; + det.score = score; + det.mask = mask; + det.mask.instance_id = inst_id; + det.mask.iou_score = score; + float x0 = 1e9f, y0 = 1e9f, x1 = -1e9f, y1 = -1e9f; + for (int p = 0; p < (int)det.mask.data.size(); ++p) + if (det.mask.data[p] > 127) { + int x = p % det.mask.width, y = p / det.mask.width; + x0 = std::min(x0, (float)x); + y0 = std::min(y0, (float)y); + x1 = std::max(x1, (float)x); + y1 = std::max(y1, (float)y); + } + if (x0 <= x1) det.box = {x0, y0, x1, y1}; + result.detections.push_back(std::move(det)); + }; - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate phase 7 memory graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; + // Include confirmed (active) masklet propagation results + for (auto& ml : tracker.masklets) { + auto it = pm.find(ml.instance_id); + if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); } - ggml_backend_tensor_set(neck2, neck2_data.data(), 0, neck2_data.size() * sizeof(float)); - // Mask is in WHCB layout = row-major over (w, h): just upload flat - ggml_backend_tensor_set(mask_in, m_interp.data(), 0, m_interp.size() * sizeof(float)); - - sam3_graph_compute(model.backend, graph, n_threads); - - // Read output and apply no_obj_embed_spatial if occluded - mem_out_data.resize(MD * H * H); - ggml_backend_tensor_get(mo, mem_out_data.data(), 0, mem_out_data.size() * sizeof(float)); - - if (obj_score <= 0.0f && model.no_obj_embed_spatial) { - std::vector no_obj_emb(MD); - auto* noe = model.no_obj_embed_spatial; - if (noe->type == GGML_TYPE_F16) { - std::vector tmp(MD); - ggml_backend_tensor_get(noe, tmp.data(), 0, MD * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), no_obj_emb.data(), MD); + // Include pending masklet results: on the detection frame, use the PCS + // detection mask; on subsequent frames, use the propagated mask. + for (auto& ml : tracker.pending) { + auto it = pm.find(ml.instance_id); + if (it != pm.end() && !it->second.data.empty()) { + // Propagated mask available (frame > detection frame) + add_mask_to_result(ml.instance_id, ml.last_score, it->second); } else { - ggml_backend_tensor_get(noe, no_obj_emb.data(), 0, MD * sizeof(float)); + // First detection frame: use the PCS detection's mask+box directly + auto di = pending_det_idx.find(ml.instance_id); + if (di != pending_det_idx.end() && di->second < (int)nd.detections.size()) { + const auto& det = nd.detections[di->second]; + if (!det.mask.data.empty()) { + add_mask_to_result(ml.instance_id, ml.last_score, det.mask); + } + } } - for (int i = 0; i < MD * H * H; ++i) - mem_out_data[i] += no_obj_emb[i % MD]; } + sam3_resolve_overlaps(result.detections); + for (auto& d : result.detections) { + if (d.mask.data.empty()) continue; + sam3_fill_holes(d.mask.data.data(), d.mask.width, d.mask.height, tracker.params.fill_hole_area); + sam3_remove_sprinkles(d.mask.data.data(), d.mask.width, d.mask.height, tracker.params.fill_hole_area); + } + tracker.frame_index++; + SAM3_LOG(2, "%s: frame %d done — %zu tracked\n", __func__, fi, result.detections.size()); + return result; +} - // Compute sinusoidal spatial PE [MD, 72, 72] - mem_pe_data = sam3_sinusoidal_pe_2d(H, H, MD); - - bool ok = true; - const std::vector tensor_names = { - "phase7_mem" + std::to_string(slot_idx) + "_pix_proj", - "phase7_mem" + std::to_string(slot_idx) + "_fused_input", - "phase7_mem" + std::to_string(slot_idx) + "_fuser0", - "phase7_mem" + std::to_string(slot_idx) + "_fuser1", - "phase7_mem" + std::to_string(slot_idx) + "_output", - }; - for (const auto& name : tensor_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t || !sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; +bool sam3_refine_instance(sam3_tracker& tracker, sam3_state& state, + const sam3_model& model, int instance_id, + const std::vector& pos_points, + const std::vector& neg_points) { + const int D = model.hparams.neck_dim; + sam3_masklet* tgt = nullptr; + for (auto& ml : tracker.masklets) + if (ml.instance_id == instance_id) { + tgt = &ml; + break; } + if (!tgt) + for (auto& ml : tracker.pending) + if (ml.instance_id == instance_id) { + tgt = &ml; + break; + } + if (!tgt) { + fprintf(stderr, "%s: instance %d not found\n", __func__, instance_id); + return false; } - // Also dump the output with no_obj_embed_spatial applied - if (!sam3_dump_raw_f32_to_path(mem_out_data.data(), {MD, H, H, 1}, - output_dir + "/phase7_mem" + std::to_string(slot_idx) + "_output")) { - ok = false; + sam3_pvs_params pvs; + pvs.pos_points = pos_points; + pvs.neg_points = neg_points; + pvs.multimask = false; + auto r = sam3_segment_pvs(state, model, pvs); + if (r.detections.empty()) return false; + // tracker.frame_index points to the *next* frame; the refinement applies + // to the frame that was last tracked / encoded. + int fi = std::max(0, tracker.frame_index - 1); + const auto& rdet = r.detections[0]; + tgt->last_score = rdet.score; + tgt->last_seen = fi; + std::vector op(D); + if (!rdet.sam_token.empty()) { + sam3_extract_obj_ptr_cpu(model, rdet.sam_token.data(), rdet.mask.obj_score, op.data()); + } else { + std::fill(op.begin(), op.end(), 0.0f); } - - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; + sam3_store_obj_ptr(tracker, model, instance_id, op.data(), fi); + SAM3_LOG(2, "%s: refined instance %d\n", __func__, instance_id); + return true; } -bool sam3_test_dump_phase7_from_ref_inputs(const sam3_model& model, - const std::string& case_ref_dir, - const std::string& output_dir, - int n_threads) { - const auto& hp = model.hparams; - const int D = hp.neck_dim; - const int MD = hp.mem_out_dim; - const int H = hp.feat_size(); - const int N = H * H; - const int H1 = H * 2; - const int H0 = H * 4; - const int MASK_HW = H * 4; - const int half_d = D / 2; // 128 +int sam3_tracker_add_instance(sam3_tracker& tracker, sam3_state& state, + const sam3_model& model, + const sam3_pvs_params& pvs_params) { + const int D = model.hparams.neck_dim; + const int mask_hw = sam3_eff_feat_size(state, model.hparams) * 4; - std::map meta; - if (!sam3_load_kv_text_file(case_ref_dir + "/meta.txt", meta)) { - return false; - } - const int num_slots = sam3_meta_get_int(meta, "num_slots", 0); - if (num_slots <= 0) { - fprintf(stderr, "%s: no memory slots in %s\n", __func__, case_ref_dir.c_str()); - return false; + // Run PVS to get the segmentation mask + auto r = sam3_segment_pvs(state, model, pvs_params); + if (r.detections.empty()) { + fprintf(stderr, "%s: PVS returned no masks\n", __func__); + return -1; } - std::vector neck_trk_0_nchw; - std::vector neck_trk_1_nchw; - std::vector neck_trk_2_nchw; - if (!sam3_load_ref_f32_data(case_ref_dir + "/input_curr_neck_trk_0", neck_trk_0_nchw, D * H0 * H0) || - !sam3_load_ref_f32_data(case_ref_dir + "/input_curr_neck_trk_1", neck_trk_1_nchw, D * H1 * H1) || - !sam3_load_ref_f32_data(case_ref_dir + "/input_curr_neck_trk_2", neck_trk_2_nchw, D * H * H)) { - return false; + const auto& det = r.detections[0]; + if (det.mask.data.empty()) { + fprintf(stderr, "%s: PVS mask is empty\n", __func__); + return -1; } - auto neck_trk_0 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_0_nchw, D, H0, H0); - auto neck_trk_1 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_1_nchw, D, H1, H1); - auto neck_trk_2 = sam3_reorder_nchw_to_ggml_dwh(neck_trk_2_nchw, D, H, H); - - // ── Memory encoding per slot ──────────────────────────────────────── - std::vector> mem_slot_outputs(num_slots); - std::vector> mem_slot_pes(num_slots); - std::vector> obj_ptrs; - std::vector slot_spatial_tpos(num_slots); - std::vector slot_ptr_tpos(num_slots); - bool ok = true; - - for (int i = 0; i < num_slots; ++i) { - std::vector mask_logits_nchw; - std::vector sam_token; - std::vector obj_score; - if (!sam3_load_ref_f32_data(case_ref_dir + "/input_mem_mask_logits_" + std::to_string(i), - mask_logits_nchw, MASK_HW * MASK_HW) || - !sam3_load_ref_f32_data(case_ref_dir + "/input_mem_sam_token_" + std::to_string(i), - sam_token, D) || - !sam3_load_ref_f32_data(case_ref_dir + "/input_mem_obj_score_" + std::to_string(i), - obj_score, 1)) { - return false; - } - - slot_spatial_tpos[i] = sam3_meta_get_int(meta, "slot" + std::to_string(i) + "_spatial_tpos", 0); - slot_ptr_tpos[i] = sam3_meta_get_int(meta, "slot" + std::to_string(i) + "_ptr_tpos", -1); - - if (!sam3_test_dump_phase7_mem_slot_current(model, neck_trk_2, mask_logits_nchw, - obj_score[0], i, output_dir, - mem_slot_outputs[i], mem_slot_pes[i], n_threads)) { - ok = false; - } + int inst_id = tracker.next_inst_id++; + // tracker.frame_index points to the *next* frame to process; + // the instance is being added on the frame that was just tracked. + int fi = tracker.frame_index - 1; + if (fi < 0) fi = 0; - std::vector obj_ptr(D, 0.0f); - sam3_extract_obj_ptr_cpu(model, sam_token.data(), obj_score[0], obj_ptr.data()); - if (!sam3_dump_raw_f32_to_path(obj_ptr.data(), {D, 1}, - output_dir + "/phase7_obj_ptr" + std::to_string(i))) { - ok = false; + // Create synthetic 288x288 logits from the binary mask. + // sam3_encode_memory applies sigmoid then scale/bias, so +6/-6 gives + // sigmoid values ~0.9975/~0.0025 — practically identical to real logits. + std::vector synth_logits(mask_hw * mask_hw); + { + int mw = det.mask.width, mh = det.mask.height; + for (int y = 0; y < mask_hw; ++y) { + int sy = y * mh / mask_hw; + for (int x = 0; x < mask_hw; ++x) { + int sx = x * mw / mask_hw; + synth_logits[y * mask_hw + x] = + (det.mask.data[sy * mw + sx] > 127) ? 6.0f : -6.0f; + } } - - obj_ptrs.push_back(obj_ptr); - } - - // ── Build prompt and prompt_pos ───────────────────────────────────── - std::vector ptr_tpos_vec(num_slots); - for (int i = 0; i < num_slots; ++i) - ptr_tpos_vec[i] = slot_ptr_tpos[i]; - - auto pd = sam3_build_prompt_and_pos(model, mem_slot_outputs, mem_slot_pes, - slot_spatial_tpos, obj_ptrs, ptr_tpos_vec); - - // ── Precompute RoPE frequencies ───────────────────────────────────── - std::vector rope_q_data(N * D); - sam3_compute_axial_cis(rope_q_data.data(), D, H, H, 10000.0f, 1.0f); - - std::vector rope_k_data; - if (pd.M_spatial > 0) { - int n_spatial_slots = pd.M_spatial / N; - rope_k_data.resize(2 * half_d * pd.M_spatial); - for (int s = 0; s < n_spatial_slots; ++s) - memcpy(rope_k_data.data() + s * D * N, rope_q_data.data(), D * N * sizeof(float)); } - // ── Build graph ───────────────────────────────────────────────────── - const size_t buf_size = ggml_tensor_overhead() * 32768 + ggml_graph_overhead() * 2; - struct ggml_init_params gparams = {buf_size, nullptr, true}; - struct ggml_context* ctx0 = ggml_init(gparams); - if (!ctx0) { - fprintf(stderr, "%s: failed to init phase 7 context\n", __func__); - return false; + // Encode into memory bank + float obj_score = det.mask.obj_score; + if (!sam3_encode_memory(tracker, state, model, inst_id, + synth_logits.data(), mask_hw, mask_hw, + fi, true, obj_score)) { + fprintf(stderr, "%s: failed to encode memory for instance %d\n", __func__, inst_id); + return -1; } - auto* neck0 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H0, H0, 1); - auto* neck1 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H1, H1, 1); - auto* neck2 = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(neck0, "phase7_curr_neck0"); - ggml_set_name(neck1, "phase7_curr_neck1"); - ggml_set_name(neck2, "phase7_curr_neck2"); - ggml_set_input(neck0); - ggml_set_input(neck1); - ggml_set_input(neck2); - - auto* curr = ggml_reshape_3d(ctx0, neck2, D, N, 1); - - // src_pos for pos_enc_at_input - auto* src_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, N, 1); - ggml_set_name(src_pos_t, "phase7_src_pos"); - ggml_set_input(src_pos_t); - - // Prompt and prompt_pos - auto* prompt_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); - ggml_set_name(prompt_t, "phase7_prompt"); - ggml_set_input(prompt_t); - auto* prompt_pos_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, MD, pd.M_total, 1); - ggml_set_name(prompt_pos_t, "phase7_prompt_pos"); - ggml_set_input(prompt_pos_t); - - // RoPE frequencies - auto* rope_q_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, N); - ggml_set_name(rope_q_t, "phase7_rope_q"); - ggml_set_input(rope_q_t); - struct ggml_tensor* rope_k_t = nullptr; - if (pd.M_spatial > 0) { - rope_k_t = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 2, half_d, pd.M_spatial); - ggml_set_name(rope_k_t, "phase7_rope_k"); - ggml_set_input(rope_k_t); + // Extract object pointer from the SAM decoder token + std::vector op(D); + if (!det.sam_token.empty()) { + sam3_extract_obj_ptr_cpu(model, det.sam_token.data(), obj_score, op.data()); + } else { + std::fill(op.begin(), op.end(), 0.0f); } + sam3_store_obj_ptr(tracker, model, inst_id, op.data(), fi); - auto* conditioned = sam3_build_mem_attn_graph(ctx0, model, curr, src_pos_t, - prompt_t, prompt_pos_t, - rope_q_t, rope_k_t, - pd.num_obj_ptr_tokens); - auto* cond_spatial = ggml_reshape_4d(ctx0, conditioned, D, H, H, 1); - - // Bug 3 fix: single not_a_point_embed token - auto* sparse_in = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, D, 1, 1); - ggml_set_name(sparse_in, "phase7_sparse"); - ggml_set_input(sparse_in); - - auto* image_pe = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - auto* dense_emb = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1); - ggml_set_name(image_pe, "phase7_image_pe"); - ggml_set_name(dense_emb, "phase7_dense"); - ggml_set_input(image_pe); - ggml_set_input(dense_emb); + // Create confirmed masklet + sam3_masklet ml; + ml.instance_id = inst_id; + ml.first_frame = fi; + ml.last_seen = fi; + ml.last_score = det.score; + ml.confirmed = true; + ml.mds_sum = 1; + tracker.masklets.push_back(std::move(ml)); - auto dec = sam3_build_sam_dec_graph(ctx0, model, cond_spatial, image_pe, - sparse_in, dense_emb, neck0, neck1); - - auto* mask0 = ggml_view_3d(ctx0, dec.masks, MASK_HW * MASK_HW, 1, 1, - dec.masks->nb[1], dec.masks->nb[2], 0); - mask0 = ggml_cont(ctx0, mask0); - ggml_set_name(mask0, "phase7_prop_masks"); - - auto* iou0 = ggml_view_2d(ctx0, dec.iou_pred, 1, 1, dec.iou_pred->nb[1], 0); - iou0 = ggml_cont(ctx0, iou0); - ggml_set_name(iou0, "phase7_prop_iou"); - ggml_set_name(dec.obj_score, "phase7_prop_obj_score"); - ggml_set_name(dec.sam_token, "phase7_prop_sam_token"); - - ggml_set_output(conditioned); - ggml_set_output(mask0); - ggml_set_output(iou0); - ggml_set_output(dec.obj_score); - ggml_set_output(dec.sam_token); + SAM3_LOG(2, "%s: added instance #%d (score=%.3f)\n", __func__, inst_id, det.score); + return inst_id; +} - const std::vector mem_attn_names = { - "phase7_mem_attn_input", - "phase7_mem_attn_layer0_after_sa", - "phase7_mem_attn_layer0_after_ca", - "phase7_mem_attn_layer0_after_ffn", - "phase7_mem_attn_layer1_after_sa", - "phase7_mem_attn_layer1_after_ca", - "phase7_mem_attn_layer1_after_ffn", - "phase7_mem_attn_layer2_after_sa", - "phase7_mem_attn_layer2_after_ca", - "phase7_mem_attn_layer2_after_ffn", - "phase7_mem_attn_layer3_after_sa", - "phase7_mem_attn_layer3_after_ca", - "phase7_mem_attn_layer3_after_ffn", - "phase7_mem_attn_output", - "phase7_prop_masks", - "phase7_prop_iou", - "phase7_prop_obj_score", - "phase7_prop_sam_token", - }; - for (const auto& name : mem_attn_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) ggml_set_output(t); - } +int sam3_tracker_frame_index(const sam3_tracker& tracker) { return tracker.frame_index; } - auto* graph = ggml_new_graph_custom(ctx0, 32768, false); - ggml_build_forward_expand(graph, mask0); - ggml_build_forward_expand(graph, iou0); - ggml_build_forward_expand(graph, dec.obj_score); - ggml_build_forward_expand(graph, dec.sam_token); - ggml_build_forward_expand(graph, conditioned); - for (const auto& name : mem_attn_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (t) ggml_build_forward_expand(graph, t); +void sam3_tracker_reset(sam3_tracker& tracker) { + tracker.frame_index = 0; + tracker.next_inst_id = 1; + tracker.masklets.clear(); + tracker.pending.clear(); + tracker.mem_banks.clear(); + tracker.ptr_banks.clear(); + for (auto* b : tracker.owned_buffers) + if (b) ggml_backend_buffer_free(b); + tracker.owned_buffers.clear(); + if (tracker.ctx) { + ggml_free(tracker.ctx); + tracker.ctx = nullptr; } - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(model.backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "%s: failed to allocate phase 7 graph\n", __func__); - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return false; + if (tracker.buffer) { + ggml_backend_buffer_free(tracker.buffer); + tracker.buffer = nullptr; } +} - // Upload neck features - ggml_backend_tensor_set(neck0, neck_trk_0.data(), 0, neck_trk_0.size() * sizeof(float)); - ggml_backend_tensor_set(neck1, neck_trk_1.data(), 0, neck_trk_1.size() * sizeof(float)); - ggml_backend_tensor_set(neck2, neck_trk_2.data(), 0, neck_trk_2.size() * sizeof(float)); +/***************************************************************************** +** Visual-only video tracking +*****************************************************************************/ - // Upload prompt and prompt_pos - ggml_backend_tensor_set(prompt_t, pd.prompt.data(), 0, pd.prompt.size() * sizeof(float)); - ggml_backend_tensor_set(prompt_pos_t, pd.prompt_pos.data(), 0, pd.prompt_pos.size() * sizeof(float)); +sam3_tracker_ptr sam3_create_visual_tracker( + const sam3_model& model, + const sam3_visual_track_params& params) { + sam3_video_params vp; + vp.text_prompt = ""; // no PCS detection + vp.assoc_iou_threshold = params.assoc_iou_threshold; + vp.max_keep_alive = params.max_keep_alive; + vp.recondition_every = params.recondition_every; + vp.fill_hole_area = params.fill_hole_area; + sam3_tracker_ptr tracker(new sam3_tracker()); + tracker->params = vp; + fprintf(stderr, "%s: visual-only tracker created (max_keep_alive=%d)\n", + __func__, params.max_keep_alive); + return tracker; +} - // Upload RoPE frequencies - ggml_backend_tensor_set(rope_q_t, rope_q_data.data(), 0, rope_q_data.size() * sizeof(float)); - if (rope_k_t && !rope_k_data.empty()) - ggml_backend_tensor_set(rope_k_t, rope_k_data.data(), 0, rope_k_data.size() * sizeof(float)); +sam3_result sam3_propagate_frame( + sam3_tracker& tracker, sam3_state& state, + const sam3_model& model, const sam3_image& frame) { + sam3_result result; + const int D = model.hparams.neck_dim; + if (!sam3_encode_image(state, model, frame)) return result; + int fi = tracker.frame_index; + fprintf(stderr, "%s: frame %d (%zu active + %zu pending)\n", + __func__, fi, tracker.masklets.size(), tracker.pending.size()); - // Upload src_pos (sinusoidal PE 256-dim for 72×72) - auto src_pos_data = sam3_sinusoidal_pe_2d(H, H, D); - ggml_backend_tensor_set(src_pos_t, src_pos_data.data(), 0, src_pos_data.size() * sizeof(float)); + // ── Propagate active masklets ──────────────────────────────────────── + std::map pm; + std::map po; + for (auto& ml : tracker.masklets) { + int id = ml.instance_id; + auto im = tracker.mem_banks.find(id); + if (im == tracker.mem_banks.end() || im->second.empty()) continue; + po[id] = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); + if (po[id].mask_logits.empty()) continue; + auto rs = sam3_bilinear_interpolate(po[id].mask_logits.data(), + po[id].mask_w, po[id].mask_h, + state.orig_width, state.orig_height); + pm[id].width = state.orig_width; + pm[id].height = state.orig_height; + pm[id].data.resize(state.orig_width * state.orig_height); + int fg = 0; + for (int p = 0; p < (int)rs.size(); ++p) { + bool f = rs[p] > 0.0f; + pm[id].data[p] = f ? 255 : 0; + if (f) fg++; + } + ml.last_score = po[id].iou_scores[0]; + ml.last_seen = fi; + float cov = (float)fg / (state.orig_width * state.orig_height); + ml.mds_sum += (cov > 0.001f && po[id].obj_score > 0.0f) ? 1 : -1; + } - // Upload not_a_point_embed for sparse prompt - { - float nap[256]; - if (model.sam_pe.not_a_point_embed->type == GGML_TYPE_F16) { - std::vector tmp(D); - ggml_backend_tensor_get(model.sam_pe.not_a_point_embed, tmp.data(), 0, D * sizeof(ggml_fp16_t)); - ggml_fp16_to_fp32_row(tmp.data(), nap, D); - } else { - ggml_backend_tensor_get(model.sam_pe.not_a_point_embed, nap, 0, D * sizeof(float)); + // ── Propagate pending masklets ─────────────────────────────────────── + for (auto& ml : tracker.pending) { + int id = ml.instance_id; + auto im = tracker.mem_banks.find(id); + if (im == tracker.mem_banks.end() || im->second.empty()) continue; + auto p2 = sam3_propagate_single(tracker, state, model, ml, im->second, tracker.ptr_banks[id]); + if (!p2.mask_logits.empty()) { + ml.last_score = p2.iou_scores[0]; + ml.last_seen = fi; + auto r2 = sam3_bilinear_interpolate(p2.mask_logits.data(), + p2.mask_w, p2.mask_h, + state.orig_width, state.orig_height); + int fg2 = 0; + for (auto v : r2) + if (v > 0.0f) fg2++; + float c2 = (float)fg2 / (state.orig_width * state.orig_height); + ml.mds_sum += (c2 > 0.001f && p2.obj_score > 0.0f) ? 1 : -1; + pm[id].width = state.orig_width; + pm[id].height = state.orig_height; + pm[id].data.resize(state.orig_width * state.orig_height); + for (int p = 0; p < (int)r2.size(); ++p) + pm[id].data[p] = r2[p] > 0.0f ? 255 : 0; + sam3_encode_memory(tracker, state, model, id, + p2.mask_logits.data(), p2.mask_h, p2.mask_w, + fi, false, p2.obj_score); + std::vector op(D); + sam3_extract_obj_ptr_cpu(model, p2.sam_token.data(), p2.obj_score, op.data()); + sam3_store_obj_ptr(tracker, model, id, op.data(), fi); } - ggml_backend_tensor_set(sparse_in, nap, 0, D * sizeof(float)); } - // Upload image_pe and dense_emb - sam3_state pe_cache_state = {}; - sam3_populate_pe_cache(pe_cache_state, model); - ggml_backend_tensor_set(image_pe, pe_cache_state.dense_pe_cache.data(), 0, - pe_cache_state.dense_pe_cache.size() * sizeof(float)); - ggml_backend_tensor_set(dense_emb, pe_cache_state.dense_nomask_cache.data(), 0, - pe_cache_state.dense_nomask_cache.size() * sizeof(float)); + // ── Encode memory for active masklets ──────────────────────────────── + for (auto& ml : tracker.masklets) { + int id = ml.instance_id; + auto it = po.find(id); + if (it == po.end() || it->second.mask_logits.empty()) continue; + sam3_encode_memory(tracker, state, model, id, + it->second.mask_logits.data(), it->second.mask_h, + it->second.mask_w, fi, false, it->second.obj_score); + std::vector op(D); + sam3_extract_obj_ptr_cpu(model, it->second.sam_token.data(), + it->second.obj_score, op.data()); + sam3_store_obj_ptr(tracker, model, id, op.data(), fi); + } - sam3_graph_compute(model.backend, graph, n_threads); + // ── Update tracker state (confirmation / eviction) ─────────────────── + sam3_update_tracker(tracker, fi); - // Dump the mem_attn_input manually (it's now x = curr + 0.1*src_pos, not just curr) - // The named tensor "phase7_mem_attn_input" is no longer in the graph since we - // removed the explicit naming. But all the per-layer outputs are still named. - for (const auto& name : mem_attn_names) { - auto* t = ggml_get_tensor(ctx0, name.c_str()); - if (!t || !sam3_dump_tensor_to_path(t, name, output_dir + "/" + name)) { - ok = false; - } + // ── Build result ───────────────────────────────────────────────────── + auto add_mask_to_result = [&](int inst_id, float score, const sam3_mask& mask) { + if (mask.data.empty()) return; + sam3_detection det; + det.instance_id = inst_id; + det.score = score; + det.mask = mask; + det.mask.instance_id = inst_id; + det.mask.iou_score = score; + float x0 = 1e9f, y0 = 1e9f, x1 = -1e9f, y1 = -1e9f; + for (int p = 0; p < (int)det.mask.data.size(); ++p) + if (det.mask.data[p] > 127) { + int x = p % det.mask.width, y = p / det.mask.width; + x0 = std::min(x0, (float)x); + y0 = std::min(y0, (float)y); + x1 = std::max(x1, (float)x); + y1 = std::max(y1, (float)y); + } + if (x0 <= x1) det.box = {x0, y0, x1, y1}; + result.detections.push_back(std::move(det)); + }; + + for (auto& ml : tracker.masklets) { + auto it = pm.find(ml.instance_id); + if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); + } + for (auto& ml : tracker.pending) { + auto it = pm.find(ml.instance_id); + if (it != pm.end()) add_mask_to_result(ml.instance_id, ml.last_score, it->second); } - ggml_gallocr_free(galloc); - ggml_free(ctx0); - return ok; + sam3_resolve_overlaps(result.detections); + for (auto& d : result.detections) { + if (d.mask.data.empty()) continue; + sam3_fill_holes(d.mask.data.data(), d.mask.width, d.mask.height, + tracker.params.fill_hole_area); + sam3_remove_sprinkles(d.mask.data.data(), d.mask.width, d.mask.height, + tracker.params.fill_hole_area); + } + tracker.frame_index++; + SAM3_LOG(2, "%s: frame %d done — %zu tracked\n", + __func__, fi, result.detections.size()); + return result; } /***************************************************************************** -** Debug: dump state tensors +** Utility — image I/O *****************************************************************************/ -static struct ggml_tensor * sam3_find_state_tensor(const sam3_state & state, - const std::string & tensor_name) { - struct ggml_tensor* t = nullptr; - - if (tensor_name == "vit_output") { - t = state.vit_output; - } else if (tensor_name == "neck_det_0") { - t = state.neck_det[0]; - } else if (tensor_name == "neck_det_1") { - t = state.neck_det[1]; - } else if (tensor_name == "neck_det_2") { - t = state.neck_det[2]; - } else if (tensor_name == "neck_det_3") { - t = state.neck_det[3]; - } else if (tensor_name == "neck_trk_0") { - t = state.neck_trk[0]; - } else if (tensor_name == "neck_trk_1") { - t = state.neck_trk[1]; - } else if (tensor_name == "neck_trk_2") { - t = state.neck_trk[2]; - } else if (tensor_name == "neck_trk_3") { - t = state.neck_trk[3]; - } else if (tensor_name == "neck_det_pe_0") { - t = state.neck_det_pe[0]; - } else if (tensor_name == "neck_det_pe_1") { - t = state.neck_det_pe[1]; - } else if (tensor_name == "neck_det_pe_2") { - t = state.neck_det_pe[2]; - } else if (tensor_name == "neck_det_pe_3") { - t = state.neck_det_pe[3]; - } else { - // Search by ggml name in the context - if (state.ctx) { - t = ggml_get_tensor(state.ctx, tensor_name.c_str()); - } - // Also search PE context - if (!t && state.pe_ctx) { - t = ggml_get_tensor(state.pe_ctx, tensor_name.c_str()); - } +sam3_image sam3_load_image(const std::string& path) { + sam3_image img; + int w, h, c; + uint8_t* data = stbi_load(path.c_str(), &w, &h, &c, 3); + if (!data) { + fprintf(stderr, "%s: failed to load '%s'\n", __func__, path.c_str()); + return img; } + img.width = w; + img.height = h; + img.channels = 3; + img.data.assign(data, data + w * h * 3); + stbi_image_free(data); + return img; +} - return t; +bool sam3_save_mask(const sam3_mask& mask, const std::string& path) { + if (mask.data.empty()) return false; + return stbi_write_png(path.c_str(), mask.width, mask.height, 1, + mask.data.data(), mask.width) != 0; } -static bool sam3_fill_tensor_info(struct ggml_tensor * t, sam3_tensor_info & info) { - if (!t) { - return false; - } +sam3_image sam3_decode_video_frame(const std::string& video_path, int frame_index) { + sam3_image img; - for (int i = 0; i < 4; ++i) { - info.ne[i] = t->ne[i]; - info.nb[i] = t->nb[i]; - } - info.type = (int) t->type; - info.op = (int) t->op; - info.is_contiguous = ggml_is_contiguous(t); - return true; -} + // Use ffmpeg to extract a single frame as raw RGB (frame-accurate) + char cmd[1024]; + snprintf(cmd, sizeof(cmd), + "ffmpeg -nostdin -loglevel error -i \"%s\" " + "-vf \"select=eq(n\\,%d)\" -vsync vfr -frames:v 1 " + "-f rawvideo -pix_fmt rgb24 pipe:1 2>%s", + video_path.c_str(), frame_index, SAM3_NULL_DEV); -bool sam3_get_state_tensor_info(const sam3_state & state, - const std::string & tensor_name, - sam3_tensor_info & info) { - struct ggml_tensor * t = sam3_find_state_tensor(state, tensor_name); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in state\n", __func__, tensor_name.c_str()); - return false; + // First, get dimensions + char info_cmd[1024]; + snprintf(info_cmd, sizeof(info_cmd), + "ffprobe -v error -select_streams v:0 " + "-show_entries stream=width,height -of csv=p=0 \"%s\" 2>%s", + video_path.c_str(), SAM3_NULL_DEV); + FILE* fp = popen(info_cmd, SAM3_POPEN_READ); + if (!fp) return img; + int w = 0, h = 0; + if (fscanf(fp, "%d,%d", &w, &h) != 2) { + pclose(fp); + return img; } - return sam3_fill_tensor_info(t, info); -} + pclose(fp); -bool sam3_dump_state_tensor(const sam3_state& state, - const std::string& tensor_name, - const std::string& output_path) { - struct ggml_tensor * t = sam3_find_state_tensor(state, tensor_name); - if (!t) { - fprintf(stderr, "%s: tensor '%s' not found in state\n", __func__, tensor_name.c_str()); - return false; - } - return sam3_dump_tensor_to_path(t, tensor_name, output_path); -} + img.width = w; + img.height = h; + img.channels = 3; + img.data.resize(w * h * 3); -bool sam3_get_model_tensor_info(const sam3_model & model, - const std::string & tensor_name, - sam3_tensor_info & info) { - auto it = model.tensors.find(tensor_name); - if (it == model.tensors.end() || !it->second) { - fprintf(stderr, "%s: tensor '%s' not found in model\n", __func__, tensor_name.c_str()); - return false; + fp = popen(cmd, SAM3_POPEN_READ); + if (!fp) { + img.data.clear(); + return img; + } + size_t nread = fread(img.data.data(), 1, img.data.size(), fp); + pclose(fp); + if (nread != img.data.size()) { + img.data.clear(); } - return sam3_fill_tensor_info(it->second, info); + + return img; } -bool sam3_dump_model_tensor(const sam3_model & model, - const std::string & tensor_name, - const std::string & output_path) { - auto it = model.tensors.find(tensor_name); - if (it == model.tensors.end() || !it->second) { - fprintf(stderr, "%s: tensor '%s' not found in model\n", __func__, tensor_name.c_str()); - return false; +sam3_video_info sam3_get_video_info(const std::string& video_path) { + sam3_video_info info; + + char cmd[1024]; + snprintf(cmd, sizeof(cmd), + "ffprobe -v error -select_streams v:0 " + "-show_entries stream=width,height,r_frame_rate,nb_frames " + "-of csv=p=0 \"%s\" 2>%s", + video_path.c_str(), SAM3_NULL_DEV); + FILE* fp = popen(cmd, SAM3_POPEN_READ); + if (!fp) return info; + + int w = 0, h = 0, num = 0, den = 1, nf = 0; + if (fscanf(fp, "%d,%d,%d/%d,%d", &w, &h, &num, &den, &nf) >= 4) { + info.width = w; + info.height = h; + info.fps = (den > 0) ? static_cast(num) / den : 0.0f; + info.n_frames = nf; } - return sam3_dump_tensor_to_path(it->second, tensor_name, output_path); + pclose(fp); + return info; } diff --git a/sam3.h b/sam3.h index 204e487..2f5c374 100644 --- a/sam3.h +++ b/sam3.h @@ -101,38 +101,6 @@ struct sam3_params { int encode_img_size = 0; // 0 = model default; override input resolution }; -struct sam3_tensor_info { - int64_t ne[4] = {0, 0, 0, 0}; - uint64_t nb[4] = {0, 0, 0, 0}; - int type = 0; - int op = 0; - bool is_contiguous = false; -}; - -enum sam3_vit_block_stage { - SAM3_VIT_BLOCK_STAGE_NORM1 = 0, - SAM3_VIT_BLOCK_STAGE_WINDOW_PART, - SAM3_VIT_BLOCK_STAGE_QKV_PROJ, - SAM3_VIT_BLOCK_STAGE_ATTN_CORE, - SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, - SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, - SAM3_VIT_BLOCK_STAGE_NORM2, - SAM3_VIT_BLOCK_STAGE_MLP_FC1, - SAM3_VIT_BLOCK_STAGE_MLP_GELU, - SAM3_VIT_BLOCK_STAGE_MLP_FC2, - SAM3_VIT_BLOCK_STAGE_MLP, -}; - -enum sam3_vit_prefix_stage { - SAM3_VIT_PREFIX_STAGE_PATCH_EMBED = 0, - SAM3_VIT_PREFIX_STAGE_PATCH_IM2COL, - SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW, - SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT, - SAM3_VIT_PREFIX_STAGE_POS_ADD, - SAM3_VIT_PREFIX_STAGE_LN_PRE_NORM, - SAM3_VIT_PREFIX_STAGE_LN_PRE, -}; - struct sam3_pcs_params { std::string text_prompt; std::vector pos_exemplars; @@ -308,193 +276,6 @@ bool sam3_save_mask(const sam3_mask & mask, const std::string & path) sam3_image sam3_decode_video_frame(const std::string & video_path, int frame_index); sam3_video_info sam3_get_video_info(const std::string & video_path); -/***************************************************************************** -** Test and Debug API -** -** Standalone tokenizer, intermediate tensor dumps, and debug utilities. -** These functions are intended for testing and development only. -*****************************************************************************/ - -bool sam3_test_load_tokenizer(const std::string & model_path); -std::vector sam3_test_tokenize(const std::string & text); - -/* -** Run the text encoder on fixed token IDs and dump standard intermediate -** tensors to /.{bin,shape}. -*/ -bool sam3_test_dump_text_encoder(const sam3_model & model, - const std::vector & token_ids, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the full phase 5 detector path (fusion encoder + DETR decoder + -** dot-product scoring + segmentation head) on an already-encoded image -** and dump intermediate tensors. -*/ -bool sam3_test_dump_phase5(const sam3_model & model, - const sam3_state & state, - const std::vector & token_ids, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the phase 5 detector from pre-dumped inputs instead of re-running -** the image/text encoders. Isolates detector numerics from earlier phases. -*/ -bool sam3_test_dump_phase5_from_ref_inputs(const sam3_model & model, - const std::vector & token_ids, - const std::string & prephase_ref_dir, - const std::string & phase5_ref_dir, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the phase 6 prompt encoder + SAM decoder on an already-encoded -** tracker image state and dump intermediate tensors. -*/ -bool sam3_test_dump_phase6(const sam3_model & model, - const sam3_state & state, - const sam3_pvs_params & params, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the phase 6 prompt encoder + SAM decoder from pre-dumped phase 3 -** tracker features. Isolates phase 6 numerics from earlier phases. -*/ -bool sam3_test_dump_phase6_from_ref_inputs(const sam3_model & model, - const std::string & prephase_ref_dir, - const sam3_pvs_params & params, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the phase 7 tracker subgraph from pre-dumped case inputs and dump -** intermediate tensors. Case directory produced by dump_phase7_reference.py. -*/ -bool sam3_test_dump_phase7_from_ref_inputs(const sam3_model & model, - const std::string & case_ref_dir, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run the geometry encoder from pre-computed backbone features and dump -** intermediate tensors. Tests exemplar box coordinate encoding against -** Python reference. -*/ -bool sam3_test_dump_geom_enc(const sam3_model & model, - const std::string & prephase_ref_dir, - const sam3_pcs_params & params, - const std::string & output_dir, - int n_threads = 4); - -/* -** Run ONLY the fusion encoder (6 layers) from pre-dumped inputs (image -** features, pos encoding, prompt tokens, attn bias). Dumps per-layer -** outputs for isolated fenc debugging. -*/ -bool sam3_test_fenc_only(const sam3_model & model, - const std::string & ref_dir, - const std::string & output_dir, - int n_threads = 4); - -/* -** ── Debug ──────────────────────────────────────────────────────────────── -*/ - -/* Dump a named state tensor to a binary file for verification. */ -bool sam3_dump_state_tensor(const sam3_state & state, - const std::string & tensor_name, - const std::string & output_path); - -/* Query metadata for a named state tensor without dumping its payload. */ -bool sam3_get_state_tensor_info(const sam3_state & state, - const std::string & tensor_name, - sam3_tensor_info & info); - -/* Dump a named model tensor (weights/constants) to a binary file. */ -bool sam3_dump_model_tensor(const sam3_model & model, - const std::string & tensor_name, - const std::string & output_path); - -/* Query metadata for a named model tensor. */ -bool sam3_get_model_tensor_info(const sam3_model & model, - const std::string & tensor_name, - sam3_tensor_info & info); - -/* -** Encode an image from pre-preprocessed float data (CHW layout, already -** resized and normalized). Bypasses C++ preprocessing so that numerical -** comparisons against the Python reference are not polluted by resize -** implementation differences. -*/ -bool sam3_encode_image_from_preprocessed(sam3_state & state, - const sam3_model & model, - const float * chw_data, - int img_size); - -/* -** Test-only: run ONLY the ViT encoder from preprocessed float data and keep -** only the requested intermediate tensors alive for dumping/comparison. -*/ -bool sam3_encode_vit_from_preprocessed_selective(sam3_state & state, - const sam3_model & model, - const float * chw_data, - int img_size, - const std::vector & output_tensors); - -/* -** Test-only: run the exact ViT prefix up to the tensor entering block 0 -** (patch embed + pos embed + ln_pre). -*/ -bool sam3_test_run_vit_block0_input(const sam3_model & model, - const float * chw_data, - int img_size, - std::vector & output_data, - int64_t output_ne[4], - int n_threads = 4); - -/* -** Test-only: run an exact ViT prefix sub-stage on the real model tensors using -** the model's backend. PATCH_EMBED expects image input [W,H,3,1]. Later stages -** expect feature input [E,W,H,1] in ggml 4D layout. -*/ -bool sam3_test_run_vit_prefix_stage(const sam3_model & model, - sam3_vit_prefix_stage stage, - const float * input_data, - const int64_t input_ne[4], - std::vector & output_data, - int64_t output_ne[4], - int n_threads = 4); -bool sam3_test_run_patch_mulmat_host_ref(const sam3_model & model, - const float * input_data, - const int64_t input_ne[4], - bool use_double_accum, - std::vector & output_data, - int64_t output_ne[4]); -bool sam3_test_run_vit_block_linear_host_ref(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const float * input_data, - const int64_t input_ne[4], - bool use_double_accum, - std::vector & output_data, - int64_t output_ne[4]); - -/* -** Test-only: run an exact ViT block sub-stage on the real model tensors using -** the model's backend. The input is always provided as F32 in ggml 4D layout. -*/ -bool sam3_test_run_vit_block_stage(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const float * input_data, - const int64_t input_ne[4], - std::vector & output_data, - int64_t output_ne[4], - int n_threads = 4); - /* ** ── Profiling ─────────────────────────────────────────────────────────── */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt deleted file mode 100644 index 8e27787..0000000 --- a/tests/CMakeLists.txt +++ /dev/null @@ -1,129 +0,0 @@ -add_executable(test_load test_load.cpp) -target_link_libraries(test_load PRIVATE sam3) - -add_executable(test_tokenizer test_tokenizer.cpp) -target_link_libraries(test_tokenizer PRIVATE sam3) - -add_executable(test_vit test_vit.cpp) -target_link_libraries(test_vit PRIVATE sam3) - -add_executable(test_phase5 test_phase5.cpp) -target_link_libraries(test_phase5 PRIVATE sam3) - -add_executable(test_phase6 test_phase6.cpp) -target_link_libraries(test_phase6 PRIVATE sam3) - -add_executable(test_metal_phase6 test_metal_phase6.cpp) -target_link_libraries(test_metal_phase6 PRIVATE sam3) - -add_executable(test_phase3 test_phase3.cpp) -target_link_libraries(test_phase3 PRIVATE sam3) - -add_executable(test_phase4 test_phase4.cpp) -target_link_libraries(test_phase4 PRIVATE sam3) - -add_executable(test_phase7 test_phase7.cpp) -target_link_libraries(test_phase7 PRIVATE sam3) - -add_executable(test_debug_encoder test_debug_encoder.cpp) -target_link_libraries(test_debug_encoder PRIVATE sam3) - -add_executable(test_preprocess_match test_preprocess_match.cpp) -target_link_libraries(test_preprocess_match PRIVATE sam3) - -add_executable(test_jpeg_decode test_jpeg_decode.cpp) -target_link_libraries(test_jpeg_decode PRIVATE sam3) - -add_executable(test_geom_enc test_geom_enc.cpp) -target_link_libraries(test_geom_enc PRIVATE sam3) - -add_executable(test_debug_fenc test_debug_fenc.cpp) -target_link_libraries(test_debug_fenc PRIVATE sam3) - -add_executable(test_text_enc_dump test_text_enc_dump.cpp) -target_link_libraries(test_text_enc_dump PRIVATE sam3) - -add_executable(test_e2e_pvs test_e2e_pvs.cpp) -target_link_libraries(test_e2e_pvs PRIVATE sam3) - -add_executable(test_e2e_pcs test_e2e_pcs.cpp) -target_link_libraries(test_e2e_pcs PRIVATE sam3) - -add_executable(test_text_enc_cat test_text_enc_cat.cpp) -target_link_libraries(test_text_enc_cat PRIVATE sam3) - -add_executable(test_llama_pcs test_llama_pcs.cpp) -target_link_libraries(test_llama_pcs PRIVATE sam3) - -add_executable(test_pvs_point test_pvs_point.cpp) -target_link_libraries(test_pvs_point PRIVATE sam3) - -add_executable(test_video_2frame test_video_2frame.cpp) -target_link_libraries(test_video_2frame PRIVATE sam3) - -add_executable(test_video_text test_video_text.cpp) -target_link_libraries(test_video_text PRIVATE sam3) - -add_executable(test_metal_win test_metal_win.cpp) -target_link_libraries(test_metal_win PRIVATE sam3) - -add_executable(test_metal_vit test_metal_vit.cpp) -target_link_libraries(test_metal_vit PRIVATE sam3) - -add_executable(test_metal_encoder_checkpoints test_metal_encoder_checkpoints.cpp) -target_link_libraries(test_metal_encoder_checkpoints PRIVATE sam3) - -add_executable(test_metal_flash_attn_ext_block15_compare test_metal_flash_attn_ext_block15_compare.cpp) -target_link_libraries(test_metal_flash_attn_ext_block15_compare PRIVATE sam3) - -add_executable(test_metal_vit_upstream_compare test_metal_vit_upstream_compare.cpp) -target_link_libraries(test_metal_vit_upstream_compare PRIVATE sam3) - -add_executable(test_metal_vit_prefix_compare test_metal_vit_prefix_compare.cpp) -target_link_libraries(test_metal_vit_prefix_compare PRIVATE sam3) - -add_executable(test_metal_block0_stage_compare test_metal_block0_stage_compare.cpp) -target_link_libraries(test_metal_block0_stage_compare PRIVATE sam3) - -add_executable(test_metal_block_stage_compare test_metal_block_stage_compare.cpp) -target_link_libraries(test_metal_block_stage_compare PRIVATE sam3) - -add_executable(test_metal_python_ref_compare test_metal_python_ref_compare.cpp) -target_link_libraries(test_metal_python_ref_compare PRIVATE sam3) - -add_executable(test_metal_conv_dw test_metal_conv_dw.cpp) -target_link_libraries(test_metal_conv_dw PRIVATE sam3) - -add_executable(test_metal_profile test_metal_profile.cpp) -target_link_libraries(test_metal_profile PRIVATE sam3) - -add_executable(test_metal_overhead test_metal_overhead.cpp) -target_link_libraries(test_metal_overhead PRIVATE sam3) - -add_executable(test_v_cont_removal test_v_cont_removal.cpp) -target_link_libraries(test_v_cont_removal PRIVATE sam3) - -add_executable(test_metal_real_profile test_metal_real_profile.cpp) -target_link_libraries(test_metal_real_profile PRIVATE sam3) - -add_executable(test_metal_conv_transpose_2d_stress test_metal_conv_transpose_2d_stress.cpp) -target_link_libraries(test_metal_conv_transpose_2d_stress PRIVATE sam3) - -add_executable(test_metal_conv_transpose_2d test_metal_conv_transpose_2d.cpp) -target_link_libraries(test_metal_conv_transpose_2d PRIVATE sam3) - -add_executable(test_visual_only test_visual_only.cpp) -target_link_libraries(test_visual_only PRIVATE sam3) - -add_executable(test_visual_only_compare test_visual_only_compare.cpp) -target_link_libraries(test_visual_only_compare PRIVATE sam3) - -add_executable(test_sam2_backbone_compare test_sam2_backbone_compare.cpp) -target_link_libraries(test_sam2_backbone_compare PRIVATE sam3) - -add_executable(test_sam2_pe_cache_size test_sam2_pe_cache_size.cpp) -target_link_libraries(test_sam2_pe_cache_size PRIVATE sam3) - -add_executable(test_sam2_video_debug test_sam2_video_debug.cpp) -target_link_libraries(test_sam2_video_debug PRIVATE sam3) - diff --git a/tests/analyze_divergence.py b/tests/analyze_divergence.py deleted file mode 100644 index 1b57b9a..0000000 --- a/tests/analyze_divergence.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python3 -"""Analyze divergence between C++ and Python ViT outputs. - -Focuses on understanding the error distribution and whether it's -just f32 precision on large-magnitude values. -""" -import numpy as np -import os, sys - -def load_tensor(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32) - return data, shape - -ref_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/ref_phase3" -cpp_dir = sys.argv[2] if len(sys.argv) > 2 else "tests/cpp_out" - -print("═══ Divergence Analysis ═══\n") - -# Analyze per-block error progression -print("Block-by-block error progression:") -print(f"{'Block':>6} {'MaxErr':>12} {'MeanErr':>12} {'MaxVal':>12} {'RelErr':>12} {'Cosine':>12}") -print("-" * 78) - -for i in range(32): - ref_path = f"{ref_dir}/block_{i}_out" - cpp_path = f"{cpp_dir}/dbg_block_{i}_out" - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - continue - - ref, ref_shape = load_tensor(ref_path) - cpp, cpp_shape = load_tensor(cpp_path) - - # ggml [E, W, H] → NHWC [H, W, E] is the same flat layout for comparison - diff = np.abs(ref - cpp) - max_err = diff.max() - mean_err = diff.mean() - max_val = max(np.abs(ref).max(), np.abs(cpp).max()) - rel_err = max_err / max_val if max_val > 0 else 0 - cos = np.dot(ref, cpp) / (np.linalg.norm(ref) * np.linalg.norm(cpp) + 1e-12) - - print(f"{i:>6} {max_err:>12.6e} {mean_err:>12.6e} {max_val:>12.4f} {rel_err:>12.6e} {cos:>12.10f}") - -# Detailed analysis of the worst element -print("\n═══ Worst Element Analysis ═══\n") -for block in [14, 15, 23, 31]: - ref_path = f"{ref_dir}/block_{block}_out" - cpp_path = f"{cpp_dir}/dbg_block_{block}_out" - if block == 31: - cpp_path = f"{cpp_dir}/vit_output" - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - continue - - ref, ref_shape = load_tensor(ref_path) - cpp, cpp_shape = load_tensor(cpp_path) - - # For block 31 / vit_output, need NCHW→NHWC transpose - if block == 31 and len(cpp_shape) == 3: - # cpp is [C, W, H] ggml → ref is [1, H, W, C] NHWC - C, W, H = cpp_shape - cpp_3d = cpp.reshape(H, W, C) - # ref is [1, H, W, C] NHWC - ref_3d = ref.reshape(ref_shape[1], ref_shape[2], ref_shape[3]) - diff = np.abs(ref_3d.flatten() - cpp_3d.flatten()) - else: - diff = np.abs(ref - cpp) - - worst_idx = np.argmax(diff) - - # Convert flat index to spatial coordinates - # ref is [1, H=72, W=72, E=1024] - if len(ref_shape) == 4: - _, H, W, E = ref_shape - else: - H, W, E = ref_shape[-3], ref_shape[-2], ref_shape[-1] - - total = H * W * E - h = worst_idx // (W * E) - w = (worst_idx % (W * E)) // E - e = worst_idx % E - - print(f"Block {block}: worst at (h={h}, w={w}, e={e})") - print(f" ref={ref[worst_idx]:.6e}, cpp={cpp[worst_idx]:.6e}, diff={diff[worst_idx]:.6e}") - print(f" magnitude={abs(ref[worst_idx]):.4f}, rel_err={diff[worst_idx]/abs(ref[worst_idx]):.6e}") - - # Check: how many elements have error > 1e-3? - n_above_1e3 = (diff > 1e-3).sum() - n_above_1e2 = (diff > 1e-2).sum() - print(f" elements with |err| > 1e-3: {n_above_1e3} / {len(diff)} ({100*n_above_1e3/len(diff):.4f}%)") - print(f" elements with |err| > 1e-2: {n_above_1e2} / {len(diff)} ({100*n_above_1e2/len(diff):.4f}%)") - print() - -# Compare block 15 intermediates -print("\n═══ Block 15 Intermediate Comparison ═══\n") -intermediates = [ - ("after_norm1", "norm1"), - ("q_pre_rope", "Q pre-RoPE"), - ("k_pre_rope", "K pre-RoPE"), - ("q_post_rope", "Q post-RoPE"), - ("k_post_rope", "K post-RoPE"), - ("v", "V"), - ("attn_out", "attention output"), - ("proj_out", "projection output"), - ("after_attn_residual", "after attn+residual"), -] - -for suffix, label in intermediates: - ref_path = f"{ref_dir}/block_15_{suffix}" - if not os.path.exists(ref_path + ".bin"): - continue - ref, ref_shape = load_tensor(ref_path) - max_val = np.abs(ref).max() - print(f" block_15_{suffix:25s} shape={ref_shape} range=[{ref.min():.4f}, {ref.max():.4f}] max_abs={max_val:.4f}") diff --git a/tests/analyze_f16_divergence.py b/tests/analyze_f16_divergence.py deleted file mode 100644 index 84d7a87..0000000 --- a/tests/analyze_f16_divergence.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -"""Analyze where f16 precision breaks down in the ViT blocks.""" -import numpy as np -import os - -ref_dir = "tests/ref_phase3" -cpp_dir = "tests/cpp_out_f16" - -print("═══ f16 C++ vs f32 Python — Block-by-Block Error Progression ═══\n") -print(f"{'Block':>6} {'MaxErr':>12} {'MeanErr':>12} {'MaxVal':>12} {'RelErr':>12} {'Cosine':>14} {'WorstIdx':>10}") -print("-" * 90) - -for i in range(32): - ref_path = f"{ref_dir}/block_{i}_out" - cpp_path = f"{cpp_dir}/dbg_block_{i}_out" - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - continue - - ref = np.fromfile(ref_path + ".bin", dtype=np.float32) - cpp = np.fromfile(cpp_path + ".bin", dtype=np.float32) - - diff = np.abs(ref - cpp) - max_err = diff.max() - mean_err = diff.mean() - max_val = max(np.abs(ref).max(), np.abs(cpp).max()) - rel_err = max_err / max_val if max_val > 0 else 0 - cos = np.dot(ref, cpp) / (np.linalg.norm(ref) * np.linalg.norm(cpp) + 1e-12) - worst = np.argmax(diff) - - flag = " <<<" if i in [10, 11, 15, 16] or max_err > 1.0 else "" - print(f"{i:>6} {max_err:>12.6e} {mean_err:>12.6e} {max_val:>12.4f} {rel_err:>12.6e} {cos:>14.10f} {worst:>10}{flag}") - -# Detailed analysis of the jump between block 14 and 15 -print("\n═══ Detailed Block 15 Analysis (first global block that breaks) ═══\n") - -# Check block 15 intermediates -intermediates = [ - "block_15_after_norm1", - "block_15_q_pre_rope", "block_15_k_pre_rope", - "block_15_q_post_rope", "block_15_k_post_rope", - "block_15_v", - "block_15_attn_out", - "block_15_proj_out", - "block_15_after_attn_residual", -] - -for name in intermediates: - ref_path = f"{ref_dir}/{name}" - if not os.path.exists(ref_path + ".bin"): - continue - ref = np.fromfile(ref_path + ".bin", dtype=np.float32) - print(f" {name:40s} range=[{ref.min():.4f}, {ref.max():.4f}] " - f"max_abs={np.abs(ref).max():.4f}") - -# Check the worst element across blocks -print("\n═══ Tracking the Worst Element ═══\n") -for i in [10, 11, 12, 13, 14, 15, 16, 20, 23, 30]: - ref_path = f"{ref_dir}/block_{i}_out" - cpp_path = f"{cpp_dir}/dbg_block_{i}_out" - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - continue - - ref = np.fromfile(ref_path + ".bin", dtype=np.float32) - cpp = np.fromfile(cpp_path + ".bin", dtype=np.float32) - diff = np.abs(ref - cpp) - worst = np.argmax(diff) - - # Convert flat index to spatial coords (ref is [1, 72, 72, 1024] NHWC) - H, W, E = 72, 72, 1024 - h = worst // (W * E) - w = (worst % (W * E)) // E - e = worst % E - - print(f" Block {i:2d}: worst at (h={h:2d}, w={w:2d}, e={e:4d}) " - f"ref={ref[worst]:>12.4f} cpp={cpp[worst]:>12.4f} " - f"diff={diff[worst]:>10.4f} rel={diff[worst]/abs(ref[worst]) if ref[worst] != 0 else 0:.6f}") - -# Check: how many elements have error > 1.0? -print("\n═══ Error Distribution by Block ═══\n") -print(f"{'Block':>6} {'> 0.01':>10} {'> 0.1':>10} {'> 1.0':>10} {'> 10.0':>10}") -for i in range(32): - ref_path = f"{ref_dir}/block_{i}_out" - cpp_path = f"{cpp_dir}/dbg_block_{i}_out" - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - continue - - ref = np.fromfile(ref_path + ".bin", dtype=np.float32) - cpp = np.fromfile(cpp_path + ".bin", dtype=np.float32) - diff = np.abs(ref - cpp) - - n01 = (diff > 0.01).sum() - n1 = (diff > 0.1).sum() - n10 = (diff > 1.0).sum() - n100 = (diff > 10.0).sum() - flag = " <<<" if n10 > 0 else "" - print(f"{i:>6} {n01:>10} {n1:>10} {n10:>10} {n100:>10}{flag}") diff --git a/tests/cat.jpg b/tests/cat.jpg deleted file mode 100644 index edecb9c3520047ee4cdf4e4fa7b7d1761821fe77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 215264 zcmb5VcQl*-8$TR`pmvgeP^&}`9fGvf3N<2)6^Rk5iW)ImqeWC}wIx-=XbH7L#NNB5 zHCwwujjEzrRMqU%{(OJW^Uw3=bDjH~oO55v`+na!*SW6i^}1f~e{cWI0WJc7TwMRV zfLuUsAds7fAH;KHf_(hE{KA4FBEo{gLZV=Ch^W|qVnV`DDd>L^5+_ca5P?X`N=wR$ zOP-MY-y$49Zf+iK9sv+YKvGm#RPz6~{p$jV@&c=XHC!C>08UX3E>VtuJpgF{fRl^k ze+Kw}4am*G$;ATz@$wzH%|!qlTwGk7KyJ>XopW(=9y@Z10)QvE#gK+JVELIcL00AzJqxU%4 zjRD{gJ$~Ym_}lzhPAiEb$wjHiN4LLRsc+Cuijdh0Hj4J5YgfH!TigU_r@-Tlyfk&( z6AHn0$6rSSCTn>RMwY(gvJ_=*!H9RxlTo>Y_tW2z33?%hE0C z-W&0(PZOOJg=&%HF<u@@? zV#r$kYS=!3Ayl`)}5oQdxGe+XM!jseuCeYKCG=?%j|$AjTyqs3r{-a&c+ z6JFj{`!IxFVEy?Vd+Q`z6)L`gdVp#A8ccXM%J5`FYga(NVF)>rP9{ZMH_r&VH8;gj ztg9{uBl73l#ay?IK7YQi)*RqwoJbk7ygZ zBj|#2pZUAeHn7sqXQ89-HS3n~3I71(G?(t$!za+(ZJ-Y;#yNEAslX^=ZQdQT89sA)!1`B)eq&5dzgF2!QKgXx@0~T%+pqE{d{$6Cex-#D zV2+4YC;45geZKTMo~Hd%G-gawB~bA$x&2lz8E%2{vV0GwA!8v^bGWnCf>i{JjN#Z4My8c8fvs z&DZjE#!qVPP&sF!TtoURy-#5Az(7(Q?{2UiS4)2(#S+8r|!+r&!Cm_QVF6mA{kIx0U|Fo@K|K4=dx-*4tQhy{>rLRmcMBG3!4? zK)+i+@NJnexN(OOMOV%; zMFTlL*>KD%!mNtS|C6;_>81<>wp(xR`Y=*MO&&g0$K>syEB>S z>%wB7M-3=7i@&B&MU(V+YAQaL$5)>=?0&gWXOgeGjA+eL<@vJBvIt}5HsalCgaZk7 zVMv#EdgL^o=y;HX22pqSK+66;Uk)2A!9dA;_Epy|w;=ojs0?TtQH!&fz+TKcBHxso zBf$y^q}bmlqcvy_JvUK{8un4zC{ehniMtFJea}8>l&8bO9geotrshUbEbN)m>uct; zO=x1owJ@d9>@U8T^9kaMc1RjjB*Lz7y-#BX%6-E4dnK957Y~-NrAC%i@kUSN^V0XC zg2F1?#&suOgjI9Erph2T>SD7Gb`E=;&qF=fv6^h4y%z(6vmkYX@+%GNUlg;00&Q_j zpncO$#I{SB5+o8>GGVpH>e?d7p2`TRcw8&`drSuDrTeNlYglqzM2q1kW(66{>WQz0 z3k@bp*dLCPk&S%6$4(2_SmYdTQ~|{6YC#KBLp|@~=A)hB_k`ZqoAQ~LQ;WzT23cKA zKBw2M(A4$sB#>Ix>n^jERl!ylwruoaQsK&mx(t^U0`Dlp2W4KENa+ynG)~(8dcQVV zatB9{vS&IWZ@ACq_mdMu37PqBA8cNDZP`4lM}g6O?d|k$_Sn(T$}m+;p35gJRZ*(I z$xzCqXEXW~k^b~0!%K@ARWT*#SYU@}XR(w5S%a|-Y@g?i)+zD`KD9_h0|jl;?3SN% zlgUbxns_^?>5#i`&a{aI(TMNRip^O`8F(f$hv-=+zBir9N+QGTxSza-x|T^zxpitd z2+}PD@x`$`7Iw9?wXzLs0)Um4lt+0fR>aAK=3Q}7onNl*B!I`QRmI29oX}r|oiHzk zwN#|-0V@p}9pVRC=n$APClEI{U$b#iY%u^di+DBl>~pBmA~f|0c-{^Xni5uFr{)K1 zO|(~%n`Hs)om973L-{3o7buX}a6-tU3Ap84~ zQrqNx(|-VGNt4iWv`la^$?_=Q_W9inWO2Zl=ubmFCL@(YB6s&CKUJYjWZ7<%iPEj& zJme=i86W4r!e+9GDLbEtz(d13?6N~+=Aar}1MA#|Pe?LxEhQ9X$kvv%ep$CpMV>MC*U&YCtWR5NM6_gsLiQo?J zD$i@RP8#HRF?fNxTOoMl&m#Ac&EubzH8TU72`3 zCteke$oQZT3$dZyK_EDjZBbwaT^sRKW|IE?R{w0Fb~IFc%Yi#O#`ioly7}pg3SMhz zGZN>{hf|6H9fe}F=sw_t`FhIuTcsXZ5oFZePXHl;Te?u{Cs^=7=MaS zf1iPfHsyErF)zs2li|IMIx7>T{=GDHE&8P)uW(`n`0pPZ2C$@}cE+quK z?j>OtySOy{=PF5RocQx0(2wU|2YHn=9`y3PHUYWe^Fc2Amwyvr{z(>BdLk#EmLb=X zq^e0nkp`>A;6A&(knL*riNf^}pYIiwg#c_!d>-lW;!j?^2ojDjER*@-u zYIlCyAtPjS*xOej$ro_29z(8mSp!kKlm$es86NBuGaa)U7S7T$T{(urwlFb7-FJdp zg#FDK#(S3VR$Mcy(?Bi4JA^51!AJ+Ue9@^6F*D45c`v1=iNKdzi|0X*1>GHyP*Gc# z3PcE6)Sip8K%vtGuv#;~YK4A=N)dJ)JPdnH9QdjOyZ_j6|HW{c5avJiuru#R&})eh-17u`^R9xDiVR7WOnKioj*Q#GmfN%5 z_aA|K!v4v41%NQQRLoLLyE;%^o{6I$cQb0*9<~pR;W%Uo0EB?w ztNC(LBFni|&j{lw_RSm)Pvm`3I)K4KJFS_vr_`YqO5m1J<^+{XTS zA}(U$Tg==xE!UG}i==ZSaZxJ-lBvuT}oVMFm7kjlBG ziVTTmo+mnCxSu^Tt`j>t1O*_9^5aEZic&R6!XCCK`jZje+rm6)96~)eJ44py*Y{pZ zo%1?(1+HfLCFVny_I}B&DSbFL3{0AFNHh(e$eZ<9NrDd9So?j`P1cSkt$B3w0d~_Y z{X>SL@_8t>zNgQicgj<=N^i~HkWN07Cx7erihc4rJ@?-6S;F7D^vt=n-#AlGQ@GE^ zrrfN>v`q{Dy=Tz~nPuorU=p9l^;+p?bWHz#$+=%w4;B6adaeBT#-hg#nX6x-63uE# zF%L5m-oC+L6Z)^C#E0)M`~&8I_mq4OV}YmO0OjMLwZwmtn_!Ain&J}!A=Fy zy0Hg^PH0PIYk@iMe%L5ris_w&y zif7|>9gs_;N6g^k{ia-|IOIuKvHdh!o&Dv;{&F5kcI zvThJ!Gj7DRM~Gv!Lr<=+B#B-8#s(^SXJ`B zsXCP^Ij|f``uMtkZ)Q9>ZRXO=b8k76M;%c`zwXr=I9)TVLcBsETKLTMxOp5e#+4!F zmEcAD&YD?cwPgXAHkansD}VAcLZUB+cD3E>ds==juG_~G^q7+Ty*lqgW!2NT7-6|L zPnGP0ts+a!`7zcpLcH{iZu-K=T?Iiiv-MkS86SSF`K)+Wo{NwD%l|%EPmls-g1}O5 z#F*9onp4T#tPtES)}R!dJBAm|ZPYXf7(f1yL1Zq^)?&@La`*EU3MwEZ&ngh95Hswp zRpIgUu_ZZFrCLaJGIsn!Od!%$LW^s* z6I6-{DDmaD{(z_$@inw_e%yTpXVReidBFbj%F?*jKKwz1Lt_be%Nt6&!3zx2OkOU&2)Wm*4=L1vwh*REob zUsMNT{%{`U^Pm`O$8g(i# z`r^sUz66#}q7QKc`dNv(x9Tk(YlB*Lp(^(mTXLhLQ6EF;Lq~}B)U8w;QLAgILtwfY zJQba6LJ*L7Q8`g~!;Hd(C{$`BYFh^B5TNFp)_k1s8obi=J7j^7G9H#MXxuT{g~leZ z(d)`b`AF1-zKyiD-~vBWeM4oAVl(?S(GekRQwuZQ)e0UpRZ!uBL}H1BU2z~9NFOxS zBk#&IW@*#A3*i(K3X5S1SSoL!ANM^~)L=r8S}i220$S8v315-a;;y4&5Bpw`=&N-^ zk(4oU<_R+W^jsbKlvul$v|e}_=LN#vE&vo>%>m|2s3;ryp0p3lZOs*QV8-0YRY<6y zK3J+vt>YW^WZW7ynx5mSQ{5T;egqfJFrp_AtRN67W&yBk)E+#_qce5EJ@MrAC|;oX zet~mqOa)bRBP+R>V-2kn#<95pIcfq#W1rY+L%`F~a^Eof4Ez7a)kAOXF|5{gJnuef zR8nY{;dnW{h$}PBHJ)gHWT*{enpG;cvJZwbrMrrsP#W&}!+ulEU=*!C5hpIMN9!~( zx1}ajD;+p*e`?lD+9hg?STSI&lllc4x&*0;{ck@SF0@VFF6K_p%A#8YmqE>Qtl;Se z%rp%yFzwRx&GuV?(>1>d&RL(8O?ll8*YN3v;4SoppVoH4V`Dp|e7V9`of^rd0%qpJ z@}c9SPX-hJbA6@QD_E%Wi{0$mNE_4`vNhH0Jr$*2l~wwfWRwbePATO%LJZqN2e76$ z=7Zq)7?s1MCxCFV3np5P<7uy2Khf{{Ft+*3#ndhAv_|uWSI;Tdzmbc_#3u@==GVV@ zu*3WrVSB<~J!lsUY$>&N53NKe!V@6L^ zqyTzdpYU+waHH7bUxX44~DJRbl@C9b;7&j1pzwu^}GWCqIQ-Ipj574Gejg(Uu!-w$@)tuKEW! zzNVrSWMrtPn)>6@m4J6bY7@}_k-%ZBI^O>oEkM_%_qJ`tpLZK^WlrW%d_(qX(c3}<=*b*J;5oK^BFNS3sDd#Rc!*o!6H$SQYg+Uz>x z%xsE&VU!Gw6^!Zpx}>L7B-P?Hpu*+rgQNl2sUMr{&GE=E%|_t4wC115q+RMI-jin_ z`w}^0Kk>~_k@USB`%3>mp@XV6BVxH3uiN z6+a5T?DYMyB{Mq;mSg>&a&w?;7aGrzi>U2+ysV&G*~b8H%6>T0xceB}9EeW;8NPYN zEK-MmF(~i~*FS*ZS?>E4_!-PRlk?f$`uFtm!nm&L(Ef}iyW3yG0<#O#Tv*%|&c^nJ zVnHA^)ow@T$b=PV5ySOFfkUNXLG~w*^hU_vvNZ>lC-glw&b*BFM$^QvW~o4K2f;(J zi?>uBE9T&-l?RSGum`G~$*|~QTn|SVlhfmdFiK1qdVh~}!)ufHdzg}+S;1Y$$NI`tLfJZa zJp7u37*z8W@gSC{N)-9fQ(V#ATMqYj?Rb3of1RUEqwuwvq68(70j1(qpu?b}Vk9@w z)ML^;hi$1Ulp&*px;*Zp4V$t&rKH5Tx_o>X`y!mVZDt6OpV$5epmc|xFtyrp_?)CF zY@5#o6Ydl!kK$O2Tnm1ZDr78EY#p%9Hx#U~H&$OKe#ZE13*Lz%5VQ?E=~wG@x6MCD*kVk7$fw6xvO{T z4(gZ8^D=>_J;D^*o8{<7D|=a0luu5&P@#VS3;Fh`p7JCMi=n;kF#l&s_jm2OcfqT& zO#F#=elStES7&f@+t3j0J#_RPxqUD}O^F&;5+UAU+;ry`b;lBEKPSbGOY5e7G0%IL z3x?hrw)GQO+0!d0#7nOaz?lhyvpVAzv6%(5W+1;JLRzcT6vd;L3@s!M3qeT?V+-v^ zF`d3$gB8s(lP4YX-F;XY#84u(Rl}h_y}~ZY5VcgbPA{xWRpohjkJwUyca^NM6SmSE zt0{h%F#^bBjcQR#C|OK87c9%?3v9Y|#l(c-Eno$9LCFUGWPMWDb?9ZGX+){XR9DgI zf=!41cu_(^`5v<@k^&R9|_s z@bV${5lN|m0_O12@Zn4b$ND}Rg-XO4@8SVjtYHiil^7O;{=`$M2BEMqVmxGqLm?zv z=_a9kf$2P2>DIg`si?p+oW3ZPSpiw>uM}!e461JA-F!O)H)C*q>k)58^O*2|?Bc3C z8+33dLo_AWP$%-la~t^Mk+)a7vSf$DJzVMqcRDe!Y%{IJN`kR6 zdQmn>`>r+C3l@KLm)WK7id>+*BacD&m)}lYs`QWmWUYJArD&g*x94Qvkz@25wii+_ zuNUdNQCugEReC_rj)ZNQpV<$%-4l}wFIh>p`tRY6Ui&02PjgkxM|y$92sjb-U6XN$ z{`z9_(tInTJ40%P?abUt((3j*{d)&a!wA4Vc1e_Pk~zGk<= zE1FFD`lux*2U=qDGQXs5>$UW_rZ(p^6)owq;VSyk544bwukS6fcM9O1=n?@(m>B4B zF73pBB%cXfaRi6O+d2#m(%(|}w)s(Cl7HrE{3+3LkJ40SwkALg-hefO7TTf`FZxIK zFT`VJ^~fR`xxU`r(s5Jpp#yd~h?Bg=qfW-UdDfI@>t|fG$86Z~f`{G30AMJ8mUyLD zG<36;X57bVC0_xwYpo8^FfV2` zr#zb4r)w$vhI)zV9EMG~!+4|Ge-R@-xPg(fX=}kgH$odcKP*qy-N3AMADjF9u>GZ1 zKr@t2QO|cN??3YEXYGMac6`|S-1^I-{j#FRXHTmcajG#lMpRcuoE00ORd(A$%$3|m zsR@VLZwJ3GeCuDgdQ$92KHZR>nxc&)Ub|iPT>Zyh{)9~X4d&VFl^RBpu7X|S`>P$F z=l=oRJFAZEkYCwFVLQe*9NxLKF^qcSFUsVC2URN1$OTrf17YOvB8-691s zFj__JSN&#B6{}3^_D2ar+y(Djqvt&E{8SXqY~ZRh5g(alKXAD6h?)G4&7y2qJtPHfw7$PMN=_t~~DtAcee z{>>)H+6MEm&}Z|lAp6-8j18d5P)n4U4C8Nv2WC*$fNpOeW{=2PLbXZoJ=l==A{ z|MUu*kz#3a=TRg6iB~{jIOlV%+h>l^>qV*=wz8B%X|IO_j<=Hg9oVJM-#Xe?{J?lH z82W))Ly-KcWvc?@+OQ7mCFTD%ad%W+j6oX9VfuOQ=ZBgYsL++>#5sHUlUR z>VQC>cqDg{k3O#?~k zhwJ&bB=R|3Wpf**kNj9*mpOIBZ_Ea2o##tFT1t4Kp6EXcwauY-Zv|Aw^F#V2>fR-Jn_aQ81vOKo=UPGX zPxM0j!iIT1Z}!K$lx~aL(eHcmJPe?uzbD=MO@m))3tM%z?{<$!IONf9JpmW3-i;69 zc=xHt2Fzf7S-O27Gu}QkLDxd5W%69wDvUx;qpi7bXG>isTQdS2Kn>gGRr@#K$8~}+ zTc{i7J@p=)Y@hsjPeiZRd|`L0U&n=vYS#YFIGcfp+)&FnwX6=$Uzr*Elq|-3 z`PIDcU%~Ju$hPjb)YQRa;{GD4;itUbPL4az+I9df=v;i4?n9ydHI%T17UwnoN*=zSOB5 zJxmD5)9eMvXiPB0jMJA8uUPT>qm`${nId6jxny5ONw&J3H^)D~OJbTt|3cmSzel65 zCDRBSlB$&FdN6(1(mlqT_zbw}O?(%;Dw|m_@Ht7YRF+R$JO5o+Tb>@+g`fN{rkB7Rb2tP-Qgb${vUuRb>y>aL3wk} zG8|-cgwZipikkFKHhgK%D?BDnyt{JjUMxt05u_(@N5_4x(?M`KygOGluVYFPhFg&> zNxe)?W&8t(fRV$ncJ^SDou{kG|Ba;Q3K5kKsm!g}w0ZJal*TJ(RM%lVN*T|- zrgB;MJ=L!-{liCfGtH@K3_kv3y{N8DqLZ%k-@9E;9~pq9-T6zV+8phTsi8i_X<}*b zsW*KXv=gbg60>yTvas2`?4j##>95V-^8orUUY35G`M4|jjLxvEf9Ynj${*X;sv>|L z*~7+h@d(|rk3q*tYaM1f5pN5|M^ zS57yasDkEjxt56BuW_Fy2`!~0Reqky8Y{rzt}!EF(MDfDyi#OAIB>&Z7J1PXoaCKe zVv4jDg|cpFRd5zaS(4WZhRL6TMKqf6k*)Y>n)ifJ@vf9kaF6clut(L+#?s6gbeJF9h$ zL)IvnLmSl6(bzJJsU(q4hOrRkVZc|3^ugX2#)~)e&jw7)U6MQMFKq|%vmD%muH95> zzUZTE?>F`q-7fb{d?Ql;K#k{KQDVr;!mG*6`@R@ z;VBKz`y(Bk%-@%!VAo3Ae|eAmIJNq``F<64bw%s%>%bOpY*PaJI&sxUwI_Vi(8+xF zoY3rUy~V35t!p11@XiVSOipjcVEG?MA}?j^HuI@oNdc$z%3O*uSGFq9D!?zV2fCc% z09EO!1~%xDaEwN3O1?`-Qo^k9N& zXTZ)55$kQNH@94Ty4WX3`ebwM{s8usVVBiz#MsBabm7;WdR7$PH5ZNS9DKbc1IALO zCpxv(eR3qdNWWLiyAm)QQH)c$6q#e%ONI6J4rRS zeR41-HgA>jpCru&srQrFn%ie^9QoyDXgS6j78y6;^b~WJIl3EoNxBt3m{7qRNS3}E zj1RhwX(&w=Ls9;eVF3w=qW!D)=NXAvi{hHLc4U`x%M+PSYxUz-sIN}v>fc>LCL7H~ zK(!13qU%1xRXH~}-J@}pnj`JRAEoE$SG(_d#P^cae(HxB?%R=Q5f7`C^bZS+w(2zc1UsBrE5sHVR3>c{d4VmiBFAl2 ziP3eZCU@9yd{ql)B?jZeWg2b!8^NNRF4Y1S5!IbX;)iO42mRTa1WS1-ivQ*?MooK= zmBv}7jzoK8pCMX9wbmRnM29_{lmB`@PSU`4`~y5kQRi3dF86TJMMHI?ydB4-43C9M zjXiAb9@sSM_#s9J-a_l$zTG7-v~4sp@S$cp0L>4hy@7OYyanmvKW*hNgYr;_7B(OY zCQ5@;K5h9X@cvoCz-lE> z^B4IAW2*K8`FC?8CQb|!BzBd>bMU@TjhS5v`sqn#1rK|t`p;7a4Jx7({>PT4oA1Ww zmSj8+ZIPgI_cxCKju_7W2PC34dPPik_bt5@TZrl&Q$D~w*!bgv_9UPc<>}ij-S}lw zIWeSNz+^vP_h_Ky>AfY)i)+rd4=v<|Xo zLVT>5e9qD1ZNeji94=|sH>&>nd^)y2N-caAV}E%~C3$v@9`RT=@u`Y#O{d({K#k`c z8?kOYVRU4+MJH&#D9cUWR-Y^%Qy&15^(o5q1aa!AdZ-6SJDnTH#5#ad7L?=URR;|@ z8?nCpGuG5?3qM-3ZZuIrVzs8py+MkD-r+HwnnQ21mG=dX2KaUZx!}G=sTYX8oHJ;8 z%Tfmj1$O9R?^*lEj5Xj7)52;o5i%Vxj0Kf zyS7bH>{lx}+eC$Dk-lgiG6d+(a2=;T>ggHVx{zMr9Ed8Xav+8kJ3D4GfMrUrXIpnx ztnF+eQZt<&$9ytGW2IHI+MW?79%3@C10?n<1FHEq2Z1TU^fw>;wVdZqe=NOW-m_mV z8LqVAVVzt*-Osxio3`g4HoG@`syn7w+~4aANHck+ z+1~@ragq8%@8dVhCfR5-=kTzzynP_&Mz4Fq=yLbRv~1;N!Q!$|ew_#+{CPm`PqoX}`J40ZBW({SwWF@(Lfkg{iG z6$ElESeMXTnra?IR7xnfXPqP;3}+IXei$LC7bG3{O~q(NCG+ZqR{&bCy(O@c{zfR^1(N?n*ucO~TUlaCkk zVxgnwRi26@x^jL``kgXC0F`~(o+$d0)o+)fTMvc<`AU4kA8P*OBv>J`=vTs-7nY(7 zoneatTt|vTOVyy6af#t|rQmN0xY_qjv#w&UtY?4v{JWFT)x&v!)SISLA8d4|*L1j^ z&C~_UaiPBac%J|5PtTO8k2FHQNt*?su4~yi^ioRw*kAMl9Y*UO9H|W3(k3baF{&?u zv?|Vf2-sCds|$L9VZZE{IJ@J2FEQZS)tD_inaRzT+&v@0Bzc9%8|@${mX<+bpGF5~ zu|@<5uw2j|XqFp+Q_^2XI<`Nyc(9n<9p^QS^*qLl<9=)ThG@X|?_@ zV=XYlr5f8Zu%g-6*vLFE=_u#51{tRYqBriJ5TGn3Y)IihRTb?+l3qNZd{fSZd1_z- zsue=B!8v1oM-njP>(AQOza>eBe`1V^DRig6JHYW1+6E+Fl1+|0**H=# znz=2mnY3k#Nl6XQ^)Xqk*6A42PN06eelwasX}i}5ua~HQ;?9FmIukIM7wr7*ptd;# z+~bz!vc9W>OIZD1!Bq;)a^8dT;SRq2U@1Htc3y>maFQlBnarUxUagGe1t?fl18G#O z@&+NQ$8A2e9(*4r{&6WotxWGXQ3O&yqOHGQbo`#{fW%jyBY7>hgd+``=9Rp1SHxz5 zgFK9tH+1T7V~=|6dPyAL=-W5G5n5V2%v59l`p@2e zN3zi0+xA8{PPEKqjbR#`++F7>U}v1F!AV9dv8STtf;=l|#%v=eV0iNTMsKkQ`>3S` zs|jE6YUqckNZ)sdh&m1J5{zK(DvRi%pJ5y8U^}_w8fjx=eM1$HDmUq2i6bOv!+pom zDo}+`M<`yc5FRCf$@Lw`Eo@NOa+$pl zeCZ-Q%q~GLV2`Er>h_C@9EzRcmlpLPx8mR-cI0Y@fUVqrd9zGQOttxEax3+uE{=eK z5dv20Z5GEkAJM_6EbEF_hw^o2qBWFe6T+5J_(ivRr$T{Dh|Ux7jqmyMEe6HiTbP>{ zMey&F8R%6Fqt+1EAO<`%mOTFfU!2zo*k4R$ns8N#`{~Wqzyd}q0h?RevSec5U8&T_ z;nOYbSA>eLr2855%|}DX?_G-c!c?vO;_nw8V-^74i>EKN;dNi>em{!Xaf~jPw4jp0 zbJ@hdpioLm4CE7#4|3G;JsQhbbF->cXrJQ$n~|GFH^0eDCGaO1t|R<&OU^=qZ&rvR zl`7KA&orVCS1&4bo}%ny@lo;@JY#KHgzoOL)~DrCa!r!^Um@lh#sFbW$*%-U9#yq_ z%CU#sBJZ;Baf3StjFkb8QE_oOsE}M}DQYk+!!Xu_}nIAgy@wIA7)trTR&+j8Nojt*=-dr0M zrn`-M+aSWhsle1+vpI3R-jzq)idp1XmlPXc@@O`cFNrqMY|>H)I3nc@69cI#?XKWd z0|?21t0ZD7;d%h}zAB(w0FYJ(Z^qvGrbqr@uw~m-;U<>(53t&V;fi+L!DN;n%@rzC z>U>{9ZF();YT`18zHiszoel%(DOOkQ`1xdPQP{my*y5CA#ZCrl{lKjg`Utacdeo{e zl<0WYoT@`$dJb4cfJthc;jrXbaVZEILK_=)lghcF+C|HvPq`*#WIgR7imlO^b{{#z zDj@#vXR8Dfr0h<5G3hOKyj%;>#!djcODlX2XBau%+177o5svBy8k~&MDvg?VJ@ak?sRsZ7wwBXXlmDHn2uH4!k}( z>31nB!;0GHZe5K{R92~Lzn5_eSm?o|pFZbTkp*7o#!qA0ovv99ph2P}t=TC~vkbHl z#U{pk#JBi`|3X8vqB43RM_i6(s(>dYyp1c~1DjWdY{Z>no`VP%|J`${WU{cA-0Ms| z+vHa|#mJ`!B=5&l{3Uv2qzad|^m{FFx#m)ncTBR)w`CL~(FEJw zOv1^X%bTVLi^|32H$L;Jsf7Ls$LF*VIQo`Jce9T(+CABILVV(X_A6~}?7vmHV^(O2 zxcR!d?sv=@m)yLwcE$_6pOy3N7h(BE0|}y#;cOsJkag;vzFO1!QP@A|`f*gQ%B{Fx zwVlxqlAKAPHy^Dk&vdXM_5fMDj;&M4#fyF!LIqrqR*X+aCFl)QqW7uzN^bu@0M~rm zBXfqRqP(sP3M^{^>k4WvZ%AYF3a;+37q}t)Ph|SR;s7~z&&x4%eI|L>fME6KJEN#MQW)Q&T^?<^qM36CQr@v z6_zbncPfFi14vB6rNrGJ4+59zb!d$iw+b300-0d=mo#jyXKP~O=_C@VJRTimM&3yj z(+MQepwu6l4Q_DXcfDxqP5nj=v)L%`mJ`7f6*^05t7l%QtXt#>1B)hAmNdD?Ll2u*vdKmuC`XqPUFC?1+`YB0o#F zd`rw}1xSrD&QiYcjm}2h`QguV7XhjLz2l{@YaT)o|0Zyy zc7J^L5#-HG`(!}R{@=js?bIjh#0*8`2BsQin2C=Z>{acUoOn7>NC|dsJZOyN7d2qA zbCR7v_gDI-+s3fORbAR}a5H`WPOa94^WoPzVhgv7`D*L$PNM1U^>}J}saflTmo`op zH-LitpU@H$xs+8~n)Z=b;;XW9^UTUm*&c#jW?4H*R3|d4w3) zvwHXRYbW4};C>X5>r?O%a~X7?SU*86dR34)P{%x#_kblNTlV%k&{Jt#WSj zEmCjE!-0W4>Zk$@6`0|*oHOnOx3)+ypK-FCMb3SNj`14kk1sb$V~h!7!xDHU<`SJqa+UsSo< zz%a?8>L_7NuF-gXH@d)jf6@!7)>VPAfcdTrj7(O3PCGzU<Yo7|by`;AWU8u19?C@tI2)B<6 zi#=k&4P!^)>ORsVRUF8)O>o&;Xn-55EF`!Iq{0GdeVVP)8jEQ?H?S3W;Sa> zy)AMf-1l{Nl8Y%;PdClssypg*r^Y3nm{6W#*}yA#&kUxy=QeDj;%HY0KR(%?gfASO zFw(aLHQrXeQs+|GVjC4>%CYIPmHK1(SD@~52^Eh|3VF>At$`5(u`o~WkPA-!8mS2< z0#S_nQL{0|GDjp6WU_D5pA%oy1~iU!3DCuFKAlnT-z(C*{N$S>-sXKFYrtoE3V$CU zyElb>He3BcGrjO6(y_5>{JRM zpBp+f;|_^H1OkzF(V$b=C1>aHZSI#Ii8-T7kFTVMxJ}pG`;(2Bje>=ugq|I{`@#_U zmT3A|!mo0IkRLp~eep?*qVlD@U$ss?kcMB^#x6VROxA>Oq3ZD8aEivi@fq)Qmc-Ki zW!F*Jp-%uAPylDLOHA6O+LrTUACD(1Z{g#mOMhE$HKZCriaLUH@YAR#z@usa%Wd3< zG)%=luCuZoEVB%xUO+Bojjx^C@H{WdK01{6{p`2FrQ)diRfN3$vjKM{1sJ3Jcw$O2mdM|cTKbKbYAYL ztzid&>C{}x@H^T7>GJlTo|1-QC- zJAQWSc2}zA#2c6MgdM3o7rDLiu$P1Hxc+<~y(T@y7~KZ+>|#C^y?MFpu(oLo@GzUr zcwUx(%xesuCY1rCmv=D%`~U$QvYmZ)@!> z%$far%H=}}3q>l$5n8tuMfVTtAeLWDP5(y&aHdyZv#-y zU3v3Z^W~#K>s_-WkY#q1^9OPf3)p3qe}HX!^J@$bj{fi?b{^{$b3<6BVgHdPS6o)+ zQrmHjk;jf0Gyf9fT=%IQ(%O$vgMHhFk(zPkTwhflbw>TQ{SL(%pE^1|MjK4(%O<^z z5a{!Hcn-}st5uZQ4EO-ckx`C%F*_8kCZs?|9{+<;csuS_2k0`O&YbqF$^Y!8HGgNH zdA-<%kIaMk(8^PDO8WG!-M5X9=7ggRS#j*@s%J65kmm@SiI;Jjv*L~6R$!aJMk zQJEEzr%@p#UQl$dpkF4JRw0o2%~ZE967h1X!%(4x*<;P8V`7K298Ie-Up>qM0b;Te1=4(p`)ecb2U4E{r z^Al`;NW6dTDfep)QfJmB>w$xpv7aio7s@Yx)OCj|w#vVoJ2EHMz5F4UcW(J72HAT~ z7~NEyrbAkS5}Bsu@I7^Ema6rRkR$ zv_Zs$?~b906tdZ4L8x$tk6HkK`9&_*`J+p&q2x2U0>Tq+Or?7{ni!e=gP<9r-T|C^ zO${xV_Y`@?n=3BDvL_GjbuatzQ~f36!5#{-e8M=7C5o^HaxJ6oEio2EGcc{yEph^7 zYm=stQ7a=aq@>&AW(N?;yhE(ZOv6l?31p7z9P>@AFCHbuooiQch4AFon{t`JV;uIj zh)$2u{Oy_9)(%<9EAc#4tXIT!)TE3apG#`OT7=u=`2tyWM3qKYjH}J7>BG(tvINsR zKEqHeQ0Qk9U$D}T7O2O_BHzzCJaJ9y0HJe5d-V|0tDUx*+CFKd(I6(|Coo7GVUaA_ z_y;8;dx{n$_8_KGHIKe%_=BQW>!kjsElRGkQoC?);W-K+CSgQ?o)X%TT5s|bxHLFq4nNs%6I6BVbH2J z_mBos2TnF(vb$iV{&GV%C0RM6;p^%KgC=fH2Od3xqmvaacH+)C^$c1XeGzkz39kiX zI|)OGXhtCBqBigA+^_vAmCe*9ad2syUbAE}GJdpyCwE}oP719dbP#pLjHYjy6^10Y zsC$=uGvlsuua7#T=Hkfavha?4nhG9qAGRbVQNr?S^!RLSj;3gTUwavL%l8ysZPU9% zzf?Ce7NIW0Mr-r_@F;sW8>95k)!cM$u<7LNpJ&c*-=Z)u%s^Mj-_*~}lm}5+OAl*g zthFX0f)K@-ltLBS+n$q^f7_fT<1bI*8uw&tMD9(O23rsLJ!6&q^J0(FOL1tM*&u55 zp73>b(<8&UX{el%@Dka>j3aeePJAy-Cb>eOMo|x2#|7%fojmPp4>IPq zT_1`-A5}Zzi<-dXWnRW%jf1$X)eg)x*RvsE+kfBDKddwOeh0W-+TIxEEor`kr{i}~ zK0BLo4&_2mHvfHAyoILe@MON0nQI9$L%g;osXy$#OJ8~PcY+vdIUD?TWtr^3F3Flw zW1;`gXNTv2K&@k)oAw5|8@&VLH>ywHoPtiWD#eN-AB@ED@CorAlJ|?3$kTtE(EAV2 zKpbnA?o3cit?+*!*O+jfkEJ)x)Tc%9N z`43RNZt_i}y-CGX^8s5Hj-+5#QY=!`n`=kwPiECMJL+trw6}qD0jsiiGS=hYPcC|6 zGFsYo2KAUP%PT~bUVB8lG}9?FUIe0R8-p4TJxrqmXIbvXu*l1(ofI{Uj$p(_%T|{{ z>;D62j6u~r2yl`R$7SpcHmgPNW(_X%AK()5m~BdE+*6p5XcGS&Lh5v`rTA?Q@&nht zFly7sm1)_#8z%*IZ@d%8vQ9`mlb+2{kw%Pxw$mw*Um+B%oIR#ekkeVf^~ZV9jr^g= zIawxBss!X{Uz!5ja&k?Lz|sFO$q09B~54+#8mx!Z6HSQ z?wf-d73g*fa2Da?3kLqjHfLYkVnta5SWE+}*4}*V{=>C%QohTaA;?TfBdtEi!bE5} z*a;tQb3j7a=*U<8Df`1NjhUZ*zdY2^TY`(l=R(sw33?*Og*7TeiLn2zIU9Cy{tn;j zMb8ltlWAIUs{DGx{BHaRiCj@VDgXOe+0U8RNSuvxr@Zl!b#846%7bt;;6e%={0Q5D zma)5&$czNYt`%-t9>p4Wal6*=P*SiMf9b5$o-n%BN%`F(`0M$7#}~HJa9%y0%Hj=o zk8EQ971L8SA-qsdH0;>eTP~^0QK?~qWB2f4vkjP+fzu_O@QCoNieb($S}dpy!8+)1 z#Bm?5b{C1Vj|N(#DR2IN3U{sFl2KJg?Kxqcpr64nSU9WP(<=Mn-SPu3Kh+^ek163r z8rF^D#YNe9*6kN!BvgM*Qqa+kicA=X)y`p(3%6*hB1PD1w@xW$b1LHS7~Sqs5J}ZR2WHE;sA`ERZFgwGP zk~#4Z3obR%nh`#YEk}znnV-dn_3HQl;mgFg)Hg{4vq7ZRPt_4D+PG*i^#D~~X5Xb* z9)%t8iDn>H^Hc+-aLS3F&(iLf-<^8GNA^`{kBEqqk5tV_}DX+)tmyP$PQyG913M+P*9`s}M86et!5T5>-JB{$6Ow!&%3pDVY2@ z0`u4%$MgsAgS>rw^ZmZ-tze`yusQc(^?!gf!UD|THZEqPpVL;7S+$BMXXgVv=QomS z0^aGrxcljnQ|ZQ5LeMqOIr#z;=86MI#j$J@Z@RZ}A{YvSN+EKcajRzYy1eAxzQZM*}%t0E+a5VvC~Oayk6Dpe98JcNBirU-A722$5yeA`Zd7> zFQ~Qi&=pZ#m2x8RlnflZ3cishW3EF*Tz@t(VFlU~gbAI3Z=~6ZO%C9pk89Fdk@JUL zMieYaCZ5VB_Cw?F@dmNLSzYZpL-+0>VtR2huvnpxUPCoRE!7}=79QI8ARRyfBqf${ ztQPUN+tVI%iU3VUyVPs!`tNiCj+#OPOU21k98zXhXc>XljLHWg^=CJw&$}F05sVOX z7s!UJND`cu(a@EV5(To>b8ei3z>4XwUtoBs_oVb@wa$s($4Za8o8~X%lF`oWuCv*6 zRW-AJ`EW$yCSV%>vsxj47-Y5UGsX~E=e?zw6A1$Knqq_s z)SnBqD?nMK6lW<}iSy`bXKl;&suPIKXReyPcji(NXiI~yWu}%DdUvScDXy5yPon}1 z0OO51S1+FrKE_KL)%wM~6?JH_q?hKI+Bf*PM}bVN*v~U9AWba<1vU-2n@%&DQ_1qHeyHxj z#tZKU5B{JY%8@*NtZuh`j4oB6K~RG@Yk?HGIhM;2+?Ah5;geAJG20#q*O?0LF#!`_ zmz91PPnXLk3k~P(wdK@{5!w>CS*5i<(*iZic&|`&{4Cr%1BJJO{bY4wi_jA|D%Ow> z+UjS~L+-T*Wqlx|xyiL+Pyz zZ<=pL|BB8TkpY_7UfxUJD{na)_))_a;`<;(Iitl+th#kXNP>zZneGsu{Tn#T5AG@w z5}bP--{5x3<z$Ur{mTLRZpd^VG&kxG!BmmN5gWu;>xjqs zUbCgS;qb-K->20XiCdQY;27xf&y%j;`>g!JLd+W3JXeq3-bLHLkkp2ZV938N!YE$* z3u|3fL$8|eiwjkxr`VQQ^0m($)bZjUzDf1-Q1)PScuiDzC~f$?>nI|$UhjpyoV?HIVVOz>P|C^UoKAnhVqD@8SCmI__i031S z5(-6!1cz3a4=>AVhYV+~{eJmqy&XbP4_h)yW3`?%>_+d(a=Vj#?KD?03*GZ99b6j) zTl@#u1Kv?qYvcijJRDjJNe~jFWuFavg6;1(Oc|#W>R)B;OUP2epw8Z=iM=j2_*ajd z42MgN#(fQzXRbI*4@aHOsqq1$JdS&iqa4T~d`+WS>(e;)NxTn~%~zcfeTU}pJ?^Et zT@%!w@Tu1h5jq4c8Pz{#H8Q2nrQ~A)MS&)Euix{6zfwvXn9=U>qt*Xx$y7U*eu~jm z?*`SPuu7SqD7fpc6Emeh0kr)*78C#1n+*V39`hcHN=w@cr=>wsYdO^Ia(3PtL`%_z z-;H#yVYwCoe}6vpNnbppW?mn_07^bHI_n(a)etcfz;3)FO05&zC~9*CyoUT}&=E>{ zAlkEB6`xL6WLw5`vrHCaoL39JWG~O^D)^FcYFyPY51Y6!>jEkly_1e-*%SQ07HjD>#c`Lh&U@hPYzb# zNdd>aB_1b+DP5HYO50&Z)MhQ#I*CcwoC7UK;|4^rndXLQrx^K+6}K$Ns6{f10^b&nbU$FPc$41N`^JlsJz(`4rnwY}5 z()2UArke$#tW42DTt^EJV8WMZis1Cy4gAC8YGuxd7~boDh157WigtIrhUC+)GryMx zotkDxrG5VOqdTqQzI1?s9`u!*+G>G~1dV?#JzEy-(p%1$fCIS`=60t?7IFb5LDN45 z*JHzo$ekmok8%phajD0c=AfJSIy-57d?B^NGY_3Gj|UPS{<}m|f;Sd{N1FbF^QQXt z1CLa){U)!O5R58ip(3t!(wEoVKyvk;;*KV+zyjfR@u zO1xN0$5g)8&cU;01QH(Q`h3w3rn2o#p8b{SGw%z_L*^2Ziq4k#xqI4EZ@-81bpHB zM_5)y>FbfV&P~K@%LS$$lS+rmylB6*tU^iauM#X7MkeRSr$$!GkUvPhQ;e&3Z6OVj zhE^OQ3A|r5lc&3#bk_NWn6}K+u&EK4er9CZN-f(ZCYZ_kdS_OOlDH5oIBDhW5V|iD zXz*_N`_-6iPY3U`Ymh%w-VG2WKg%79kmQPu zZ-`7MwidNgbV8{_0A(Jmtn{3lC`M&N-}bQjUk*W)8MMP!n3yzi?^nl}R--S!B zIfJ~;eNd9e22>%CyteK#*Cx(Pvs> zG76hCg*fnx$=Sfz%J-T{c&ZG4;4>!C6?5e0TzWE3t?yoQw57e|J?@gWoVN{$ zD2D0ISP!+HlB$`_ac&9){TZa-FS4PbQB&`AKjwM!_zXTBtrHlaImKL|x)R_lPRTgX zi+(-rFuA{$`C+SC*)c4|WZjkDR(Xj|X9jZMJQmg`bX@YTa)51Hy|cHHIL%Ef_#>W5 zAOdr-Wtyn{AzLc>E=ZtNX;vnJX`PA9~0taH?mLfc>ltX*2n~+ z-Hg6=2Z^3As}7p0y5DcnzO-Kw+`E$>j0aU#qDg}bSjonwgKqe0zSZ&ZhO1pk3{8Ww z%2`BL{sU}edxCai`s%yPwq_;!h9we8w* z+tDpz2RE}+je~bB4MwAE{Fr0R9QFq| z5)|YEpez zda0SCm`Cp4Y3dBb5(Ix&_K|vgDKF#h;Fgv}(>?QZWKb0srynI{~IMHNWv63^;X}l~y(BJT=i+?F2l(g_)eS@H5BPd8Y;B*`e zLbocZE-0X1q*MzHPyslwdIJYd9vjD^y^4+YVnMG<41QplbJ^0=Qps_9|A>sj22gN@ zt=~-1q}iHsIc-@)T$9H}626?8NT+4hm7-UV^EVg7@49us{2g$%fv~S#6v*RmD{u-e z+t}>O8Sdyp>9o~}lY4o*8`dT)ToycT;LecUw?2i}&%SwlH}W6?myA}=Se&pK9Tp*W zt1S*be2e_oKLU?oY^3_pvX(a@oLC+eW7qTTx1HedEnkDUWcZ?h|@QVXrG^B>_2pE7dzouPgegjW3WDk(su)qepQGIdQz{RuE&-R}7Oj^ho* z5fFvelu^4v#Z=e07CQn!lDmQCMXQK;*(^BfCxze_cb~Msp(Hy4dIh7&XZpM^Ie0hR zBWHibfw^=IMfz4CMrM>h2~D+oc8ZJtAYuRf$3VR+i64w8F^0hnk0AFL6vDNF8{;SG zskTQ~Y`VkWJ=LA~i7k&+2}g3y+5jB>?e(2*`%Sk;IIOFMHHpn+{;%h;!%LlpxkRTf z_{)w3_H1mCd8-Lxj=pv|Q70I;;e6~s?>Trn5Mc~3r$qU-p}9sFf{SP)xB1wJi>IW4 z73aewn?1VN{X@qiNx&Fhdf68kabXepgwOBp#jO1bBj60oEwHPCfwwh&z{ zUlg`-!gVg};i&azm}d>pUv|fRyC%gfD?Cn6?sgf*C@kX2gAVq@MU$4t8u(kJ$sPs% zZnhhJ&@VNvY(PSQ8tPIdF(n_CS-U^gcMgNX{oQMz$7;RJ1^d5M>yC2bQT+ytlvC`p z1aUS;G#;_sU@J$&Nie#6eQOCtx%7YXI;?3CK(zZXR8cz4SfVi`S`)gp==F{$GfquM zMh9=?z^HACxJTC@`;O=gD(}lh&7bG5E!-kDv&J^EG(#dNH0zT`=zg7yNUw=gxw!i@ z=gn)NMVrC9Vb7Z$`MjsnRcnBNDF#t-TpTrMs)J;TB^+OAuBrjO2NW1nsj0^Stkk8# z>tPoKM9kxsFjSQ$ENG&Q`ocMZE(0E8V8WNw@_$x!aTL5mHUKqG!ko|S4eE+(nOQ`| zVTZ*8J69$yIq>9kmRLkK=DP?Tgau&^ScHJ9Fe6&P3fZ-G3oTUyw>a2H7NKI1!qbIx z*o)r2OH77x$}}e*WC$eiDhyVsjQm9g&B$Hak!^97jSlWFl~+Qu=%^c#$oMeqJM{Il zPL=GjYu`8u)yQZ+%^2;o()5rBl|B6qc@-WAF7~%PNn~7a?n}_Smg~A%5RE(4Z}uG~ z*B(v->5P{H%vfSn5&s)z@}9f#26fPL!XK1T0iF|eRex#<;O?DplS{m`0KowJ>vE~3 ziT8Q_(S7Yg#)jOB=}A7!FqzbCEZX z<`IDJq*q!!z}3NF)|Cd{c8&v)Oh?^}dmorMpA=L7QcG;VZkp8TXfT;uFt<1VxUEdaH7k-uV9S zlgFVgzs}Thfd#^fm=n39PV4p$F@6$jZ5jCFmN<$xOAY>vvKXaQm%{~x-a8N}N|H;n zwD3*s$NN+D2LJBd8g-ZVbdB$6F1WZQN@9af(yn(u9EAb5bKSeOyKKV*MDk2p(a&0KXMl#t4#+f$k3)=oxk?#S#FLk9pGUqg;f7M zfiDj-){TRC`gU^1za;JDy{H3w5ygoDXH~H^c8tw2JsK!M0n8l`9H2ze^f+h(&RR{o z=dPd6$!WPA?XaFJEGFA7p~lTD8LMfQ0A^_N^~aY>ndLHeX3qM%ujcY;kW^FF_cb(m zNYsZE6KgL+A972nKI!CrhX1=+_J)`*<-5_#JFh6I^>VqZn*6)RM*Lc1KnGXNMcN|U z;(p~MQUqAtQ>&@RpPV~%{Ht)i>iozgBBl&u;GDfg*S#Eud%A|acu%?xuU#qD&VArm z12S(5J~f<)Ih{Wc1$Pwj+(DtvI>p12Ie!`*ANHl1{>lew6^hWwn!iV)GGi1cShK!> z)|XmCRo59-3`_hcMoG*9huvY2@eN4g%)NPVbJ}mBQ89ci#xl>Om5=tqO{dc~NOXf8 znMc2?ZIZGzEF_{wEGUng?OyfTD87`xBgU2&|B){xXWc>#Da_tH)AYgmt2g0Edevmp z|E_h@aEbza_M_qR+LWn&u%>{3l$gkl?b!ZXh@yy!`Re<3Y3e!`FTJR|_{zGe%|{=|RJ;~)R+m^D_OX{XY37fG_A|Zow(GaE5-uo4;%L5S z;Y9F95pl)}aq3Qm%^GiS2A)48I6`H|yWTh{M;2xKLhJ^&fR~JSAe1(>&MFhBHD^xOoF4OkvS-A@Iqxrqju8=CN*- z2^q;rDbCijlubEdP^i54Le`MI)8Cr08h`I#GU;ko<3i^|KIF+OdPQ!yHcaq4+Or=_ zO;e{Q3@Q}NwscESj^JY!`d)n4-)WtyIkm8Bay51 zE}4sv9M6ePt$M!-gdQeZ*QZPKdLfK$S>j`jTY#rrtppEc0B}J^diFlnCDIMJ-m%ZauS+{#^FaT@_s~9 zC4-aIcQ(EmBp(os1uATOR;jSB?b4ZzP@Cs{t=~~GxR#W`8sM5~b9UINrye-(Z*#rV z)(2-9K7-o~N@_njv$>cIZ=0aicPeT!frE)Wk_55HPkY4$TQDvZ)mCGGvBntdY@7Wm z$-9;AIh!3X=_^{|HCI(4CLQbdbsp?YC!o~k{2{GUm33(CXW4FzSSUL5^?dQgv*hih z#?CHgDkDDw>SwOF(zxqj;F9HMKbBqx2%~3t$KvoVCH47ikQT>f-{9e#$2?3*Sn%Y` zmM$%mXtjBqiI1oVWKS!tJGEJ1taXQAfil3)o`Xo0DaGVB= z@VriQc~^Utbj2DNvdgeK=EV_{{6nO*n?6z7AJ~334!cd~(g{}{bFhlFju~@{540vg z1#iq^SS;BPqHBZNGv84H;5JLzSn@>y2TYGCdkax4on?fflLQ;K-J{%m7~{moWu8@R zMs#X{6&uA#{|A`$DQ&|JN059K*Df7|GHIi}tV(m~xRrVeVV9t$tNSYFEuj3}=IqBymfqMZOi|(K$8TkwLHYf=R5ra-doF$6PR>;qayQ}cxTLJxK!cXJm zfm|0N)FI5)J{aGh#+r$Zq+L&ex}gf$Y1C;wI{jV!dXzoZlJrzi`5(NAKBgB%%|&{G zL?(0Md;Q?AURl<>dn}k2b&)~gF zTUI+=!fLxN2~$@RIw7<^NMJ(FJjC;@zfsGtD1O!8^fC4#ahGICvWBJR7l~QV664c7 zEFvEO?Q)}W3f^HEY``h~c3*c0s*6JqkO{o}22)@$8SCXF}vTb!UZtjS~ zGh{qJzhe07weLYWuvL>NlqSABQD|H>J{2>vtiF&>OhuaYUeV$aI!h2ZbNuJ^eTh$9 z37MH`If3En7-E(_adNY`jsCuMSB2%M=ahD8o;o~n7@M(a)F7oaH8=hhhm7{*mE7@r z<==8g?*dKKb0KYi3@~`hGXs(^Mvo2-e8FJ_Jd-t&h_fan+XBF-v6rBq;owt-<|n)c zMac0z$LhJe**vQMnH?{nGz%qcNwfRmoKSX=+vE6iD3z4V! zJ=%I5B7TzGq7d|;F!LD3)lbS5!7@*gi>>NvJW%iRRN6!S1wnDd=0M3Dt$+F5%PqQWvi$Fw@sJ?4Hc_z2QRh+GZ zG?5PZsNa*t4_SS4@>OnM+7HSTt#7C&$E`X(+yJtl$?)LI~u7psgDYU$;T|W(hOd zI~|_0lG-cwz0Xbe9PT+U)ErA}H1-sdI$Srq|1_>_`>f`nJ0^C&8>Dkx5!=ao-yYr6 z^)XnVS_q%kInv7GKC1Z8fCN@oM}TufZZV)@1T#YUVo_Jpi>BeY%7KrPhTo;XmXn5FaXQRD1UNkHzCmlMa&yRS+eEb&!U0jQvxc0i2jo z$jGct&35|&9N&;n%Kt6vu~1t0!ptBJU8l#HV~S+idk2Ck2~DYDt0>WkJ%l!~l>5iU z>dYgizUl|n%#|97$Cl`gx2K|4Da$q9Z(j;#0jLN-iUH(8P_-U7z03Te^I6dS$aJRH zNK$%HIqrXjy}_M+nYQ8&CcovhfRkV{#%Myr=THwl*fxfgsCX>YJ+}5E;$AFY+&VBd zt$4eNLA_}l2FQ%FH5?l-l*$Q;;e7oL>!u`K1a>nhU1ZlBcuZC2oEk z(3MnoP*B>)6g)Ru{VK7dbx&|IuFO=YtJa=x&@L!GlZr^UPo+fdM>7VETQ=Tq0_;C1Bg4s?4__|PxNXviBr{J6LH`$fBXYv918eoK>i%ZYG& zmc#}EoWR6;Zl=E*g$ll_KDc@^pERur1=pN=JfN|B9Iv%C{+>WCAz^z+2=+R@qn7Q> zY3YX-JrFTIHH^-TG14CqcbsH^CN;w&ej2)+b-_{%DRfBI*QDqRqwBJWC=> zw8Q|4PxO*h{|%U6b&mBLkSj1XWm8RmgQH^SW#TRZK$UY-Oc=s6&)smflXa0;=0grm zlrhFV6&Zf;wf*g=W^8=6{o}4d`)M3)R)LY8s<~8WJFMbt4rHMv`oWC*=!^2wr!BFs zSx9n*A?LmASloRQbuVPZ&8G*sr1(cx&9?@p6X^>&JLLd=(#)M~wP#c&CW0)Y7_yts z27@j4W(+Jq^?x;TD{uvIN{2O{plxyEfkA{Od+#JYKp|MEIqJq5Aa+g~$F+VR@45Lz zLuhJBz_WO09X7XrwNiW)@lKUBx1(lgeRzv_6nbfKKe7csxkXAG*6ed>iA-bDMm}j_ z>k=blR`Z|_Nk=KR?OAhed&2EWVJWks;nx$?s?}%!sz4R@UYXvd9-dU?M%Mr}g45H|n@To}Op_g&V(1+tlCw z|mxRlAn>=%y%U1SxB#piM=R>pG``1)Qi9jp9!C(X>(LDsSt+P0D}dRU8w)0mK-iB>Vg&EAtF92S z1NjJ-dRho|-;dHW7TE@luRD#jdBl}7M;d)*UBFZ3F0|i~%4u$4uYLz=MB?%&NK9*U~6S^e6p zcfg>aQpfd~zyb7qzx2VgLgl$M_Ru%@J;m(n0SlGLMhRV^G+jF3-j~6{4%uscpYqb} z*!fev0-rSIvIyOR^fxfY{2-x-T?C)SD;6I1*>|6R*@4q_2>l&b%zkd@!na z*zK|$;5>0yxN+P8NQfldL2J&U1<(eoc#6GpohNiR?I%l@4TKSZjj-mtH&A#E>qaa} zO0NcmHZaa%@HaY>%SYuEQO6IXYjKcbtCu;Z(pOILnw2$`7T2`nBYWw{qd5mTRMw$VHaO?g}hlv2*DM6ou@ou>i3z-L-SFSfn8JLdw}H@nuWOCe$meui=RJI#;RiRd@9% z6#_jQ^!$W=8uE4Iee5pE^y;*g1NnKEOB5(!pFb!v-EQw@lrW{S{MG7O>N@8GMU zfRFEOj>1go&Pla(-6}bQ7$1-<%*%nkTch=Tm=!dLI}%el|}F3POCzY!7DrSrc|TtF!0-x&JLkRQcOTf2wp&cfPCHAU5>r z{ccW=Bd;L8v4E*%E6vA}kR)&hnaf#f(Q(ce^UDEQ)^B;1FH}QoBJ~Z{ySU}u%sZNz z)Kr%-e_Y=@$HtK}Cy znz$!?ed($tS?*O5ng_hRm%gIJWEWd3`iGmxxk40)3Fr8=WUXw^6_`Hj>l-9MbV|__5NEd0PFn+>^@x^4h960r z+i4@pG)gL0`-Cu~>dvvcAZv7%e&E29##OR!V3C$<6XDd99V>#bo11+<7AS;|*}ymC zNRR^y=0Mgxypo?mCj6G^Xp8LK(7m>f=4YhD49i?0tb%utfC4@ZEUWDuG%repi$HqB z(%IzT3q*OtbJ~GvLP0c7A=y)yu?Z6BsD=`YhT?_U4y&+Iwx#4v1Y^OlDEW&FGqDGk z2r*w#awXQGAr0+1p)ult;lu4^_QOQ~E4kfVJ@CJVJk!Yhl!rto0~;Nr=`ie~LV`9f z5^6Fl(AWE}yfq}wO6^%6A3qG)mW&^K@tEzwtgjD=hjnUl3Vw9yi7#ewDoOQ^nZ$Mf zeyr}2?BQOA1OsIj1GjjZl~Pc!uBu^_w>#i#x`F{>&Zg-|lZ(hyDq=yQnRS;bnvwm87!y>$i<2^)vLbq6eqv2(ea+%)Z^>s4OwbD$Ukwt zUXqq;zdjhGMnX#K+($Xt8?ZQwA8QQ3xRMB7hFd;@I zS)19!ioNN{E5WtmEqn4aM5VJqgoTeP#>7>qa+xS(ySid+k`r#8CE~Bgx_8&zJtn~! z${Mkbxx`mZ6Uon?C>$G{jeq0~KPs|$=J@SJKcxl_UvS?$|4i1kjepFVDfmcfn+|M@ z$L6CGWh`(A@&(z$)X~rKgWQwFcE%62N+AeiU#8NRH|K;Go!Vd^qp4aW_#_3Ehuys*7?bZXKs*UJ9%|{QjBj4QSy9 zv))vo5F&&C=y#{Uc$~Kt{Aqd46ItFNyFZ_PhDf_#30tMP2b5ZS#%DEOE+zN$BCXN; zvje0d^>6r<_J5d~A>VaVifGxrOLm)?)wW)T}XVAS;W zCE`2d4MzRvforv266C91tTy_VczbuBKkp4oGSfISl#}2*Xv{%X=L?>QsNZ&uw)Xe} ztmvw!*$!fEP*e@#jk-qBj52&Lpv*p?vTy3m05y$X83oTo=ymy_jcw*@0&t`c^gY#TnW$sP2 zi{=XWSGq_A=(_1evI??_)WG7cwpthb^%K48<`%tPXuQL`s$)Z%xAT{HM_g6Sz#8u} zx-(5kERI^=@1xzdgiF{SfwZm6&ttOFH{^4?b{CB}s!-{-sJJ#cd4w^`i@rY7&IRS! z$0f(v@HB+prO7uUDqhQ<7xOVuq&T6#?wy-mznzwXWMr11i>9hhRgDzaUGkM<*9S?G zap&r_tOw9s_UzfaMLLWScy3ZbmDiQLvhG!O@?LFUdeN`=M~idd_eSiDEgER~+9`wp zSxD8#@7_MBK57Aq{3g^o6nUtbzR|$tT}tn}XGC!0!(9M?DEG6{Oup3mL*20E#AU5X zGSUA4-iqCeIV_A7s(VfBJoLWu(?9sSm*Ef!nNrYDQ~W;xK3AMd&r%UtwPW#!CgrJNSQx@@3SF8CY49$>qVi} z932ZAnADMQ44{7I65($-y2p1os7JzO4tAbkfjCVgzP1d}_W+95jMTcFZ?9<9Gx-Ic zg_o3k8+#-kDPXMLDcj>fa%iy=9r9H)gzkRSFOh&}z_fRd*oVU6vefE4e=aC7o$5=< z3n*xap25Bbt8&b5*6!+&Da0%@mHhBP*yI6+pF&CuO^b1l$V+fSw z_p?@?CzxYcHcBl&-{>M`)j6AhZMS&sWt&=7moGiVW#sS>MxxY>0g~AizD_9yyI^!Il6PF!QYyys zZ3w>h18Eb+!K!?$8`1@Wp86?=g(pS?-)?cyX1gUcxb%zglf<{{dKRiG=lK zXXYQC+0a*K3~%7X@D0hCb|IF3OFyJaxW&J`Yarj}Bfd`D^9a{_cm$J?C6?8EZw>Op zZy5PQ6;XLVy5Z?VVe`<|vX)2U->`a{eNH=_KA(979pq1!qTN5_84QqK7q2x{t5dpY z|Fe{vdZpPb*DWpnE<+RNnk6pNR&uizvEZ|zo>3;75zcjYwY$Xk)P$rFTmKR(w0^KW zfrNeIw6X}kY5?i%(oJ_Ph~mN|yN9N7*-z+vaynx(HsCWxq|4P)3QgtD00fV>n&D4n zr=B%PNF5b2%p#}x`mTurHD`nl{XjP?Iw$>)wyD@cabrVx!lW6#%6-=#$g8+N7u`;6 zOeFYcrRMz;YHBIwh&@m&d&i?f^s-D2|3Nb&H61$W3CSiIojW1P2zR(w_B;5?)t4EskTHL?C4@kdz}wYWM(!<6yJaSAh(_KCKJH-FQ#(rxiE_(`LAEt zwEc)GSqWm>aeL)bj&qo>9}{(d{@w(c1%XW_VPN0>1B^e`FhVD=3QD$D3b$>!z&)GW zexEGumW|Au4ei z(PYe9S+vaffO3#0C$bz@nC6O|Jwqrjn8=vgocyHPjw8@>w4bXkM6GeGmAr&n0k9#yc{#$OuUII><%0IUX5(`s@cAujSRUg zV(ib{6+W)Zd>XxlzZXXY+vgUCw%B-&_S2HYefGri*f!;cB6)`*NuvY8Zm(bDe{}Xq zHf{pzDlV#`(xpTDS>^n3%ddE`EFW;GPRNlG~<9+bmc2L{KD-i3vCmQ0ey z4P;|vd-+kr*K~6uzSBr4m6czge+w&ha}gVFVyz;A&f375)kN&G-fn+<*pVAD33>>% zy}o_X`&$EP(OfHLdmX*0@3B>!RSE>ujF>LuTL`l6=-Na5eYGKMBkKm@$&nbEz zl!XTQY;+KvBPuN}VCe z2m(}lQc6u|tRj}duU`srl%_~12~_)kgSo5G!p|D%hpaHk5>({c9WYqLdN@ zFX@q*<^3%0jWk`SM%O$tt-#D0F45%_FLH`;xDnof(v)6G)U2reL~%@7U=Iqm`jNi^*qwXTD&)86WGD4=oW8SY6wHFtHN{e4F3RrYst23+PKANg-~MJY-BKr~XZ0quZCS-Ep}Q z{{Wu0%8HbOiR9v?OQEi!+YW}^)SU!wOG0_) z{{SM8-rXwl`Ao7QEo8?^aHSU?e~t>1Co6w#BWol>`1^_NNTAlW4CH?nr=0>S)F5K$s9p zl*-;z0!L12mbUY?FF{h1Gg@EVB{tC5Aw=y1`_SllVUID|Qxi3=yA@kn0aJjw;!q@{ z5KV7uC`3Dwv$`tQt?mz9Y%?81f%dC1fS_(17~m3nR!?>~LvF!103>zkSFu@hd(IM5 z5Rhqet`((3N>H-`i)l*?>`6}4oaQOkN@U(N(+o?Y#cl;?>&-K@yh;jEi5beLrE{oS z)~Q*D5NmIm@Il-}M{0OwcVs1nHA8#5;N{9vfxRHgXMk(ULA_sx2?N`jO=|QgsP)Zs z8bjzSE+~Kio<1u@?liV+EsBzqm2gEUSlbJ6K}k;u0-N97B|w#w?cF)e zaXPNDw;c^z?mMM01V^nLjVGYujF@}+nOmC@;($ruaa}*d_fU`tBxjS&eEF8tNZe$R zB%0`~9#yyrk-9kMxK!UOJ6tL{K9q$Q5#?H>r>7OmYu1h^83kFG1MyFEJCf^Z+(Z&2 zQ>S#+ZcrqgiNT8XF-wv;)<=#<#+}gKbQL-z0lOou2XFAo-*a(B*v!R!e)mf%{uR0b zi2K!=(GuAHAln@~*GC=EcjsiSA2V;^V1SuyD3iDdz!XawSB9bjI5JFQ?dxAi{{Sd1 zZc>B-BkF@fv!L$YNFfQFZsxkCNWxF^38T`M2S#Pdm3|=nw*o7nx6>h0=@B$ov4Tuby%8)^ zZKtIp2p@WBZ)lW^Cm!@;R-q#>D3;htcol(WY_bwCB8+XE)S)0`VAk!x=}eG~`YeP% zt&*4$YKbGITYey7o1qOv3X-sT^I2}1gVQ`#F`(y(tH}1D!Nk{!!1kYa=BK3&mkG$v-mDx3m^N$DsmHmhmm~iGnxS7FlmKbF?^X&Q%UA=) zt$V~7&0_>l+M=-z^qR0BoSN2SQC1TiR%wjK^{WHxTFhE%YLnj;AwXdBM8&7ps9?=_ zoYiJ%42kxs=Zf$VlTZ`|X6QC^Pu+tsW8RiOeG>{#=7G7?00va7j`Wi2QU3r@>C{wg z`pkF+kLm=-B_|WjDZ13Qg-U@j#_lOC=BxD$$Ai|n`^{u+U!07d4k=qbJYFl1_U~5A z6^M`Ku6plS+c4lGzUo1Lt^WY5^c>eWd94pOoSc&c;-uwWp4JZvyGKQPt@A4%n&+EZOKwOb9s=*(V9ZuR7Xgo?ez5=6szq_Z1mEv(vbtTbT&F)mmuT;fmvg; zeD{<5O757R*GrU-DF?RNZ*8PZLQmql8%-flj1Q-^IJMG@87_r&Y^F}vpdI|Ov(H5W&rae9bwg%0ym?<&pD{#aG5&#@eY9S~ij8(R% z0h-)Qnfc%RX0mGEP;JH$4k|>)12yq?Ey|aKezdqh> zNKgl44A-0FD`&ajw9$yTZL}#32?G<_v1vhSpP?hU&3T{|Au5E-5!}_r)e?X=9{sDy zo7wbGNh3lK>WY8_4l2lT^e6_7(fO}a{i><6^hOk&?m<#I?JhS>-q)Tto$=BT^9 zso89WBmo~PS>DyY8OtA3el7$1m^~>${YboxJrCM$tV5lh`C!z4Kjj)ay_d- zZs}I%-%*oN} zQ_UWjD6V-y_F6e~C2PQCdoJf`>YT+I+FQ19x2}|ie~3g;RGgr&qGz?g}MBEEm6rL9?BGdAdBP(Ekv zK>;sEDoT1EK}Om*#k1v1tHN=4#wL9#&QeKo+Gie@cAr79^Z61YC1h+)mj@j}c;2fcADmX3!CXH{mB{kMj=3LOS! zuz8gh)TCslm8VtkNHR!0M|vRYLEycL5|sDNblhaO*&31qbhv~rXLR@Cm$au_eWii| zW_g;{?(ww)B&T;GaWswnTsGlkz)bT|(tA217d4~lcKVV$G`6KD%m!;MNlEax1Ys)4 zt}4q@DqCq%iBe3Yj@0_@^~GC-5+IN0YqiZdyExn&WG>aoLtxAk1Vt*+?$>Vf$&dzA zLXIhxrUmt=4S7@}uUb)H7XJW=Cu_|T2sq&Pu3C55)f(a-aItiklHeR}N}_$KQSk+^ zQWYUE7C5E}f5SjR&`$MMKPmF=ERQ-=W;@c`ZK)2abSgu@JRDgeXu%5WMBW5kp)rz9 z;R0z@`6;<@r|2YrDlG1nggB+E=;V4*aeW;LYmiN;Q%;qHr~pnW?dxf|w+l!mMgiI} zOD&ybzJRDGaE`rAJGN;%mX(YpQxnLbqb0I4YR-9t8&^xILW)5fgmGP3TvfRbGJIQS z(Ugqj)mNu9g?OzbI@5z{w`M52TSC>Nj0Gpl=0WKd(FE?!e3Mc_OQy?PJS42Nq69}7 zsJhbD%8+&`AwzOcdg_yA+m=O2Erp~4VKZL2A33H3HlH|jB&l7y)gul|Wd5aTA3XSi z$?Ka2AGb+Tleff3KBRwBO50D(>8yu5K;R&yN)a)N`g(M7t2>5Pf%6Fl712|eq~k+N z*6p~K(v7M+(HxN<_o+NmY_xoYqemTX*UD`p>2QRY>(l!hW-&WGnwyZ=< z)@f30AHHP-ZCZDroMei4EfTG+Z4j~&WOFnYx1%gP+l{soQkgjBicKpKNjFSN*g}zP zpd~9lX%xYvC3fz%HmwS35;}r9(6<(dw6wNlsbS=u$&Lt`w$<7I@`|?ZQi^@XX=3T3 zkM8JP^Mc<%HtQP$YYERJl^Q!pxHeX|FF2KLK#YH1>G`1bF9!18Cu=XLZU>M@THVzl zn?~9ySC{~)>snG8G0-2wmzN8+xqjnvowy`~DDXQ|`(!fRqSO>O$%OOxdt_Ir#b0R;rI<~S~p|!dagn=^HlBGp_PWGM_OWYA!n-F45~p)TXWyC;nt4cdV`* zTdPA22uKpD&cVX9Ih7$QeMEb4G)>7CsVYh#OUji^dKvb4vnn&Rp#lVV6|-OjZ;Tvy zgl4O3(n|!!-u|_-r~!o`0Xa;`?M8*La^jSPgeQ(`*9{$^O0AkeMY>hg;MUjdkzia> zgzSi>QFJFI$O)w-Kq?ghT?Jjs7U-7HVkIIyD=(LEPPpU<97Z5}QHynE zYAw*g`+!>56}a_RV@n*fX}MS)D~j~3&^$#Q++z`1tpW%MB}SRf&>;>i&fTzF01!z4 zM`~`xp-Q(XeI^VF8%}H`#VUN$lir-P;M#iRkx3n`hHb4Owv@jJVp2B@n(H*Tcc=xN zPCaXp)6!f_jCrsr^@&zw@Z*Ry9oorw}4Vwka~p|J`I_KIlLvQy?zCO8yA7$y|5RHNFuQ?d*N zn*gN~1VHL2maKvjWQt&xk}<&?ik}P_9R(G#ZO|BK1La8f6$PlD_oKr@Y{ZJiNR9+i zqgyjd5)T4mtx^ZANJ5WUtb}gL%~>j`@ajMSnG-|?so;9l)e<07?f@EOV$&5H@il_n znKg=?GePLVJ}hxrt?3jhpNvq>ToNOtEtn|W0HsA{yQmULk4#W5Tmc{o6;~%YkF_m< zw1d@hXPS>6)|D>g=1phwL~?0)Bu}2k#^Mf4eX8iW9p;f6z?q5lB+XeDvK8hd*wEw- zM?jmN(od?YP$K3zKeSaMERkx~~JngzG3Q08Qn4{B>LNMCA=-n20%y?8i%sHR}ByjQn(p*PQ3 zlJ3xB0*!<#ekuZ|l|>2TlU_@`=^HWe%iUXOg%Akzu5R~Pf}o-SnBuwpM_qzpW0?A@ zmcG|H^PYMcu6aoxtl;pzPL}&!fIuSv^sZ|E0AI<1B4>)`uXUvouQHF91HB-*)RwaW zQ#p{XmCr1{2dxY+#n}4yUER1$!R<(HH8r?YQyCN#y6Bvry%%j@wZr9ZLKK!dRTJ*v^h&zsNYx9m}6q+~z`-nu(2D*`~792#q5rGXw( z0yAl1N%BHJ+y;Ewg}lUvjPeLoZcwqfDN{{R&`XBDp8 z$>OJERtd!XeWHJw=#R7zaGY(&Ltm9OJ1+snw)P=;@5!&*H+I6|g&^c8e)aj2{7?8{ zT3({#fh0ERCP3?&On zl~pC}(j~@Ap}RIND9Vym_L|)~a<`J{4J7f>hF-F?1g27gIe>Vof0?T7U<8tZ%89P1 zaOmfMQ??$PeLBs1mHc-T(B`bWrWOH9L?^b&5yu-#dsVa;D$?!n2aZT^i87)OjQg!3g1CPVq73h9WYd2lG}s2Aik1xwpw#Fm@4KtCtH*7Z#(pA65kc z{&Bg_4%n-aDqjcMkyGguN&9D6;tORdA^9VrHLDiO)DGruLuv=NByr7i4BR2tNOi;o z6V-G3SFW{-i)0a}`I3?ptoEoh(M3&Y?9$WitqNVhZsC#(YfqGdJv!HCqiXGI>H%de z>kTGW*f5)vgY_gH7aW1kE8|wb5Id_GT79Jwhf<=F2<`h) z$0LJgmM&`jpKx8$^tT(luu)SBKg2=79mab4S6!lOjoh8fQbKlsL}I>wpzBS1PiV9S zB^t_73(TbXE~6ny9T$U+wUw>lwR9;Dr~pb-kO(8(9D)ujtCtVDJu`=klW~`4(sj)l ztHhPILWbZ*Vk?YQitJw@J*u5aDoKe<$KI9QTRmv%7SaynjjG4s z*FqF+t-(}fI~IdUAJlfZ>i2~S;F{9o6=sJf#Mg>al&Jvk%*H{ka`B6g3Dxc0u<~oF zzjY2L#z&N##VmQ`rJ{!fp=0XztybvjgK_9tpZl^|AT4 z+)C0Crtm?*AlBQyX%3U*SkD}aA=_P=Xayn^LEYF6XmPb89mll(>6z2kv>CUN7cn+wi z1W$4de*D)p)Q;#%;jlIyaS3IhWk^&eedfCTBUxLFI?ccyIygIeS23$ws@0OV(6ueM zB{=4oi(=dCUs*YYHj+fa&(sgKQaV;nO-n~o#-NtUQWpWliBCXjgLhvqEs?n zFV=0@`GkdS3Iv}@BS_Q@(MaF5GbeHX0MAND$*zgKcC70dx=W0$wCxHe0VVq>>Xo3>OSEooRFcaCVbtzKQlrR}H*h3#}G;YsXALp|#8a*4{+N9?c1 zERX~(Sv{a1_NBUkA=eyaD{3QhP(5=+_^DgSZ~j`8WiiKDp_)-|R&g#Jp+j(v3QrZ* zsipZj{ex;Yj5do)i*pSvA!=Gj1gDQfIHY&>q`KqDNd7P?QjaGe*18v2X|ZsGm4K7x zLHvUhC9AAzHZEGFB}*9fjto-CDLP?S15~SIna;>=}a6saOcI_{PCg2p5qzMO# zU#!07l83fDzpg-Esft};dX>7M>xn2$A=BD=Z|qz`QU3rC(_>zZzTsoF7-0*^cQUoeFHRl^vm2jQXme$x& z+#mvZs%f`gKCDBCBHjr~ls6S;(zIhry^AK~?7hPbzU{xy0;DpKy}Zh(fv9}@N^aZ` z8df%rf|1-@TwL8=1S0`(*Z_=7Q_DRyb{51Sl{ic!1%h%jipe+7tt7U^PF(_vljRwN znClQIB{G{vRP&@2$CUHZjVMc7((+WbrQzj>>Uw>uH9rl;+GRi`YjBO7NCJP_q?XF5 zS3q_)N^Pb|LX@=-NFqjaL|(V^P8QMT2_O^RXp*(JZc^Npl|&9e=~QWj+cJPsqT-2G zM-xp{m^oXvV(U)HVW9vSC(^UIbReW4`2dBEX}fJKFkol(DwS$hg}pV^0}ED1b6G|9 z1tsjt(&OfmuMI?`FA>%aQ&+k#CAJ77%_I)hT|U`XmZ%fvDJuNdVU;|fGfdVMTnm!AOB2-n|rF%@bBWjc)NFHovk1YhH)ivi*Ate2(@Lz2Zo7j}F zP_UeZsDT2sxj=0YXLQAFYL&Tig)by$iZ$yKye)Gi?b-y*O{=;cO1l-<&t5tc%{NL& zY)%gu#MCEGC|?`DnmSJuL##mqFaYL)j}@>vDLoA=T@0`yCPddkrmKI|)Az1o+>n%} z1a7BRcR`;lU__Jr$BOC4B$>@LbS$IAbcFB$u7cG{X#ft8ee0jo7BZ!jf~kQt`rSdu z+Xe^*E7HeRXC&{Vp|D=yXM%VX;@z~Uk~kQFNo|Q~vBaMJDE6*)H^`ED8tsI*qmoE0 zw4$&RkSg1D?J5A6fnI5QTHbn}&3O=IOXT#f*hw>)sx(NP)~RZ7-l4RE)+mIC1dNa( zYVjh>TucqISQJrZCyIjKQ$^Dh6gDF@V38bFvD+e|FlcldF(R`vCO@Tb130Sf6h$$Z z25T~{z{L>&`c(@+j8HHd=e-Q$n?V54>uxAV8=6x9slz?!Rg;;TV77fa&~Ksv9jO=} zKn|~IllIRv8tzm#W+;n*V4h4;ht#1s>HE|hBFz_h(2`HJ4CU|=Cyr#$7Z0irK2e8)5I<^9sEthN+SCG+K2qV^#yLe|H{kW`6%9MIX zt|=QEkCg_s{qh#0#ZF_#zt|{s-2jms1r2`C#7nnl}y%qO%oIaL#_Rmf# zfw3bL>c>O@BNLeBx*IJUk0^}RnPd&0~(ShXT&gIY*o|rbL2as?n<({1oNCOdyKF-1bkWWk*r*^hsG3D!B z2~o(~0<+V=AjjlWdwUQuABr)yJTb)Pb4Ik7$&YFvF>1^XaYi=6K<^czB%YO~eJcui zv;YrE-6P(!O1USIS_R1Fu%?Jd;8w{<9GV>tfK6&1>_uT{-8cjaksOgUIC3Y~)}%eO zM2aZYVLOMlM2Ijbb8!MBR!V#QRjk1q>%q+PLWgAYUhxDI+JIGsg{FAxUpoH)6h0ps zu9mf-Kg39x{MU9|=bFOtv2x1VaL6f)h@na8jTY0h@s6=*EuLCb6TB<0Nv>()qO}67 zbc*{&_>1tuY_8o>NlFzQMoiboZBT=U;;LeDoN(9(nm7q zKiKRa*KMr`ZrQS>0U)c=l3i^ znWeCC;`Csr-$3ycP#YhoGx+OUSXOx+GW1-61DeMOj2D_SzES>9EWDRZtZX$ftx)-p#iR=U4aj<`|?N)AtYFG{y^@GZq4tPIE) z9+a;`C-TLXc5OGjr7M#>n5Gtw2AtUT!i8dUMCuJnN?SV>9K=O)s!}aVn=TimA2Blq zxhC$;h*8xsYSV9pHbDzd6U8z^O$QRSr)f#*IUbeB>DP*Egg6S&a*{a}(X{i(R`!pG zHsvZc%_%st@r|JHwduN}Hx@x)6(v%YX1lfBBjU?dn=dECt)Q(|$I0`k9H}RO>OkYI zaZN2tPl%K$5CnifrCClxh<&Fvsiy+cQg)Q6a71;X9Q-Xlm?wcwn(YM#!;L8jaFQc; zLyn|(;zeS@UTMXm;niVVk`61SxnVyR**kk?+EJ~tnY(GPFU^2I<{>}N9D&SNBV}}? zg(C4Xkf2broOK@lmCuqI`(3Bk`|?{9_q9*h+TT$AFYt-n$>CfODTTtq9a@CJ!3V78 z6b+)7arjygW6QLN-RnoZ?Wk3`yh5#%v#=5g5+wfsO6QCmli2f>HAp7gQ?@`1Tp2+~ zQb+=w3-M)a)7EZpN70j!Tvnx}sc3O_DnpA&+<*>BPfE{9)D>>~mTpe>08}Vr82*)b z#ih|HB`%rSuTfRYb8&?ck`JvQ)SOCOlsI9)209RVtl7O=fxw^&TY_M8pe~tl8`f}{ zlu5)?+-))O(tibRUvyfo4&@}L9)wLKyVYE2kjlL1a2)nEyY~tTP+F9dp^R5Gs61M@ zZKAnt#1fKETG5Hw9*&1bzfpB*R^bT5dREpBxVw^2M$)ne+}Ay!TsYn9ZNEE#AbT3? z^c_FuE^e&P@E&Q+$p;&O^8h}?kF`-R{YcJSZI7+P&8-VkWik?AbgsJLlsl)TEb@{} zanrqI)|T@xwetyuxX~pfa6ymU0Y~5Lx6&O_e2yXrR}<-0jy@Kd=1V6wsO4R``C#f? z;vq4>HHFTmo5V89yF#%FN%x_(cL-_4f+5t%GhEHPt4i2uYJ=kJ1zkzvr5QE$qp~>D zT#kb3+2B}xq!l}Ep&*Q@GY9r2ltHCVhM_@V^L^~x7@li}_^U%E zwCGtE-GOt7XQaHuN&5BQ#7W929KubD~nu4^YCu}NkWDe9;h@I|TGjtLH6r>~p zJEMaqt#aQFYQ$;jT0}b7JLGy-NoRPGxV!^|l4sQQu9!+PQEcPLoMg`~)x=(&e2{qU zn!c(VY0DSVHl=M35R7e6$^F2h3+V9vWgEh`Y)}C4#QRe1Ug5E)lqtaHhNT0z`eLg} zDbXvoExyzfajRP%2uMqTJQTnH``}Wi^%r#Oz-?rQ7v|m2N&f(9qVK#HLdsM0%2X+4 zrF0#zarVjIyyiF)P?Ya}&a7}P@m1f=paO&d;GN0np_=EcTmZGXD+?wdbrjn5Y@ID` zDO~lP9S5u7B0{9dM{eh%z-Y-P+h}k-aL~Po*%i zzeVHdDp~Q`Z<)qSQcXusaj%JALw+brPvCGAl@5P3OJ|Sy;@bcPrC|nf>rvg7sJNY8 zslDq5T2{i6LQ0S6^{s8To7QcbFqEkHneGfH`~B!kcRY6-L6YByoyh`)x4CNHR=Pq8 zyraZa`|_a!Myu1PzAf$3CBw`3dRNg+0b$hp71b&>owyr>-skw?mx4gi!X zvAfD}oc{n?%R;t!Z9m`?w9{%Nka?f}O-`(}wH9`(EeC&5AEbKsrG}!}rtW-EdxWGk z!EsyNR$Et4uYRbI0%2e8voz44b^qsQssc<9;aI^b@zJq)DU&C3Y%m(Mb&~R+od3ok>!&owH$F)8RT>T>uE|-ke#6)%hc9Zjv=OB z4hdR}d)Dr`;ygls<)_ks0Ul+q2;JqTz35b}Y@0@cCGNum){R5HBKIo4F{9J}P2YSx!*#e1ixoj^AQSU)%({kZkq!5ve(f&5~JVkv8 z;8R7ZD-o)6Xv<1%1xrdqSDA2?g-VqqW3?K9Zb(E-fzqzKK+4eDX@v}*N_uOeX{*>} zqSh2jPjOu~n)b*lP}~GhYUi6r%e9 zHyFU^D$8(A(0Himd(|G8i+eIsW~&vU^{me|jYSH1s%`bHdQ}Q!6F>zzk9vSP_pLTy zdenx`K|~Y|;wPHGv)oNE@Q!Axw)XU-V4+iuZ4v|-pdV*+<^~bisqb_O7k#VrD@6(#wl7W&oeXMWSve9dT@^A_<;o2W-ZAdvRTR))EXK!JwVf zIr7K7NyRZLjy=nAAOR;ehk&UuT{`saleB%Pby)4dOi!gJ(o87|7-XcGCz@ezBX%Si zk6PWkA-O|8b4532u{ZwR z?@bs%INjETwYOC9k7{vkZP1C$3T2{;w%GPpi%5Hv19T z>5_ht_YqwMo|_;2urp6A^l<~q20K%mT_At_Fh13I#k0rB`iZELQ@DMJ*lO=q2Q#?I1^48(?AkZnlYOY zK9CZ7ig5rfn9c%|>?x}#f%N0vw_?ms(mwQBEr5b{f$3PlpdQAyO2{3E>E4L&G3{RP zz(El;fn^f14g$T+SUZD`J!;zo2nK3Wfj-m#h)b{uJ!;8W743#3jP$DnE2*pl-XR@o zL*30mY@A?^TB%7Bi1ihaY|uH3=Dp&Pz^wSHnXP8JG%l(*Pu$fMst3G%ssaj> zqE0#v)J9{uaEf(xMa$tu4mwrNI5p8(X{_6&3SGD-Yxwu;ofAROht4$UWzT7@g4&cj`UWS zqb0qi)3|^+#Xo7Le^QFf$c`&0`wa3VsRN;xE_QEGOF#%nJkt*in~6m|?X7pyaHGl+-1rR@E8X>B)9{K@Ejunra?q-rK-rgaOKi0QIHMUP62ZP<*0E zG$^Cm(!n3$NW9e*y3;$o-3i!*Kuy5+ocXRlOaUrk~^fi zu`gQ^sKj-OKTdf|No~+ua1Ka{<(g7DBIxOydDNsWg$!~Dnfp+!-xpS-xZj5+AVAG3 zV^Mz7cu)M-~J*7Ucn zuGW>h+&Giesx$3J>bhm7uckk%={B)$t;*DFQs0`@NA;u*TRf4{E9bgx>$)bc)8Ov3 zWx!k$xD-q%ZS56B4l>_g@B8>C%j&__n*QTmS;wsI8@jhy-EN~HJS`n?I`yEh^=}dT zOV$U9nS-;Kr`pquC2m{WsU^gT3OEYt2=@e6A{j!KqJ_9sHJOlg}_lWBc&B;w7mYoxuRRd9th}Yt)30LoBaMD z+Tu^XDtlIkR3`B^vuN>BNJ>ChCo1iOkyzeo*0pT$DGEm@&3gX;OL2j)0;cD3 zJr7E0V`wizTxcZ^t+3jXz7$0So|-AiPzS^M8~5#nHbuk81cHuCWaH2Z>>5{(X2rr# zlDlOow^lx)37n^#aa`xaw~Eqr)~;GthEnQVbt=xqPC-vlYwUM=Wui~dTOqcdc$HkT zkT$rJ$sVLy#Svzg>PgxDf zNgI%JwrP%=cr|NMAv;?_j`7NjWAR-UU;x9zxP&PBBh3H+&mKCC2s;3_#=_E1s~rwzAbc zEk$L$@9aaRVgNY^C$?#xrK$^im0R6!*vhs$yPb#WKEza|1e0j3w-6vRetN#F*QBR&E1J*H4wH+SH`qgZpc&ro^G@@ju7z5L_a2gWV z;yZ104)T+*s~Hm_mZYGWc*-+vQT5jDn|byxEtG<%)3m|FjztHlge*3ul^~Fy2*}CB zbH54d4l>%U-%vKL{{RuC&-}H_f8L(IdM;cJ5SZTuJ$DcMpS^CD&VD>)u7G|Ty={$M z_LT8bRHL~kAWe5=DR20p)C4F4Gl3X2^80iI>#JS5fOlr(B$z5Q-oAmSJdIY-FQrA+ zQzb|y3P?U=(!SsxNC_8uDxJ9Dx;JE5MH9_4fPp5k8lep2swCv)qTNW)` zHu}^Q+ys;B#dB8LnnlX3?i4~?+}uxG#dW_K`TlK&l2Y?b2#m1Sue6HGM6{s<3W*q@uBZW$WAi(C6Sw4m8H_ti$0NrnT z5)A%(2(N1HgMuLl%BNk~ZQJ$);gQkU#^LN^xvPV-H< zY^zSVb!6MEJIYcdEXj||R{E}{y+ct|ks&EY(om3bBNfYhFQ~P2tvnE+;WxW)Law9Q zDXo)kuT|7mRvdBXQd?2U2Z}AXXh#&CpFqYJi8CkE$8OZ>>E~_jm!@uQWhe#GI+I+HblX17vuQ|8s+Q=ERN{mW$M>#@Hj+mi z6}HaHPSXRM0v>L*mR;(YHh863nZ;UPLMxuYV1)|sN-_mv31MMlolOInhu@&K;(bm zpl+|y+uKY8ArPIs%?*C??;SU(aHiEHHiU3V24;J7q1{K$w1pt-P*%cEHN!bOqplED zY$r>&OP4+?$=XW1yN4!-VJd#^;^O3&0uo>+BdDxRqRZ`~+z0?CaTMhis!PqLzu zTDx$;RkM4^R3l<%?oZ~c3@@kIIZZ#fq)$fNOO>ZE4}om8KWUVp7|LC+t;RpGr6S zzrXl8R!8s;X>Np|luA;ZNlZziZ7j{}p(n_J&$UPSvof^1qv^{C$neX^{)?kD>)kcv zEiUO7c0dvVv(=T6?CMk_(yJI_b@wmp{s~FVTcN$3yDiB{;{vbp=_^vn&lN`1q_`4` zT|)lRL}XpqhZ233(U01gx23-GabV@Pqd7k-a+N3_`c(@3tFk!lX(#(K4qqbj*i*k( zlyt8ZN|M@mCQ0p9bw4Op!@JP(Mo{8PRi8sp(snkZ!pOwpr%vc(^mACz75;c0zZga?TgFcJ^+SBuz9z;Uyyk*Egr#5F~{Y5t?k( z!d5?4F^cqY%_T<+qw*U(o=sf?K70zvwSpw(v@0qnIO$yrVlozalQl}WNvS~lp7g?* z?c%3zde>)jHDHR=3!@~7tAux}gyOE0gILM35`i356x@M5Xrv&K9V^ATHK<<2t};x} z%4FxtqwFcsVnt%62U-+`5vtgB6GA&_xDY@z{Y+zo{it^>`K97xn{vw{bpcIZ(77$6; zq4uH?Wsjv_`)Tm8jK}7*O`zBp`r;?#^E+MSy`@%R;CW;+Js^7uPAGJ= zCuYJXLcbKk-q)$Y9ExP!r!ywEZrg84bps$q?Xqwv#;x(k>WU$zM_Mf`=9wb{A&diB zhGKS^9q6RB=%2M}8x^SnfI}u~OId+OhN@D)6{si#TaLAKus!JD$OgO`8%+ut&&Rf! zECk4iPkM7{qkkx#oYzllqZtIG;Uo0Zi)|n!C&+SV>8U6rUCpzbwb6(Id5-kr*Gcq` zy>vGESQrT(dSKOnf#n}zPeklU#+VZr-TTuvED||PeW>>A;s^$cLqm@@*ijS(yHJ$C z1MgaR8$^IU)W9$SP#KE%g@Pa-YQW@xlF0#I>gKKy00GGKqP#ObssluRpWd;o9uHcb zq?ix~siJ%{{GYveH6}<~O?zh|uu+!6 zc&I4oMre((z@NQ)Z=n^8^a@Z2II4JEsJ)foi0N1s3L1}E_k=|55k9nD-t`*?0DpSH zOdK}@5%x83z%WS2qFZcs6|aX_)(RVnf}ct&ZCk^UST+Z@6sO`XCB0Wpais{s0R5@? zOjc_`RA7mdn5<;!pNzgW_-XAQP;na|Pm~-?X1GhuK-%G1P+l?3ezpGq6#f(c05H?n zacqfoBXV#sbN#FG&Z%OAy1x+s00_+2lb#<*wwdqvnYpTB<}dza!DOf;dw^+&jON23 zEsnUP>b`EuQo}jq!eS@~bti6CO4YRTE}pqj zQ_I?tWDYCI-sjOI)OZL=%C1-JBC4e1O+`c*jXTD3e%}#l*mxqBqbdx+DA3EMaAnDh$(Fa#q^Qo{ivHJo^_(_**57P zNFHBmq62KKEdVT@!m@qKjX~cGm_j)K<|%&yS!6H9b!4tj1yePpvXTJ99B#+c{_N3a6%xWwKQs%~#Kp>uzLX4PY#nK%91+t2i;mHGlX{C}Hu(nDz%76t# ziSJa=^!9Ytkm|u+)bg(4mTJ1AXj6$xi9uO8kYHCdooN+GwpQlsxbnclP7c(?DZF*R z2qnarQIM#iZ=SeHM}m^#NfWf>)|Sh-xky8e?S$ihT9*~sOJ-upca3dj;8DDuN z<{nC!bwo@Q^F-RER`0`z30dTA=kZT1mIQ~`P|SrPH3>5+KmR`3gSM#AzMR3V9bVf*{dQxtgFxt}KJOM)^;lwmc z;e|H%RutH$EV%WT(&pty3yg}<%}H*uL$Jz{uc<1*si}HnD^ExU$>#Qt^9DkeNrj*9 zQ1#+1lta>6S0yz!P9fHe5TK#@d)GgIck=e9VP5o{A!wN$Yb2$h?#_^WRdw*A*7}8$ zf$Mh9nsenmr9-Gc-mmeVpKY#asQ9np0@>4wgLiDClB-8fAwUnIT@OQFJF?fuj;8KX zQ*5AooP?g>Qy+)5?L+)K)rPI4P5y~?JPZ3W3X-;uDI?RG>Bn1y`hU=r;M3<0&C{*D zb);EdTrEj7xwuk8mkQbtqZ@`craz;?XxcR;Hym`d}qNG2#f2sNhI5&1t5d*24y+ zAGCd8cAZ~DZ&0*TV(#8NM5O&#oci>p%w8_*a-B*iY=Q^kxZMS}bW5cw(PXshrAQ<~ zhhjd}*l2ex>TSBacG7~&o*L1*!}P@iM1b`buX{%ND!x(k8pb084=l0OmlJkrH4 zOg7e^LB-Z?Ul(X>{Lu5c5lGqwFae*GuZRyR*1Z z-O7@vP#lnZinE$&S;-u2-5te~PTQ?Yw_f&{aQOD~B?(ZS!Z{KrnnSPYP2BEs*=ub| zNeT%`QEcRn{NlQ+4NA@HyEI+4LzfIGEjTa?2r2YAG&RkG_S(CauPsUmQj049{{Rs) z*QcdYaOIB4!fp~iaq%{kmzl7>@Q?A(67Iy{x{k1V@kCjqUow_%cPY@>OG|SErZSW2 z2*lTGu6#7NZC7ZuEmqfny=tPc3LOz4CxqrJfZcOo`c3K!E(Mf4l@o$a6h8;^Tyjqn zy8X`lbNZ35YoP58>EAtS{mr6$JJ5HQ#zTXX2cf~FjV%|2{-T$9nsiIc zYm1_ekJLXPgYQ}0GNx@jE)E?M#?)HqE6j6qeStHWYi2itd8t_Or@wi?n?I0Q^U}ODv-0 zmlUNXVpr)AAH96XP`Gm9(Q36;<+edHoLA8Q0L2#Gv}tW9AT;XIQbt95&qQz{_Vo|# z`E@1`a~+TGn&!(Ny3bq1Yw4d#{{RjgZp!68FhZ^GLPM!c;bS2C*W7H`OSbn+6(!WI zFqI(VBEARk!-%rGEjXsrwIN7QFcdl;#eJVZbvM>QO_Ghs2}0C5_pYA`Uw0FW&W*lb zC%zj?u4w-NC98xBrchc_ion9Q&!#JcW;ERz+*~$QbepH`cdM#Uut#3qYrV6(3fHaQ z4K~u$g|hJIr9qkgw435>J(pDDZJ`S(aX~3a`bnXUa7eon?#lZ1EJg zih=JYf3UxhX9zBJ8A6Bwmoztww<+!q? zZpSkfz-kmiuj~&*HduE_-$UqMLj#r2PJ>_2u5n zn`X>ylHjdoCxmhVrBo%a}y2aJk`C2?A1l~S;pdX}9GofWP|d}BqXfzrB6z0r`W0GsYnH6Ct^D!dY-lA%B3oa-5GI)${@J3 zgtL^NHFYS^H#c34W&i3KLT?k8t1MY0G;YmN{sI6eJIO|Z=GxW6cGpAueUnvK8g^re*&C5hb>h zgovKvx*^rFEOT#q`%EI;PluW3!npzDeifhIjasZ$!45R5Qh)voEgy8A2eorH?#hIx za1a5bo_G?HKu8^WitdJ8J2~7{=pOG(dDRzf)ZNv(a+NT>#CwiG^r8H34FO1kqA+KV z`_oH#@;ZkU%E1s)VEffyb&IC}yJ%b~K2YmtYX0+sQ*d3aHa)(9?AY;tis7BTY4)B< zzLVvT?@HFfkcef(x;lg9Q1=I)v8$r*!$?3&Y(VKwY@7%QA~_uMT{z{lmT}SAvOqxhircBAHdm@~K{$vU zA6j(TDg^PG^kF_oL0|`_)l4LQ;B|tx^E>tda>8ssvB1 zF*jxQf^*GCNExY0X073q%>bs&5|Aq4Aak0kLFrlnGgt&|5Njni4hJ={5l}>o)-W%G z4Q94+#TU2My|bAWfGbtF?O3Ycyw79BH!V*UnPbxwLWK&o1ZE@Nvo_f{2Ah^gNIz$Fi+Bf|_N`LD?L{TAoGa`q z;W5Qy10{jlwo6s8NC_ve^sDU^jb>r6D%(LM)$7se6{yIr+}EV{t5s_R^q4bH3{=!@ z>?;`gs~T!!=uI$c!~v86?M+c?2MH5fqTg%~1o~1WQpJd~N^uDT)`&&7p{5D8ANVU! z#C0Z(6R~H@1NuSttA-V~J`D6o_N#aV44I=mFG`Tm+aOS*0eCVo zP+I2(mv02ICwUXQE1w$KAFERaC@R3<_6 zHTwl&TZiR0yB?1pnJ?d?n_9s`= z+!l@{p@3l{wkwNimtQ4HTGZhZK^O*()pdnF?}KanMahIu6>B96Hq`av!(>PSW}@9L zi$*DjSc7WRU7)EZ4h3wrtvg+@*zh?C0uR{Gwto@Y->>DIa2}EXu8zY-wNLRh>+Omn zRUqg6tDV>W$aQz1T|VpQVK=t^68f)fQ$pRtL=+DZzuuWANmJ~A;?h;rrB%JQj!={Q zQ$50~nsxsGsaU^|W}5}1g|EVJjFW>@U7f=au=;KQ$pqw4E}`32%1YFzKgtz~2mHxF zL&Hc|!Ga>Wn`^V8H2N*`<;0fXi5w_%K%3<@;X!LnyEclNZ^vjVV<3_5S^PZm?Ob1q z%Y_6T?XDCLVh3K-?ROXUA$IpIt#;{F7K6Bt`&KP5ZrcvCwbUtJ^H8CWLOL3vYN|kY z+8^lGmf9_y-k+=4D(~SfLY8iX73S6HPAY8-0=p+7 z2ow4hIRnzKib_lA$^OsahD9uI>PP-p$@ACr_imKpTLl+QEU6(U3K7Wv0M3=S!?)#~ zTHQAX!G7T$8;S%&k<|Os?Ni~a-2&-0mn|+?A$FG%e~8gIgVYR)S*&QvTPnZPP^3A% zD=i?;gy8MjH3u0%xJa~T8OM}*Iv^8t{mrGq6XKAdi9{IfJ!w7j?pfa*4!V+ud8-g(bB? zZ7L@+a|C*hDcemB)kNqMs}$J$zU3Di01&KKHEXLI8{!m#8Cr>exD!qFOLel>@q8= z&~=NAXGXee_@(RpIpW;6g{i*t%;%>y@n3@V#oOx1e$}(Qqhsw9;bJ>=7_X^l3Aaw7 z?e>}q({gsY*-_g#@`K1w71N4uqtDNYMYF9<%Qr22t>*OxlmgP^@Jw$Wwe!D^E$%FJ zBHLvFTYTOlbK)TS_Z8OP%a<*wRl>aW>Z;M|P*mRnG0b4H0HqUGh>0N?seATCZN_^yF!y^R#39f_u zH@zB8tc@R4Uxv-X321;83P%nnIOI>Ib4b-3*X)$?(uS_q%F5HX%$InAA|Usoz8uzV z?(DR6-=$m9uo-!@gq95LDLo=%9jjLtuS^sBp-lRPix*mbJ~Mx&?cNpaK^T)d_dxx!RVC+}Z4z0*abcS&CF z8q(*%xNP8s9-&;pAJ)Ew@v;!bnrz&BEUM(X%0PmYk|*%0;kEncJSX)R!X0%ErrzWO zDqCj>S^og16Hv;NZF@aG8U3qz9%#%fP|y&=Dh}S+D7|R7g0>WY<)2jYE9l?h_O`3h zb7r*|2@7nix^SWHYvZN_LKf#K&s-29Ku=BVKay$ptOlCM{{VD4)-0Z8?%MvqscCg#PbAJ2#EAP*HyUwsU3huRcZ{awLh_Zw zW6}i0Xu==xUQ2e#E?Zp4|mQ1+#M7jkJ|4TRPV6o@mdwiu~R2BT8Syr3@)++=P@E-~pNS=ks5yo^R)0JBov8 zBoi0_i3YwYtZ8U&`P;Ta19CRjx3=9G0^s_FHcRc!uJcfG@xJcRudKjhmZ0GV|=>`K0w7pK4j~!`B?h+L8 zqkswhxUDo)i>s6uleXO{3R1TI)xNI1sy92zSWKl2sZyRv zCF_H0XweH=j(4B>deNFrvGX1*B`jICVT31@k=0(Ono#nM!;60GpqpN>edj{{SZc z0Et!I;y0`rZKWzoM`2g9-KL-{B`UcFOmd&?S*iQSuNzyc`Ij6c_-M!jx$G&l(&gAT zwuXNy>+LDjAV9~6wP6FS4hf*{EZbHT;eHR8g8-Umdh5?N1FnGy+LR$UpHKVNY+0hx z-A-OXQnFS?PxcjALMuYWf*m0xs6rCcMm+Ho_oeT=-Li`|h&#vBr5p(K>qcMePrMSH zIVyNW1x7uy)4eQXNxr&wbFKuFh3!66{%f9V)K<#Dy16k=@h|SyZWgsAylw&n(Tgfj z8%KuNz&-0*kj=8qyDue5nNS`5YM?0z0o4g7isW(P-S%`Ta;oUNZ-FHVpDahMHDRSQ zY)lR7&U+eL<1p6KAh-_z)nC+t+y!J5h{WO}99EAQ?Sb61hE6ROjiDi7ZV}$Ro4}+( zpdO}?IjXeUmh$0B6Sf6ljcqp|>_`9!B#yMNBE1D;XzLRG@{zoh1H|!HP1;hm69ca` z%`>Q_WS||mlZqiXO_@QJ6N8GX$*UyeM?#CA8z~?O$E`rNQBhDn)C&uUj0k`@`_m-6 zTSx{4U47k%Xx+(Vd5QZNqnXr|<=S{xT3(Qs+$vI1edvp3fK(^E@mGR(*ple&G)2e? zSp<+e`qxau{v^SfJONxroZ%oOKqC?>w$i|9DIBbi#dx-ni2fNJ1YF<;XiYK1e!5R}hKsZ8{N?^poaYN~E{ zq6&ZVDzGCb_N_q%*(F}J?Xg5aswpv820JFF!>!VQ1x!a3jA9X1OMZ#_)|+PsBCG^+ zSS126>t5R*eAba(iL7FQJ?i6wQ3JhBJJIL_j8~$my1l-&qZ)DRRVUIag>_x?Kqj*v z*!HaVRZ6{To92K`!m(Z|5yaLTsyERwnpo&StN0K&0*ejzuSH{6AZsV5t!f&<%+|Yo zYDY9^Qx6N$ycq+5L`2nM1!@R6ZY#+^h&UCYh{o9w`8QRSH0lYMsD! zs+CS_0H(pxy#`4C08H13s+Aa^62T*?s-zF*s_#7wP`MK(Xc(0N-m-z#sHY8NTpdb} zy$~BL4aPH711B+9sn0RmBBd^4BC&!}268_Ypxe?v6f#yb*0c(kfe}WG5RI~PR|BA| zLZS~^vR4!gZjv%8`2J|D0_Ln1JJt!I^{kZ!dQ=t6ip&xORx#rq=e=a4j(9aC+Q~~Q z22A=b`Gn2)3YA3O*E?N)Ut+k0s25M4PnP^F- zegXKWeZH%CYT;5;hg5vRFmfxASBC4K#{0cjJ z>vu^|Xi14AcCQZ{)F$CG=~P_tslYXqqUz-hwvZZ0l0gESi-wY*!*R7eV~Xb4eMnB! zAS9lo9M;S{;#Q?OiOinW+Uh$a8FSM*C-rXy5S)bdJ!wV$sdadt_-A$`4ZN6|&fSG- zMM2puN%WdAcA*f9R-*uJiYj)4!5~Xj{JR9-zEnuysey z{{YVPE&ilXOE&2ppk>fs+T2*q^zE`;!g9k^xl*uXR!uV!Dcmqz$$R$(FU zmL6#)I|}ObohhZQMHeVagaVPf`te0K@Po@Fcu4?fa3ociqB&yT=36Cm!U+J6aa_}s zibWy4t;4p8Ex5lCDF!8UVs&Jt1hkM3S;+mXk8e|WFJ_@+`)r<969@XyX;K($t%uzx zS4CugD~5F5&bVKvW2f~G5D>k|ll3QxQ*!ZWsYx<81Oxlf>}m@uSXvI#gB7jKi;gr` zXygy|aa^v^(*}vQymARaOQ0F8QSlI9?YKgrNA*AkmxgYcLX@VPaCHgtllP%Ma@oZe zubEmw`jFx?>L!+xMN>L8I?L(tE!zv8TM7gpdZ)_SzLKUvv2X}d0hFulT>C0^+5Ng} zja)QF;LC@@T1Q|fif^at47is(c6>Jz<-yG(i}Y*xFK(?vOL=>VyS68A@}0nXcdbR> zVfEcwZ)7Vf*d-==r;upG6_4UK;#@FB)eX?3{rc8V+irDESPz-6opUHd5Qr#b@q@i z^xaJ)Ut9kG4aZ`9ME-?{{%IN@~GtDc5WP zHV>&lo2D_jmG>qqt~HzOZ+^eW_&D9-T$0zn$LIWPuNEIZD4`E9$y)r%|_d+W3QZlz>#E?I4T+`%PrK6nKRW zgrud!g>INBS6^U8ee0@N+gze~`15gfBlF$G#m0lGg3FH(fw?eL6#(!QJ6F~I7x>R&tZHa& z2u;gGI^#(MZR#CJ9AppgTvD|!!Rla_EPL$)S!-tw~V6 zDeV>IJmP$260NQPW>SB?MSQQ~Tg6!XNZsA5iG8OK&Cb-A-MKz-9rpuW(;BF{(o2@^ zIO8v&1xZ)bLH_{As!tUp#*DW?H$`a--swqkZj~PutOB%-0PIh#1lvj1Ewuz#EG)k+ zu*#9)EGTlG#LZgW<=ft>$9QxUrPiF`U=#GuU`1$@KF`lC$pO36I4*7X9jRFa?gDTm zAIPM&Ee*A^=syGeVe^_*mZfxoZu2Qq7d|BJNl%su6-3Q^%i--sEo#>()Cf=clcBepFo<}k%6mz`^)=b*HY?V?7;{f5Q{UxDE4Fki zYm6o^Q|g%hwRl`~y7YPZV!2y4*Q~$I8(x_Df?j3CyJn&$VC6H|gI_h(P+LvQjg=>H zm4ty2>)-y>*k9@gSkrXN_U?+yK`g1%D;tR0&tgS$Z>5{(wDh>@X6fT@@)UNEq6i8- z@&!*Tx;i{-h@0U3RoAU8A8C8;EY>!jHzG)d4x=Kz(WhNL>X^88LV!z+xRpXmKT}Gc zsXSM~ejc)WMEH*DOVt}yyR6(Pw@Qcr&(g4ab+5JjHE=b3HsejXmadyoN>{gRC>-av z296h5S~r&)#-w*){{RN*YYr^i-n0R18Aw_QGd|Q!)%MLbD^p=^_(rwU`n*^nzfijY1T=CRbB;!q?Zd|1# zxFNYw;F3xHmGvEaMlZT`8?Pw2bfrZ}f=W*xir{rCR1Xr_q^ow~v$L@{HRI1spFxI7 zQAYL-eBJp=XggAfT7irT>GZ3G-d(ocuptRvEXS{p>V zTh>S|tx7=J08RxTb*Cp*OQPk}qRpQ(!~hDCC(G%Wuc60TF;YHecL|)Doo33<#zSg& zxOUqkZdQB8Va+_!-8yZsbuFyEaWUpnj!br;bo*7Z#~;)D%%dmR`P<2Kg*Y2&K4byOp|2LJq@_=~+e{~+Sd&~~ z*V}#Nl!ZE=JfrH5wMf)feAneDB#25^xc)j+zPvBwr_zzow&GtaQk39GJ3@vy_st1+ zc3EjiWlj;s@*sWZ>Y#3ZAhfngFA$jFvjtN~?lqg8UDPz)qRT$AKW? zCG;i2Et@M@`j$`C_o+>_)3W`-(w0(Zn4&GLU;6Hdw_qn6LH_`1SFCFY4XuaEl8mf;VBRVB+(z{yWNzPCO{S3 zrjk6~F6iql)D&c^08_Tol(i&`j@8Lo-G;y_aP;d=t@R|q+@g9!*Jddt*(ncgu2RVm zp_9SsU51Ns<9Vb&;tw_RbzE<7ZB9WI*!(*Bl3{8;YVO4?oS7n;JLU|80gO?J9#I`> zeY43a#&87~7bg|yBZ`6mf^#*X7_1WD9GcK7e)Q>yMzaE{Nb6Jz8O=n|vm(>lqnv$v@FEuG0>mkS{t_bL2 zunQRIW~wTF^H6I5$WLgi0k~$o6BTfp!9)u86=<(*qGAVHn6CuDHR3?#fJ)R&c}Lo_ zQN>_UJF10o#b#Amsa1_erP=8f8|w|#?bHBm;@7v$3|0V{uXuoy25TUjsmiYCtcvT~ zy{iQqZm8YsS%rheSSqk~H;Cq_VyLG85x5iUS@@js_BBBZsas~M9iDisl)1@NR3|lp z5SM`9)l}q)#tlFO^F<&QD+jMgs0svgUKp=%IRcLp6(*%Iku_d;$?sAO5=Cw%Ja9-n zip+zUs}&5%tdzWnIHhRB6BIhtlUgi}ikih(no2R6lw{UYYU)*IK*Wx<>8}p8?zNGN zB~i_46c2M)NEkJu4E2g8HwuBz_o)kvR%42SMB+Z?h*7X|Dno?uXl=(b0QNO0RN&xu zpaXS0b0(|1sm)@kAjMTHfNNi&jTEkGH)Eg0W~J3PCVr}09*ZdRl6}o$r4*bVYT-hD zs>McfDI3u7N6ml5uYg(jw@7=+H-!055OP7U#rogEcfSqZx|_CC*2O?kz)s`%-&XsVGrY#)Yk^HbvtkRorgHml$g8H3N8&t45Q#)^f zE-l+iY_8OS(FH;)o>Ge%Dc3{BH54dADNsUr{wE%pHHN404Xu$2f5>_%Sge!qz1^IU z^GcYJHD|QeLj^aCw&@BcMt`kxNyg~BZ_+(w@eQV{Y^|$pXMi^ILESos5K2&xN9p96 zSmuzXo(8}mB}POZC{WwM#r)pzpnRnj$2Ax1>y58L$t*fn97l#g!4rznt@f%*Y`PYO zWCa*MY7ASFlP(fRUi3>1ZURAY0p%mop8d( ztqJ8NPz?_15ZLpesik1R$TgX4G~N`2+pivEk($;TF}$!A{#%5Hn&l3pLXqri3oCTI zwSea0Jjo>eUY_)QwZUX5tE8#RZMaa(3eZ?tI+ZTMO}4XDv#|Q zYm?O*ZI_bO%Pl>m-+5kePSiMfQ~*f#qVzkK9=S!O{mE6{o~_LiW5s45gFKuHFKASk zv!s2IcE;o_EL<|56%HpMX_$_+$){Jn_;Xf8?UEsq>^PYuwCv!HqD^#~lPJ?J8Glc7 zvbRtYl#(H0N&b>3E7lczZ*Wq|Ev`aj^=;AKdMn<7i(P><*WsN8vh}aTmIGa6BG*z# z3BN)Fg}}o900$hUQ68T3n?t--k*m1TcVlIHlA|CgB>Hjopq|y-cq-+#nY?)bcHzf;BK;X$7fNCl{S$kDS*EIcMo14_QVWktX^bgX1oOL6L>>Obi zjRsOT7gABae^3(>*oydtif%7$ke1Sx^_xt#ps7pI`hAUk63zDafEUp0$w?l9CcY`I{5I*b zMct!tk1{t-6gimd-fQ%Q*(_YOeB%gtwX_oRaXZjwAH9740OGv`I(~}dHq5C^F!CKh zK=TGxaA%JwFh^rcmksR=5wP?0!Qbq!xf}!c^E9lxM#G_Zy zP?z0tZ#JZH-5WfPEc)@*&zlzs*oqBAkW^EXjkrOTB4fVl!rh- zGbzu1N}Q6ES02vvw{N5LgBq(EmX_7fZGSmzlH24XF15&6^-RRq%I>u@@W;njZnTB% zDi7O{DofVx1LCrMCyv;z*Wmq4+xrFI_*)sdO0C~-l(yvUPypxsLY8W}Cys-lYAS6v zX_aZ3fgo;Bk{MrMpfT%Frk2FVN^Km+#(xUg>ozI5(9&0M+7$J)BqzwfF-abVN2PN* zQ*Kz>>30E3Lk=Nq-2;+^gCY;&yG;exEbZ>J+xczQp9a_|1jbJG{{T7npxtO(Xj-6Y zvhpq5P2yC7ARxjPpZbPBGk#@li-CD22`yBqd(TSPe0 z$aR82X*t{uPrhsC`gg>44PM-M@RTiOJ6Dht3EF=1%@*sb(Di|FYNfV^(igQD3rP}w z;MSHljcK9(0M9XKw=6YexGi2%12+Lg8+SpL)~xaf{p;%5W&4WME$<$1rRBG_@v=6B z5+JB?Ay}`MTHo1fPZZxU504GHmHjInztXJz`d4e9KWfvi>2Eyc$+tokZnu80$vh}^ z5(a%Lk=~AMiBPXpbsA=@?WMBZMXJ=KDYbx6GcrE&SX;2GThus0S8sS`5@aPqq@E5b zkH!{%=LmJg%Z_bRZ)^j6jHO`BDr>>i8n$Zj{g&57A?K&*U+G*9$NHK{q}fbgq+{`= zlUI+itlOQcTYDZ#00(+UIL`)iT`q{FKFx*N(k(A`Hp`ACVbuV3l@271Vw7klSMA*C zTC-sqn<_Tfmi-p4QsQTnsC6CvD9?v&oVK*R(ipf*HiGLOB&kgloEXL+jMVC-W|XcR zU!&;yLyX)pdgE=jo7{NKAfGDIAz198n5OL%{{SA@xFrcCkmn^mN2PJ>>Mz>dxciAo zaZL+{3R4~-NC|Kbr@!8wS>HFQTQzvj<-6i7qM(?V5-@v-73xDGt4EC}v{~1#Kg%As zQ+@^04xzvtOnT3;9cw#Boo7JlopRdTKBVpWwD<`MBRL^aUn|$`oekb< zr3HmQy`w&5Cv2vDF`vDCM$+hbmW*x*De+W1r0wWGIi#AFljGSsggdm|g(WITgtv^S zgU)M`a8udcgD(vq9n#zRyVb~jt6OVuR0x+9sfzn}ppWN){ostJL=PqW&^# z{I^B8u|nHQUM->blC7s`GJV01-j)0sw7l?n=tu?>SIkn zASfjX+^OPgqGHVs+7^n#HmO$Yf}=7yk8@mRk8;6aI=2)pn@;FS`pD@|^m~UcEeI0Z zWSI&kR1Z`C0G%u7yn2ig^Et1R3n1&)K`l0P`$UxxsfbTg8Rwv)oV~TNwRBkQSwY*p z36J-okoU|orc#tP$T=zJAKTWsUyJ@VV%FF)l^|GKc}h|_`-92!r--}sGe zE*9!mM;lfD0D5)c=E(N!Z>@t8q>s~bK6781 zG|hVE=tFNQASg4jpL0yn z@uKdh<(A#DcVY=qKb%*>H2p^6=_R7#$vuU27MgR(*_R^r$2HBn(b#Mzr(c+u@BZ>xYvs8GvW(E9lH=wzkYI z;VN=BZrR0fn%~0r-w|FZ?L}wgo5)E93F%jOsarW?GzfbQC< zSdvGSZK4}-EjPKvLHyFkZxs1bl9dvEnJ`5Hr(IHsazXtd98prI(KicQXGD!esc?cq zI&SM-2A8O^T|!9-JHF<=ad&lc(+Vn-%z1Z2(;tVNakY4ktuxFV0aa7t(I=1XZ69Z~ zaktQ%5g8IPY2KLb#mJC7Yl71CAW+&vfZz?&&2%i@Edmmtq9+qxo;-=8&y;;1RnqPf zTM|jg;+`SWuzObz@Z;r3PURDk!LGrDZU=FL>BV}O<;o`nqs7o;WC^W&L5%T1_<(hq z@>jU_0fuvCrNBJ16`)-k1q2nvP_NdqDxQ)n#;{8TN#?GV6Icuy(Gwid z0_lVWL6g$16b}`WG7V~xL>-m_YQO_j1~{z~=wgToBWTYeqNJG+e>LQ$c{PlYSoNz2 z?^X#Ns#6itvM~Y(s2QmwbX7pb7Gf%eP!4#j5-Sy`2@@2fC1pyhCAou8wN@%1gIL6s z;;~bQ1annVu&+wUs|n}rTcGq@xO$QP)gB@4Pu{UqfJ{%`yj25=76fXg^P2Eitq;0Q zdvWbr!Vp*MRSFf3;6_Y(Rd;sHYcdMm*wxarJk16aLCz~wFC5mh4HWbhR$H_qChmF-W8Hq8@D;3

6i5HGG#tMErAH6`i+!Qe#DM`AK$UgN@@SacDS3AmdM;PEbqu235YF3EUEu3xC zr0#$vGn&O0Bao`yH&P%GNabPUN93>Zm;5Yp+U4u(29DBB;Di`fNyUDBxVBZZibI2G z3XZ8Y{Sf%CPG?lKbUKa1r>Ktq0PkOpf5g9pm8EJ9t)y*2mS%GquRAPwJ4dnNW^Hh1 z4X5fZS}a407|59w^h+JL)*IKcM{G>=ua>PF1Q!-(^>4(XN ztpIBF9wX`tNC^t&CX?M>Gi-+#U?>nipd!25_TVfn1w`gRu2%C%wsk>B3ds`{(6r7a zEmJtg*$%Bv+%vf;I4b`1HO1pc_--R(0K$}br}~zecICoZwkc_-^C;n1_s0~!{I5p0 zeP1@!`hNpuaMt*xwZixC4d{wfE%0hw1 z-iUunak*d22p`I*rPi8-t>8&Rt6!IyCa&{)R8-rJ1oTPZ*DUMcXy2lj33ZTVYAHQI zpWcIeV}odOJxY%?TlOC<9|^+%WKIX#kuApyg%2Qwj-Xczn@>k{BYO#5qsVuA4y8l4 zde6mJR@;^Ww;iCM8WD4D+MRcKN`1v-#-zii$mZ2dfK`f)>dU7?50*j_rH2d?Ar4^H zw)cs$AZ-J1!AJ+1grQ4HR+N_#QzU^uYQet+=M>w`IF6GM9jH?FGIl<-dcxtUDNBUm zNZbMZQD|<(xoD$mSCHGOl798eKk(J!p;lKRa3q4Dl78ZhX7^Upt&+~IXmuMv01H&8 zpilF26vLw4VrG&7+aQaDn`PxP@(BpR+`gE@YdnAc`ATbkW=}uJ9ZvmIbzyx zZIESl%2`T>-nO>Xt=DcLyRC^(+}*-MiS`PbM}CKk`3QeTLzc-Gs_=KB1;bE5BZ(Dl zDH^r2srhvamXbEDKn=+3t)Gf5r*7@oLhKo4<4;<*Y;njVq55Qs&-}jO3RAZ4j}1hS z({`#S+!AERY6_Sw#>?K{DYYqg7VV^P>Nf2P924)8N>IhHxnZ&sb9HX78rTmbifKy7 z3K%)W8f9;&rummGZ|*eboNU2O!d|v{Ky$T4G5Mu>u8R9C+p~JBmMzk=ai;1i*>IMW zIZ09uQb!<DO6k59_=OhNj)tynZ@g6Odfg7n$mp$x2#)dAw$`-^bokenZ6V8dL2cwg zI|=BzOaUa#=dOO|^_k8BoqTmJww4x)Ij zucsNSi|mgnM^f~CTj9@$nba-quCJ|B#^si-%PI;DE56>`9&713RlC3C?^`;i?^3iB zosvS6`d1u-?_VYOD$#nif}gf6?oitJLX*3x1xX&UF<(pgL+61*7n|C@x8A18fg7=u z0~rUPH9T*OnrA+EsW*CXW`vK-rHAkL9|$0c;PQ2ZX7BY#Qj+0S4Ly}-9@mV&D&)KwWld`90H!Z zg?cb+ioe+M z*9&JGkbhJjmGo^Pj{?@oy+zQw`&6h}UL<*!YT*(V2hfC?#*^HF>j$*!}@P*~BrKVdYwHNf)$bBnm-KuUzN=Z092&J~Ep(c}vg;#evb;Z+9 z1RJKFpC_0v%bltiQdS_)_Ma_y+sb81wo9rzH>2rraG3owAA0KkDfkV`a^=p7`)ftg zPs*v>6g$dLpgxqIhoy3w2}R>hHF%Y_UGhG|*eyB12NQ*48d=5e(XmP`Tu!S{Wlrh3 zU&NNX!;Cxf8ioigl0k2G_C2W8m*UG;wM$PdysKw-whSdg6i-;4rnLBVWk2T@_pKKi zyS77jWy9rBLZuD?)Z!<#Lm=+Hw`cfGcwzjULVTt@x%<&jb<5<0r@lWQ*edr?TPe2w z5JGL+4k4*3@b`#Np51A+jq2^x9z1A@h25t&YNA?d2ev#unjs}ON^-S zP^0Ha_2AdO8`M_ziu0vgl-QB#CMVvLzp1CgLp#6;F}L)J6HL56F?DTjq_)u_4+1AN z+^SYdLAY@`Ac6j5+LBO~9%)Geau0eJQ@RQWVYHxm2;5Z%*YawQJ>j&~!eLG}RNBuV z8brhCyt}tbk`}G15R*KUOU?8c$J_LCI<}nlqoS?H2rj>HrAaffsDKY{4J!C`^tk29 znE|_Y4}{rHWG_J|J{!}Emt}-6X>9n<5Md+jOT(mg zLoLPG^cy?$-6q>hW6V@SmaVX2c>9w-YV0pH6ftbtLW^^Qw1@z6{VV6+4z_LeTg&GG z0sv_s^UPzXdgA^u{{RsiL$0MS>}@V8RsaP+NWdq&*PxqQgcz#M5Q^)ZJ#_(!VcU7g$EZn z^D8x#$M}76i42t`T_jC?rq@qxJE1$be>FF-(O7X~!~y}%PHQhSCDd6Zg;mKPnoByD z!)}tGHvXX)uG>M>{{YUIKxlsx0cq#0b^ibud^c}$R#-?#&mZ@%n(1mk3+wH|LYQ%3 z3(uD)nzG3sCN9_9RxRLZr=#rO4rvI0SF{i$3FKGM^nE1W_=)gB5)t(dE0xhw&xYEN zkOGs)>0eX$a>^SDX~mfX%6j@&k&+hap2ybJg4ui|#jU!WWxdWU{X|hNX-Gp~EYn}a zTm>w2C#83Gnr7w`;y4lroEVCMZ))0#0L4H$znVyOX+Bj0B5H5Pb$t>40LX0Ttn`rE ztR^K5M9))9H8riow#~Xg+(emz6CLRHC4{X{stL*MUbVI_wr(9LiJWm2lsHAj6LWHQ z%mX2&(AIXqo|*pjbFswjLJ=v(F@JZJT_94 z^--jnt%7b)qVR$&LP`0+JZt7apfJ) zYRsiJg{etOkerFdLU@)fv#TCgmq3?S zX5r$fNCShmO-m1{By2GUsEQfoBHc1zkJQ?5Eh_w9FM(rI%c+;e<&a-F$w0aizgRHE?j=ku;_P15RiEQ zWY=kLbdrS0RPE-zOYr^ro?0*nBplb%H0Fzqf~=jr>)pdP`aGM4r;aGMFp^esIN4=#z>07O=!_GRwgh>?Vn1v;yFLHVuOmdYN>*Y z?~b!l7a>)Vp4Fpb0=rWJiwyHpt_i3L#X^=MD+yl)3nEQsFlVh?zou`duRFCkdtvsV|kM_K{rxspw8wan7z%+Gq}l+Kty!RM7GC+$P2;6+*TAmUw-7PuQ?kk^Y|FAg?M1evj-q|3 zGdqf=at$8I!IL?si=r}(8(*akFrc8N0GjxR_>u7QdY!xbQwmJ-u&0payK`mc~+*Z)Fc~7N%Q{%se8oCZG zu$}$;SH^C1_w>uOt6@n(f`JN&&2r*=S?f3TBctkfX>qjgKvLQ%%pUw#JE%T@Ww27L zV~Xhh8ofrPrlc&Ss3s*xnq91ENed+a0%9O@_pax?qs~;FGmXDOuLw)qjbmF=lLBh(46bhgbJjHYF1eGgeWJs7bqc;{igjxf$(k?C+=D163l}v&s zKK0z_p93M)p~qGpgUZ5jNVK%0!~smHIOHe$*Keh;RtJRc)Pf`p$|vzfvo7;j-P7Mhb}ri zL-z+1ym-B`ypz&GO)}N{cLR1zy8Mr&v?*qO!n)Qz9{7U(+}t&zDH{iy=G2RE9sK5% zJAbM?hY|i_`)tg`yWSA=`at%seRjSZ{{S%RUoOlAx~~>TsV1`eUzhy9>{rh8cS6TZ zZ^QopFxK>KGVpm+(ymL&9mxn4tJ+S9w3mJ)ySQ;G^EY9oDR?}2oGB6OTt|X$0)Z%6 zLUN|g(w}fDEN$dR_Ia%G5MKLx@lD5x*ii{G8F6 zKgBMGeq!i7rriifh=IAakM$?LE1O+c@ucS-4jSdjz4(eu#vlEgNMQ=JCu}6dx~Dy#0A7Or`fh~3Rcw;j2~Z0ZF#9_7O8IQ zQWv&NAnjAut5=yPozP)P_p4&zE406<%{a|g-}E_cN8uYyT&?{lQqpyHLGsL@_D^lb z4%3WMX7v3>MYBUT@IDR2?aMTrA%Ca zc-;hrC*G6lzY5&A(~jvHgMsU3H!P#eF@u13KGkQ9sM^~7#ExX$HhlA3xVB+mF#ET9 zGusGp9vOPwTph_$Or%e7Nc9$A+JnRzzz5vB%dvcxo zIP%3PpH#ThTfgwG?f(FP`AOc{SPi-fnFo}swP3~S*wh=cxpDRBx2>zfeQ~{umW=Es ze5%_X{Lo(x>LsPQqv|MHF4;iv)Km%w3VcP=d^zu(;P-KR!&_YTM@4Zbt#e-yXdmWRKO*Ob z;ueJ}-PERKT!2CEiqh)h<=e!usTA7kwxVq=H|$HLJrsI{rz6&hQ|#(iPA%6JXL!@H zmy!Pf+HesPo&;iQ6JJt1f02A`<9w0weH=LEq{4L~8a8(huvE0|@QS$-dLE$Am#rYU z`;e=IrS&0%r7%L%^(1vs>+Eak9~)?Bx%tgmiy+U;I)rKrx0HCD9Rt+CF^(&T*DM9c z&s{T@gTSp#sDfMx+=b)xr~*CWxL!96Bh!XeJ)Hjl!ygf~rD_{5p%yDpQ7b_qBX^ZR zew727?tUrMn6qNq^`_VUW2D}wI@nMJZ7T{;^fNWf>pu;%sy4&TEw)RDL&$f29&Krn z#sa~|t!j-)7fc{(R_H^GwroqTBWx^TDNsB}i2GJjsMABrF_Yw;&a2{g?Jk;URJHL} zbjxUPq$CaMiAtmSh${S76RPQo{{W2BUbk~phrCACM1X??q;?Qw9)v|R(wknPqw5-5 z=}^?H?NGg`SDV8MBXB1pZ}h9{Z}=LGyXaSgVk{7s8v;RDJ2%Iv6YLEyR7=_AT%zJwq6X6EkB#G6Yeobl|Gq{wN*1h#XOtC19{_fK0I zo#m>6t=vY}ns;}VX*{AhgWjDX#ouXW()VjZ3m~lSN|Jw0;q<900ZF7!gWpA3=+;tYJx5 z5D$1155iXtZB2%@P%cogl(tI9tl8P88G8u@IlOa zRV{nkIh1#b&LfxvB*&Ke_oZ--3A)6NwRg&t7i_?C~hUQ z$`nT?2pH{1eqC1jn;ONUn@Vq6+jC}8qz9Zxg*w?kQ7Oujcrz3}lvQi1U0B*)J9Nw5 zR+@L7BxCu6(N!BvSNH7}v~&E!L%gy=o8AiANj#`1$d1GwDBEYh%(SLlZCh_G6r?IX zqB#@jW9?kcyQbdm)5k(>mvZX1pd5k;Bz=uJe&se7e zW!O^ChpoInG%O)pl>?APFVvm+JxPZ`!fngy+Y&yZ0D8pMw^wD2NiC&Haf_CNsQ&=@ zZVV|Oh%`IMH;GlX;#+Y+^uKQB8%X4SDSCS}cG1u>@N-XfO*j$fD+y7J*rhBz^1AvP zP;7X3+BhVDMN3+PEjG%$5#WWCEhmvYR=*^)YtGy!ZK116$8GJfwv+;YnonO! zFXCJD+v#ah2}01)eA|AK0T`dPbjAuhAmrU7jx*vPh#UA`#^DNhD&T+&s!AXX6W+f& z>UXbST&>qimAOSp9E_Qo=)O1jiCW&M7;Zx>44wg3TvEF|DchwXyGF~dx%8k6lU{}h z{WZ#6v!6CZzSJ(X(qGh{Ll)^SI^l$;Y?<~H<6rnb_uxZkajUwJ-QUuH2vF)z*jMOJ z@Q3^yztY}oT-KOMNC39l#E?GXzD@rC>o<1&-^I+O+1YA>n1}$XKia*#7?Xo}J@k(o zEdJ%hW$<%WTJE0eSf7|=Zcag*12p*-MJ^4w+s-1sIsO)FYtyxl2$A8aNgQ>rz3OB( zpyNRW4ySN4*1S(M6U3c*Js%ca`z@y0g{4YdBqzU9L^|McB@74v{bL!WmXElw0Z{;v z(wnrfmUa}0lNICrTGBh-E;|<8X~}uA6bVuEua19-8h6Q(>q$sbRt)yVeZlgg0BFy8 z`Oo;3WE#^JGtRPwTsZ-;=99nzzeoQ7hkD~Ti9*^} zh(X*_+yP&ZJ_voqvhBNQD)T`-@+f=O{=Ku?@P41li}`2N=Xn@tQul%7joEAl%|^YpNR8jjO+zAZxiO;eLZxaecH~sMf<81mp%I%H1(iT?Ygn2}t z&riK^+Jg;}MD9s2Qx(zK+oh{vGvXLBO$_Fkk{08N3Q;qf>&V_noOtRIXAr9}>i+>v{*#L2jBTCYReCu~ad4f} zwJQEpnqy$?!MFg0OmjwfY$JBim3BK|RTFUH+}mu_p5W~to@uzH>@IFzkEMJsaHK7% zBc~?5xue{bGLixMc_${mAMogPB_+ZTXFF&6*VeQfxoCORJhAq#U&fw~D>QpL?+}1^ z5!}_#spb#0FKJ}*N{{mzF|FK?l1CNkr)LAubA0nOLOG$E&$SlPXWpJBbZnC{D@x5d zpGvmunj+nkyNK@<;&OfJ3GZGQ6^*me73rF;>VU9%)&e(0XpcrGY!5}+*iiUy#q;;NI26)m_ySSj3ZA@ijea z6+J5$@KpZ*ziO`GP(dR&9erxE-mp3^x~N>mdyrzYA}b0LS~Zh{R06a>%~3iKEdsU( z-O`3a;}l995mijZ!%@KOA%uN0Kq2PatzxE0Z9%c^ZqH zdsPamyZuE$f+=h?dvL7QNf^Pe9<@P?noBX@a%VA7Qkt)l6Pa)my$y$wjPy(URuU^`(}ahk^#*6 zReokMk@uu|QYx{tb+4rJSU9Pg%}&WOOx9(1iszO~FBnnftP0Gycn1~bFQKf6Gm7PN zWzY>EXrg|Oz;*`^X5Im!ST^5OW;+iOVTpS6QuE=DxPcA=Tdb2>RouKEnE@w_k zlpOIAkr5Ll{i*(<5R{Tm>`ilz-lY!UE+l1Y0xHX7lcROK&8^Z+y|a#xKq`p>n%SF& z7KMbyW+0E)MFC{$luD3WaS%i*PxPk<)D+ST6d^~VR%F(@7xFMZK^F>$>s3|7uo6LX zcqWvcQszvw2h7;5d23Nv8dbn)V#~ic(5jQ-LR)xQYc9O_w)?DI|Y4 zU(HRk7xu*~TV>Y4jmS_x{vJ&+@R4&0XdUq?N`_{l)QiWDqb7DXxH2B}Gc?V&0uZkN zp+xePj%j#rdz(;&H1Os}F&P(Bamx*dzT(Q z4clS06Us+VS_vq5FNGt_(MT9DW{g@A=~Hior0w#kCI~+CgZ>6>S*gUY<;zaE1g3n7 zK9Sb2wrOp+7V{n&Q9fzPK{+1OBM(1r%s55D(mbkJi5~RM-%x7e-3##csbD3^ zbJ)R(DIp!84R_&pb(WgB)2`aw(FzOQgB>wfHn#eDd`GSlNIqcE{?uD&xv(l0jkM!i zq@e~RP}-K9i{veP3u-CCNb<>}>+T9}^lZPBxwkv+O478GJ0d5fkMBuMz1v7yw5xR7 zB#@N&P9;lEtSc3TwYy9uO*&j|M%2hLyZx%Z0$lRjK3H&lDs3=LCr--6dKp{0s4Z&? zUeUWLQUY`9Pwnn?3wD;}y}Nfd?cv50ybya=BWZW@F1>4drCltyGU9;aq?*Fk_U}r0 zX3EJ8rD+GmLWbZ+LOO^djCAhJC^;%Vg<-4N2a>kmz4&!%G2uACeYCH!?30dbtG2&W z!B4)^A2B;ajEF-20M!D%P~XK@R{I>dbqZm@g5KT99cFN8wWr25jXv6ue%_$U=N8B) zeFibP06`wr*^espdHHZz+v?gap0#OlwWT!sa;>2vbIBbHn&F0wf*@^iAKGYre^$~p6s{g^*HlVWq}e^; zzB!bgxK&dmBRJeyEK`;@O!@Y=ql;#pE?gzeIq@oGXi|bnoTT*TxW{y>R$K6_@9mIM zuP=_J0NCtIabHbwT4mLv-z#};m7xTNmahz@U~qRqHNb1FGTW|Oy}2%J3X$~6haiti z;3J*wb}?^K`c29YVHT1>lcx zIrp!bs@GGq4Ls(Y_2rl94RYV4gCnOs>**g34O;r#G*NMXWR!mo#skJ-F$Dzjr25qy zTG}T!X`e>Dbr_rO9o1FbU{+^rm_~zW$}7I$vsaJ0sz^-yS#zIrS22 zmD4O=Ye^RNNlJYaex{W?dq)*m%C>pZf@8!u-QBw^ zU2hBz{{SfH5T@LmB}bjH---k0oo)M%nrt-dtvHtw5B$aOm5$@qpxmu&T|DAaF0ExR z1fo1-4DODEK^2~;>lbOUP1s9BH7RtcLW$#yfI%3eb)ZHjdUm&R^_#ZW9(#s7&A72X zD|@gHw-uecR(ckV$Stgblj#ac`MhrV_So7UD}w@QZES5_cdHa6#;9nq&7iY(H${jr_vVxwWWD9wUvzvFMJx_Xe6PR7Pv=tA2_SaIYcUmJE=oQpV5NT`7t4(_S)vO^G?w0i?Q~v)4I`#VdfQX?pHd4qy@^{#5wg4LsqZ!Rv) za+Q?Ck^fo{yS3ydjVB1jw!AVA-hTUil&MR_~2MPy2(w-Iy zmvs17)~VY)J5ZV>yJ1b-Hg`7S_eWx(AJ!?&)sDv&qq=$Wj74?ghL1ZWuSj30T6lXF z%oipkj-Bi0U*hk?3DY!HtddvZz$YR}f&DA#TC?jaXoM7`G;%~vO@3wn02_6`^BqB( zM3i`FBX-mF72A#rCS?yq`8eZp_&A*nh25;U^r=n~wR@WVLH_{izy25>j6NZ_yjs|8 zt0v_E(mdH2&s_0eo&FnbC`v%ws6DIhKjH7hW8vFHrL-%}22x1i*HXz3{DkMyIAs|) zRC_;7{9*BSmGC=nwW<`CS_(*MvI2=1fsr-%pZrvH9~TWzP(`_E#ksu+KT<#(V-?^0 zS9$(;^6m*JZBjy1W^?OYh0Ur|%8){c{W$)Ve2?N1l+uIO_ckvPaBf|n0MqVQV|vTT zJ5+@LdSbs%=r?0j_-UfrcevC^Bcxa7{{V~r9$uFDxImT`V~?i3ss0`6Zl^#>ZPmYc z*ksIOwO7o^444)~^e2rOMwx=ROI zNWcn%pIZ4zRp|C#Dj2K5wrlulNS=2Iuay4)iZ@7l-AOw@1RbNaANH@L>IAsb+yVd- z74ct>?ojF+$x#PxKWgpq8=gc$Skz$T5B?bGZ>g1BgaY4-JAuv(ew6qbp!MPyNm_vK zUnu_o4m8x-d@jHvW6q~}k>0-1{vTT;T`E?s#ECIo{#=~heomhrDj0v@`tF6Lw(GL# zDnbMV4CbHS6r;$5?Ghk!6uU=48QR^tkBFfvClD$%8tCgHCQ1Q(X0 z?V9s^sq*%A`0#c^FD~Kbt&zx(t-MfrtImd$m?~O?PbAhi7a@eDJ~&RqgS09pDSEX- zaQN`f@{u?^SD$IBJp~tbMZK5Hwp@6iI<-lTgSB*8Gf8PX#HLR@=`#hl*W+9(a)A+v zG~&_ng_E?UuyHhEx1xDnqo6l|Dh0*mJOfxJq43k3# z)Q_PiKNZsFM-qao4qeOTK)bnqQRE#cSDU!JkV0D%oyo`GR4VDu43YsoH&zW?-YIxq z+((flP_^3`Aoesi&;qU;@d)arLHtqemaU2TWGPT0K*a@TYWuvWc_*o-7TRjmPs|h` zPt!bApquFulTr3>r`);XH_y}~``6RH7<7~a0yyHneQcXMMSwgtD0PV+itoN0yWT)$ zuv3hjO?Kdei`zN!Nj0OqWeQLNF#|D2wgN#Z1d-TH18x?ijp`HF8a1G#g9Zm-Ub;t~ z^bp#~^b}KI52|DyD0YAXc>}F%mrM`_JJm_qzU&Zc3CZ=WH{=PLlH_K!7}`Kdsuzc^ z;;>WLF`B@S!br^V*kVXON+OaNR=ZeTYN?;NrxT!g$Q9{c)xDYZrP-}352Q_CM5Ma-G1;vT#KJ~|v zXI?l2i-KxANtmVO)45PMGl)WGzVqPc^5AC|SBRlhDOdNI9zR z_5G@)RHp3PbY`(iz%;_yf(((xRyh5L<)Eunr~*=GeNAOd ztQj1{QzrK9h5rCb9`!A$vp%A!#*xo^J~#1|iFGazkvWged^=my6dqpfy?R&aFNoSg z0?JBw{Z;YJcG)UU7n=E0t`Sh zYnFD~dWiCnF%wgd+U)9oasa))VSn=!laBP7=~l(Ew~|6rfHr_4x^{E`^2*$hl==!D z^Wl^{HmzG*P9kD!vN>ZO&Q}KXjv)M0n|1hh$XGz(WWa;ibfYbbF1RQ41;%Tj zPr=qIfxbxTAc}8q@coseMJ)ux%ovI?&la?0iLS-WxEN>^IH=pnGyBlDH;Q$#t8knj zdT{lfxs;zU6bh`EX(TkF2SP}sQ%!a=jcKCRH}^|xa{!@fBXUT^HL|&V#g#n5Qm{4^ zCj;?FT2m*$gRY!6hl4d(^tfj!x0m znJef;iquPdo$6p`5Dr+S^!rJ{;fu7#I+1=)03zii?z z(dW0hLrM(X4(V9oUiF2&<)U8sU2xxM_efX_-P*Vu&t)SMHPqW3M$MO6vRmE)e~Gms zPyqJp=}WI}H0^E0Ij>nHI#ICPT9VrFl*)MbH2i&mPtnu-E2+D5uk!b=bpHT|T|qk; zZT|q4xLP zxE@2-3XD*CbEvm%ZE7e%v2$o~l6Mr2=yEfjhNTvk&gPR^KBMr)xqe=0pNN@O<;p|k z%95b~jo+vqMRb=|!OOQVG`CQjpA079wI^?l2P%&774y%AuL}w(0jte0Ed{lv2|)rg zIR?8xP?T=*VFot4CkmMaj?~HOjwz~@ntVd##iI$-?3NJX0#Nfuf;gGzVAiWFUg&DL zK=Pm`dPY3-zZ8$eD2q6SQ!33sz($e@Tx(m6wddpw5 zyt^BehR}!KkNNF`o!+L1)Nd|y{aq+fZQ_uk!jOcY9@MLa6ZJqO9@M?pZXeX`P)b_2 z)EiT%3IQY%HmL;k&SJEA3A)oQn79Z{?vJ^))t}-ru@Is&)Ay!~_V1=EjU?BjT|tH{ zlY6ju`g@B*jitwDz&Q#^Rr*p)A4sik6Ki-`wbUFIx`9efv&S&7NFb}IGyBsUXHf2( zubS}HZW!>c5F3=1Rtie^^`$GfQ@RiNMblx`HRiUtNC8VpGL-|=f*^`r zG*Mb~4eMn!q!KI z;zSiBdW`+WECDuF;TLIDt-_L4$WX@cVQAgku6xmXW}J&%cYAx4Yj~J_3nYZxCyk_j zl_wv)JY`y7$p-d^R@%+e7IuSBKvJ21YilA$g&=HM?gWskb*<@)BKez@q}MB@)9o>8r3PExy9YeVD(V58f|(%oiazPHqZ@5FxNfwUR9$7_qOSof z0KxQ}M?xu+wBy+ZwweKp7us`_!b; zWg^>0KEOkdw(>Uki6JjGGX+3k`b{&^5?i-=*)7YAI$62}(+XjU@DpsXwr% zYC(3@oCFnu76&sXNAFAYwr(v{&BI`+2vCrbI6T)*EgHBeFhF0J6rK%wm}6-2GRm39 z>uGA)mk|mu-U;nB_>1G)<=%WvdX*+T7Zi`dHT!SluKV;`5crAD z8(F(qleuJx=DjZtDd%_T5YHyCU*zb%8+f@1-?wjLn)<)PFQskxZ<1ztuZCz>qxzM|3eWvgmDC8Q`QV8O)(t90Et;rL<@m@5=j zrNUliDhpDAfEc2Z(FDw-D0zDt@FkJ-dN79*n$>hGE7lyfxNmS!#w6zyuRzd~@Y@Pu zhYN@*^6u$fqt-#(whr(jL7~^Cs|7@o42dyK{{WFL99z&nlD=Nd5K3AK{vigPX-zop z-wiubL`7oVh%AXFdrZYOwNJ`h5=ytHAR3CAeGe!%*&45bX=rj z#0=S5LejSZQBFhy71Vrv@f)_@GHy1N1Lczgn)#NKs65rfw~eoB=LtL!>t6SRf?R&1 zI5SEs{{W4D$@&V~!B%WA(=QN~&=Ve>mG)oo^Gxt8k#OUSSdl5~iuiB9S5NB2lsf8) zf`BBBX1_>$H){HAklSJtAeHf4xm#Iy(ds(-HlJr>Z+sAy3uaFJvA1{kr8>p;go5s( z;Rz4|Mg;U7%^&8oO2aLJxCtt*LshwHhn4d#BIe}fC{f3_71K8^HqSXFdJ}7x09&p5 zhYMlP`Dqx(SvAO8IyVc&R2q;K2sPu$ zsY=gV6kKCK8qLB|xJcl3=C7w)h9$)q3mFqLG^XO=2)ijOE~|GVA}e8X<-)&&PlldP z5nT0b>Uu{*Zt&iITmTXxRn*q$)I3#`l@(?%des9?Kj%OR&I(wVagDEhS=RN3qJO2P3$s``R=9gTctw~Ikdm3S+Ss@AwbQ38U6|MRu z#VFew3y^>1qj8Q%rWP$WPWz<<49i?nfQA%u8frUH!%Yi`w|_gpv%f#qgusa1O+ zcAXtqzGWzjRD&1_Dcd93mcM-YSsSGP0H9P_ujY=_Bg09EDiBRHMZLYaFrmD5GhL8w zYq~iklo&pk%9OOJM3Xa-Us3pN=Gp-vJCzm0>DGlrEkVGZf-AlFe%T372?StQb_{j0 z6P=x%t;*DyNZNh4rcErws(}anv{@w)-kwI0=yNOtj8bs zsG7hxpIVfvIibRg!NIG8*0GZ|+z$e40$|mE?^IWMz_JReM;WTeenm(%fd2pkN$1w2 zE19f<>QlNoil9?822DXCF;FM2YQ$j8U|x*Evru@c2$Nphj8HOV2>QJ%$;{1L!<-6& zk~!&0GFP)i;1kf+iGx_BP;t#)c%3{vM`!2sr=+k$w+&@Q}8f@ZlVkm-gqRlfv*!K_zuMC6g`Xl1(tCp0UM%-0;U z=<1Fz#a!;0f$V5!9Fk~X7?MCSS4MqmSIDxB7T=#!TB3R8p(yXgYLDBc1QS%R996eL z?^PJ)psEL(G(`4EayyT`KyVY2LptCb53Y+Z8fE9Agah$2!{K{V~FtDgtV9BpF;Mi1VZI?*Q)FpO!@N^re^N}u_N7fTJt79EYzu1_Z>BZqz1xPx{ru2kw0; z_YK;(Sz_&Ovu@H)fo*J!thJn!r6;fRb618n{{XSgDf)+Ip*BL{fBW{JD4U27Q4ge1)ap2V-{{UCl;L+8zznE%Y)2;z=y*~6M#KVK; z0??D+Y5vr>TUy$0Q%NXU%2u=;zLF{#)Rk_8F7nZBxV8TP{yIEK{{U*}to0RU=+cTfH4JAVP_NR&QNvFk6LNy}uJLzCF9QcK-mvDNMq-ocPv~>xO_=CCAo&tmrIma~m z?%|7@a20QNs4#~Vq_o|x=UY@x;!xQQ3Y+p8%`Q8ZHqE8!=P7ipj zbiF%W&;k>0ay1Kl?q=i7Bcz@Pq3_$$h4Nf=m;M~SSrVRFU${5``_&iYUxr6*{Qg9r z5_~k>&x-3QSaEIrA_0l%-_p3TVR55f2kn_!(&9o~4haPMiu)Ftn~jTU^tF7-RCai3 z3mbWqE2mr$x7M*>va=}0|ZiiJ6)BARt20BVmN(P>MPv4UG&!mjK>mXiMfnWQbU0vs8L>_9axGN-KD zZ7Wh#Bp@Knl=72AXy`lBZ2E(XLPAof73NY$1Xk#2D0x=Is5o2>wt8;pe`>OaDAOU! zi#h!|;a3)B(cO(&Afe5j=$QC%qKbl&f!>-h#GY z)E4qkIcs)GN2KP5bxO4@m6Fr4VM!PyW@FS;xYn6e@1VI5Ue2QVEJk|T| zTRft7BsSudNeUegX~Y^Y)d4C_#`;#pT-@6(+Cl0g{mGuoBdy6%~4 zpx)bfPO|&!V&JJIKqQ^2Pp42pqBQg(+f8LIAC4Nf(H8~n8Qs!K6rI}$o=K5IHq)0o zm;^U=!76FAtH|R^`jAn~f+lMfCEzsokl)TO%r>OLj^|n3DKdCg)6tG$Ja08N_03#A8YxXZ~H1{l7 zbh~J?2?fO~_M3KQMC-X%yqv@AS zBURE~NxM@a@0W9M4nc(_N$Uso3V6w_5t?0))v|3Z)u(C=rtua{1=+lbDlL=1K9%c+ zO}e~AD7u1>pg`O}=&0;Q4H&T9)w80-Z(F6cZcAjzw;cY z6SgFU07S=1_k3k*v%|?&eD~vb!`cnQs%!YkL=+5>9qaHH#SE<)&DxZLkhc{Bu$umf z{9SI9*t`~{v=sA-njkCK$e_bk=WN~k1jq%e4%nfmyZ7c4CCw{ z@E;zgzJaA`2wFBRTS4#AzI=;iJld4O+@$i3zVua&qPA2+irQi(yF8o=GPH4G$nm;o z+Vt&G;>`$770rB2@ujlDl+&OSFoeLa z15r2HlLh6dp{Wp!*(SX|7r^CCdPf#-A*)c?I_|G=uih=ihZN646FH-P8qrgGbWoPg z;{%C`WAMkqw_1ukKMhBA1SFU>_uugI{3>qO!*=bZsl`F^&3YO9xL}V60}qKTkLqOT zWBwg{CY?Q&+<$o5Gb0oK0D`{n@a?6=!(cQeAbl#wqPTT9lY{1IfY>zSN#K+#gdGQ~{7J7Jc z(@15ay`U9HQpoiptML!TY%RwvE$&UuNR|{-5B{p*?yoHw3vIOTaadAQ)84uKk0RYf zzm4N-l#OmRUlpP*7FMNYEuk?V)`PxvY}zNpBHbl05R=U=a`VA+ddWko$Vxd8ORgVV z>)&v-mkOdg3iGZ^^y)F_qLdP{hC;z9oy1^KhUpAj!-t@lpnxGtSV&80=y@ip%L}?$ za^gc^@T||;s~FJe03bEKuPak?`VlB}BAlgY->T^5O8&AI}XnOfw6 zYkMUpNM~!NhnodQVnFIEq_MKMN)Q8nAlfq#L0YAhgzr+8K^{_ZPVC=huu6t|PZiaU zDYMPVH$~rRXlY5e9vC7dh>GVF!z?~Z)*N@Q{{SZx=Jx5= zr}n4IAAxZ`$2xIah6EuYBZJL!_BJdek`i1bOpz1!u6o5Rw04CRC=g*L2jaSGSCpg@ z+CiBDM1IxNl$3}WJl8R5o;*S_Mg?@5L!mAL z3}9easT!k{?`;CM0#teLMu7sA**__nR8LwsIqW^FrYz@{nX%y2e)Cxco=KX5vO&yK zS`@pB2_IAn8E%yvO;u!)M&Ve2I5Y-Gw&NU{p;3-c=BTwmBCER;wmL|;T7l%BN|5rz z@+fO9pCNVs`KbNf}$@{~v-vqDFDx99M{voc4uVz)NH<~wGkW+n{+ z`_V99dSas~(P&1?xpNZ){lx<9$Cw$R?k*r7G<(oXcpg<;Q_6M27;4^;1jwP(M&lTv z{9pzut9z{Gxt#3ijA+f*B7D>u)J`i^IU=z@C!`9Cd!wQXk#?ly_M@3dU5#o@)%Lms7tT|j8&r&Etm#~kYM|o-L?pV1!#~J z%1r+N(^pOjimHf~YR?-$$?Ubf%Wgh#ZPl zXoX4ZPZ&E;3cR$@OW6rFjGjIlW>4n1%}VV6FK`nlAXi0n@&Xl~+PQs9p+R{AJq9XS zC@|T@YR`uz50?;Yjnp1%d7YD!=B~S1S`#28V~p*^arX=~^3}OXIfz#^!wQy?M1dZNO3{69`rV>XtblOxlsh|jsQ%N20(z2J33SvpxO?3B*QFTH$ z9-@{#YRiiLDf2THhzcNLJ(^oCRVl*S5-1iuZXBK&~svAUL&L zB`P~iq$HDDTiF}pK2=9*M7T)Xw0g}Yx1=*%KrH-U6-+HrH-E+TDV? zM-aQ%dzjmfM7Z3R;HMEB@Y;DlM(i0kbY)TThf@u;!?x#nRgm zu3xYdmUrp}K~G$|lgV7qa%4d!i?_Sg+CWX_mu+&{fEVHrw2r`zlw%(>@KHf31Ar?* z^Gq!U*5NG$m{J}_WOD>`swT-g=-%Ga-uxM9+joZnLwN_Ti;vs=UxtWp8f)^yf8fdVLU@wb{}?r?X*h zuN~J?pm#QxAt}df58j1~x0U>BK>CzM!d>$pK#__F=(>%&$Wi#1Jcap7ZZ|0(y>9lI zaN<;y-s#R-6@L(+fGQZAaY2d9QboYw|UEpo8a(3ordvK`6g&BQJXtkQk^(9&CSsB2VPRj8z0=7SC4n1 zib?2t&%_#a)!0)Sh0UeANZH`B@KQu_Vt%QqcUNZN@}a4`QiSXcx<Oe;Mkw&$x8`KTB%dqhgEj?E2@5bk%NO+&@kFM}2p3vBe)9 zED*iTxQB>PG1Jo=)vk4$n}C6+Yu1+xh%k)x8jY&n`N!?n z6i5F6U>Hc~Ju8_X;vI^fbwb4paY`g8Aw-YPD#=Pq?M;jH_`kiRet~aav|0;p-mz$8 z`Ph4MW2A$e(JRzna@{@+?WU-P)ASdLS02O}ugdzT#mw9+t?ex@q(pCTZr`}7C;UVx zN?o(loU_SjiuT0Ut6H0sbA^?rG8$t{{R>6kAyEmcDAo8sAwf%eU59W(!a$GI_=P> zZ<=M%z*7neKl-@*(!5!*_ZpIMouhBsu$DCd zo0e2U1;7&~Ft2&8cj0e~ZLeKc+oU8oQkAzdm2{ImeJRE5=lOf+xVB--jX7%}_KJMW zl5#?Njtq(lnz34ObTWO2WIDno?%EWTsW@>U`Sac>g6tbo*}7#)DciXUoTL*fN%rkl zw{SAy4JARhQ6R)6W52(p8D@mrEn5$W(NKgQMEz6Vs-Tb`knX zoEYy*lNF%j%S__dlv%ps2nycO$oB^WziLxqYjJM-j#~iYi)w6;tii}9pfeTN-D&5< zb$4w7=wryCk1)YC&1uS2qW6N8EyN?n=>Qm*uQo~D9*iYvjZ<#4+LaZp*OXLJ60$%W zK~NsDCWzD@x+>D{EmTQDik=E|jmc1qf%X)Y%hs(fmd(nPt1lfYN|K^E?dvtrS+Tk6 z=55xMrr|J*!9q}pKbZ!e?NSx4F4%RjX}vbZ#~a&bi?&);!66N)TSu-l6&j0+(;YVV z>QdWX`t1IIc9f0CQT34%>r0!tecLYi7e-%oCktfVRAc*N&PgE*f**fuePj#?dPlIykVMHnbQBDczndiM@r{B4J<;|9; z(~MnO6mHtB>D&_=zx4`a({6l0r0O;|DGt8MMbjiM2mtXXkF_q=-M34&*C=sT>qf?I zT}a%tEg1xWB#a64H5QuOdMg&C_P)y6r*-2O7K}RE$$bXkw4gRE2gvaqQbr7Bo!(z* z+H#bn>4jE+t;?HK?@NrF6+G1Br1eO}b8T6&_4ABgt;-vL;&xnGx>8wq94#}*Jwd^z zh+eUzYU)+#x7FHGm9TdeL%@;32B>M+VlZ`Oq-Ecc@>i(pzRMLsyj)HiK7-^0)msZ1p zY9IZzVJ14`sr08;pAR8>7ijwI-jcO8RVZ|IkL6J5-fJa?^4eK>YDxlN`)4>NYtiuX*V=jhL_M18mWaEsd6g9> z#AZ;iv%Mx`n*3k>Ea(Vg`r(y6Pg3FKSv`B=zi8YkD^ijG6bMirzP0(E{9@B> z@3lL123&2o5>q(X2q*sl#T9&#;gT{o2=a`e{iE{2R9kgLrI4UZ5Ms6~V(1?ncu#*i zcs~BM+Fbl3i))bM?S(axBd&2n-3=p6U!K`MDL4^c!t$Ws+M8z!@nze3INgt18A7E$ z%$$lU-%-6*8bNR;vSyh-d@tPr3?khVgFQM^-3IpF%_OCyI=)mAF-z002EIyIh4+|zmxgRLfEFD{g$6E206#_st^z9Ge-AhooOYbnJ!i=Rc zg#D}Pp9=o~gRR)O9ldQSNQv4B>7hQU~s9?OINZ#l5jwl%28!g>>ebwRqCfes0>?2ocDG z{?*R>N%1Ro){Z94Nl{27gem~XQ^(?xIOB)!ZpqIq^UA1csB5}qgD1zhB?%+QhcZX4 ze1GE)i*2<)<-LuQ%4t(3RR=!wr&#!&`%zNS;WC!ffV3R_=`(u5Y*bK5xN&LfoDaVQ z`d1!rDp^+*(c6Q>jz77rfGAuoTP;+JMxaWz556ePh()rS)7VzH=JK z=86-zfKnz;T1|bcyyD@emaWHX5Rn-5t~jH)yESp-+9ayn2cA%ZrDOv^i@;ABu5yc-;o2*H`CJ@}hYE0KGD_aY`vC z%sEndq;`5`%ZvG&fD+PkG1j_EG2k#(0!djO^@Ns&Hg@P+!NtrYfz3B%N^!yZTLL8F zl3BWy5~8%GKh;*dzI9OBMI;l@)`~f2v#GSaX--4VBmg-z)7h=3&=S}MU2(;5S{P&$ z7f$C183X&8?yO!z2ns?%PR_O2ik^-fiLQbjw-(>TC2A)b$P}#!b?wVq!$=wOnmzX} zE>@7<)TGA}Dvar^ps3tDL}C&qnroA&lZ2@@!YoT|D*-A}V0mfny59rLoxNx)Ra-QX z;U_1$f4wkE`~0d(r1S!>hBfSj^gXqjy+} z!3o|wRJNnVVKu5T9DU7WAcSX{vP7E1E?i#^@88-6lI$C_Y|x`c=mr>j=#O z001B{&MI~ehLmJuwDmLgsu43= zk&2{N06yA~i4bb=Caec)z$k=-@mDynM^jKtq?*7eR(AW=MRlTvD>9lUF)0O7Y5m{xSgg3kYLZVpP#q%aHGT$a zgQ@LRE5!?ABw0&|%?`ETo<(`Z`p`~3Q@SQSYEzvev5l2|*dhc)F1U0@)U5F}tG6VI zUgC+8YtNj{+;9hZc{Cc7WacQBGeRtp*Y>VCGp;I(%Zg758LIfF`GnV%3ZB(Oq)CdC zY_dq1DjsUPaxpS0Qj-|ty-G#|OrGK^M(oTbC(Bd1D-^CUW9)0iaaB)5_@hq{ob;~} zV;G_LeT8^BMCOWMXxQ$Is2ZUGTpFNzKD8k4JV~cDGU-8{^h0Qo?5x&EOmvB@)12@} z6-bk8x=|cp^F=lQ867i2p=3Z&++s2)q(A_U2TJIP7HQ%6Wf}B5Rx2qTD1bmolS8|; zQ%ou_+)0#?6jLn~6qR?G^sE}UD(nI3O*pe{362gZBT0l>Hwo+bql%(^su@~9fTDfs zuP6y6Tya#|M>E?1xp+a5CXn5PD5#KNj72%PbO3WdYFTnOf&d(KsP@NNIBj)ENFXL= zI5omwK9HlZC=nnopz_RD=mS`(_%Xq2Q6Q%Q9Ook+vGM1GztqjKdaK`Jsm zE1J|b1O+Q-DI5?+4Nat04atb#Y!XRY_rR1{8NBb>Sonxh~$~1Dbs`R1s9S8LSq-s4icSxZT}wQF(9tjbh91v4#M5S3gkTm$7Pw*&_ZHCI6I};v)IVAM1Z&ubd4NYDtwRxxRR0tO84Updg zWE{j#rDd*sU~E_nT3Vr^*$&(|5NE{w2qg0zMSQE`KZ-Rg)}3+404=wIj_+8&u5p6p!wb~)yN?3JzoD-Z6ym>@ zxN9_A>K7p`sa|z?l`~y(v>Tm8Y$e67X$Csyf *D=l{{YQ&XP#V5MJ~=()N;n2GQ&*hSYY0did*Y9nkC3p zi;p8`DRAPx)&3v<0E(?@3=2I;(#xPhQkAo0ImBiOd95*cZ$N^WD_=KIIV0sK!*Sxc%|f}pTd&fc}l7|$HI$my3E$CM82r7tqV zk^&q_1SECtli!-n%QlPUCK8}Zl2cx8^{}K|tR@;?n8e3PkG&Ic$nl!ENBp&fr~d$K z{{X#Sl#$G?EJLU;2;7~|0HTpRir}?d9jS4*APbZPjgl0k_L0U#eMJrUN`3GWwIHb> zBag*#db>r7mONpH7a=nd89vqLXN4!!yD9QLwUN3>NjUwpO?Q>CY=#i%S-HtbK2QW7 z$M>!!w`}f}(nu!VXl1gOHh8YvEw@mRcBC0eDUN=r`%zZ5$@w&)N<*(S*)6`L?v#O< zo~8wQ@mB3zFog#Yz%D6o5Fs0sHl{~$HET;%vtx4Qu*Zx8YMxM(nGk-|(odpMYOF6q zy;F}V%cZ#3deMGlj7VIh`(T4xULw)^B;2W>yQcwD?u9^0fWRbmN{~fP*Vq0zmD1G!dV+s59)%S#dL_dbp?0!-!G_Gx}CR-ttWF}L^^}S}YS}4kQL}Z6e=h5`kfKt8VBm5))VXnVUB{Gc=(QHK;csB| zrlNGDY7Gz;rjLg31c*`z@~&|pj%v+Xqecm|zfR^0UK%b+lmJW|Dn}i2-lE>rF32(d zUhVFma8Fy1Cdr*e3GM&*Ak6b zUs`xbb-{9kr7l}KtZ!^|2lWxf5R~|-BTjbVwJj2WcL3o8k^u$@8Hx#QTxo5$Z+Mr> zi9SQ(n`J12pR1Y&1%~OQzy~a?Zrup@>>m}5oN>JL;h=^BI$7~!T)qaI5H+9Bvc3KRatW2)nc*N0b98FhPp>-l6d0$WSFk(Ib@C*e-1@G}qT{ zc!ROWRueeHhUs;I{AAvGt)ljAR^|?zC_4 zoi1ETTSJH#DM<8A^jkrvvRqpx6$H<%EVDVfv3{I!E{;!0 z_zv4oY=V~AWgk)$m_NOJH{p+k^xciP+vP2iGqajMPSef933AoTc{43M*jPVvkx$!T zX|{XT+beL%4Kv9jwA16jo8;Y``8Y!){vk?Uu{noE|x3v`zg%|-xpY{}$Y@+0r#@6?) zUcL}V@Ijd31s?Os3IM?bL<35EQp~k&u^=4hHRfAUaHxP(c-(oTqE&ligftd#;+scN zG5gWXvAsk#t;s3v$KHgnuzKLgAe9O9uCGtg+-z=fScs-d&Uwj1_LpL?P(UXU#dUW3 zt(x=fCMKzGVzycSQ4=%GVRdxM4$`7jIi5vz#ykJyowK8hyw5C!Nlj zAmS?}uWuaGE%VULVq8+Wrn(g)$|R0)+Oj4ET*qE(H5lYju$dKP)tfUXrBIOy+I{g- zf<*P22D5SLRuw>Er*&g38Yh!lH_D^&Lhkjg5WJEOCq-^`aW}ppf7OW{69IBpxdXF2j&SnzYXup_*|=a7KNogf_K6jlcC`nQREL1euDk zAPky-mQ(e-3cDc5sqs>hd?aW6e9@{ko=mKHM(=8$3Nq^Ur}wWpc=D69uV^30kSJ2= zB*`lECbo=OWdeLX$JJcbQ;{{AoaU^RXPi=i)N*R76Ez`XDmMxyAc(-KZ6~Z%iRwA1 z1zcj3#R_@M#bTuTBzso=ArcQYnxYI=3egA=HL7N7F;_|Sr5nA5K&b#uOjd;>5HnSX znmrDQ+5rQRUfUd1N{VtSz9tO|3PhD{25S|)BlLmXnyagk%?7s!1`aARofd>FyO#ua z9<&>;OyqsdLfoFT6RHMdjv~0Hkn4m9bzqa3_Mu76B-E^&!4;8FJEs#|tms-iN$nMj zm3|4Dg(@mf@>N%$Vvk7`NRE+P6WX#l6tp^Q_Tv!~QkMY{B>UC`s~{SvT+Dt46_N;E z4^PEHfUEMr7i$Q37DiNpSxFG}dZR2|i*ysj7fV7YYz2Xtuea(x5m} ziK1J3X;FeZMN%kM-3FtK=C(|9%?6j!V+7B7C9;)t#Z?huC=<_?wg~`{l?a|_aG(Uv zYehI9;Uei`-p44iLZ*C?x4k@j%gT)7I&Hy0K zwJ<;l6|RZ3VND!WT`~-sz&c66%|g^kCvf72DE{)=DnT8pH$|d_8sdt_l`ZtHNVfj~h*h-_Ja?|1`3fmitwZfe*2?)pqI=Zm zZP`?`EOoV#Wgs8)iU#2#A;KfnQ%jc|Z6|po5g4v!{_kqPStMZgt~t78BCkMq&$5l% z2v#_X=U-{Ng)4C>+;BO~Fuc?n0ZCJ4N$X16Saoudk6MFw+1ZUatphJiVYL`CM{(AG zU9GuM@s8N)1$0g{6oq+$RUf96ErfF6keS%2OuP;AxXw*5q-qxQ02Xor*hP#rwUD3 z=b5?+aVzkUzCyaum)-^Zkd&ySDk)CVX+86+N;dqe!U2&9lO5@=&@0(QSzGQ<`I5>a zWMw1ldHYtjwrK%QKX|8_3d4aRA~_V58jD-DY}}$@J|K#1k%cYGzr0aU1;VaMFpEp@HkrfqE$8fhW|lx9b@ zC38@Gg`{uKDjb?+7Wx$>>~*SqM9JQ}q`7$ckV2FQ_C0A`t@%NBQq>!lm$vzyeINM!IdV5J|DJX)RksyB5UY}v%G^F^W0Dz!*t2)vO zA(bq+RFHRin6Ec1o!bsSfnsd#Zr}NPHkSLY0009Sucmw_{s>q zbgq~DC-@4+&*Pg5eOFesw^h!ZtCSV2Ehs610=pdXUDw86AL(BRS-2GA8iLT3AtjZM zIT8UKg?e5eix~d^@@4cxK3PAqC*hCr*H?D){6oBJ=oUyrAtR&-$K+Gbfo{`rZq51@ zLfF|L4A;)SJ=LGrJ}%X7?vRF+aQSm$41f1gVw+u{KH401Vc$A?Bc#@xdBT~`7s7fzRjOH*@0_@7r7Kb)NaZC!M2^*K67x)2 zzN>@aI2=-nBrF(%9*1oasPU;GWg(HpTbFn`=(%QhYRo7!kOW_axVj z-8PZz#<<;tf0iNZ8)T`+65`5KNI3#0KD-KRYh>Ejc9(pC45h_+N!Uq{KIW-w;lA6i zx*bv(I|2ae6(4>novp{rDa&T|Hg0yQaGx;&SQAvLmdG~Rpih=n{->wgvg?c~FQ(~1 zK!l`5-NUYOaZEe}J#6Nl-NM6n_vN zl0!~yFsMC8IpVjo_z2Z%P*`1N9K2!B;q{MlQPBB;>wMXH+c}w|v`; z8+KgrH*MNhoR28)n$X|1roJ50lC?bAgKG*__MQX-jt`|De`1#|Sah3E;#Q=#$UZjS zAsa{3xEVZBtA`my>MaGO$B*L77zk(s1VGG_oYxfBA4hIC$x+g#*c+yvwY9v`7V2e1 z4lN8ixg&AqJoUs=7cDg_OGjDOZIS|%ke40so@Hx2+(tlvbAl_I(BHhY)SO7TwMz$Lk}F0-ZG-CW-8O5JUwGNzeC@TDm3OcU3>E2A29yZ)c+ zIdW;}y}rfTG>dD04I%@s+3E>>AzTv+Dv%E;Kbi*W-tmTBOV;c*)3TPr32hH2qPpx< z^_Z_+X^OgiKjpTXa?+K55!O~&AkUcYY-jt?N?(ANk#yDF#kx=md^?W_stDjLK^*f& zTU5}INo@mK_?GsJA^WSv_(Bv(JEZcJZTgegj%se3an^PzASc6?-GGxb!5sc+dxGY` zC~0h6t+Zq^w4nHCJr)V`5BhQ_moM78VfLN>bZKpihFM*TgW}yvL#>$# zNBV*L*FCD*GTCuWlAnl6>5uJ6zFqgr zX3f@Zlt|vV2XX5Xc%t2G+$VpEQR9OuBNj z_U~J(tBU~upK4C)%U!dH`Ghds8CnFB{izCdW!%WPQ;Si;H-aoQg+Wm7*sTdXHK%ZT3vI+Uc`xUN{IuS3g-Jxw7H8P;2>axCU+ltL2$QG z7KwH9v`7I;8@e3Ug$&z-goQd(Aw(PlQRO42f?H*;?=BrK6u72aCQF5LBi4pisPa%% z;~S(QK32hmq>oe1D(2mH>q~6!*q8=*s=sC#hQn!fso0?~6g~N(llCid^o2JXoA)bH z(ZpaRnyaVfd^Nl`Tr3C)GDp2LQw%l|9a38f!V-kWKl9RobvC<-zl!6Yq^5n3N-?Fd zH}EDFo9@=ViY}@F2LNW7r%76o;w*S*=sFrM)uk;!_$X-v#BP-zy*6&x%1etOAm9o& z$VoaBTG%9IU`WZtn(4It1Lk%er6FI=oJD%l*^un1Ax25}_pY~3+@O3ylq3uU)u64M zIcCx|wbHGuSV{+-dtv~@`_Twe7)-npk~$GtKYZQ3?YluwB%Za(>i+-{CC$m+w-irK z1$8bfql%oE%U<}HFxrDhdbZ#risBQiUERKLtdTt6SCn15bxT+~XMnAwgR|+ifEq2Nm6eB&6va zxgeq0+gdoHBx65{FdTC<6AA$&Njb?hN@t<3QFL*B#g)OV_j8G>9Dx-9Du~Fe10_iq z#b!`UN#=_UVyD6bgPOntC?r8O;E19Ym`ToSBP!u3p7hjWxCaJn+lc8?whSMuw_37U zb;AvzWS+^a6lb`em0uDdaUCeH^5=*D0GKFcJ!M%nKQCz3TyajH$dxZ8YCh&@R+d4n z((h{4*tPHw6%vrM9jatV1BxHS?@_*Fbfp2ZmmtvWW$k2YST3 z5+Kjof>Yza=7u*!WDq-vq~tXW{7`c>6V*8#X;8dQeBN%oYDM4&2k}oCCV=8ID}g3| zrBu2AA`X2h51iZxB>n0Q*J2N(#ZgHDgM2~`GgMR9JLFJ_xajbMn{l)mlU7&g4w2@rGuCOHv)J=Y9<-;z*r}@};*Zop9VeQk zU3N}UzK5PE8Nu|bCGz2c0tEJ@Uo@@^EszpL(lLLQIINy0-fB#ZFBzNK5M~iLH+1BL-+@ z{52U$fj)+bPTjFIu}lr2b<>Mz8Ql>WG^x_30;DVd0Bl&J+JR$gpvf{$YmL-dNd+W; zaCxWqmdg{gsFDsyu3J%Bk>&8p&INI0pUN$XZ$ewERT4_bCU_Z&96lLLG*Yi>-sn2! z;GmJyoKnXjuu6m;ZKyaYObhkj^mAlQI|fy=Hn#+>-D!a(NhS?u{lg9grKwth$A}_~y1eni)P$j9 zZyX=qlU-k=0p&qdWKLGo?oK8uq7V;X6~YQIG+OT;<4G?Vh}KDcHk~BVZD`==yFeWZa`Jf?8d(%Oz@3ltw}1P}`VkYa7%;jAZ7D zdr5uf{6r^k;Gj4#eW>jlNm5c2^_fa{`9W;N`_YS3Cq^o@ODw5iC_KJo%nC=SY3w|m z*G(nXzy=kMy?6fraT5$@GE;+7m;yWeyDnd|pL@0n{SAUO{966zOd~aI3Vf{p| z*@U!&7!XO|*VaA>y0`geOR=#@l$PBcM1iigp-4|W_@Yz!CH!vqQivkLB)F6 z*E~v5^muVjGjVA9h10RtjMt ze)ZYvwk>LQi(=&m{%AoQf43TT%aL$@hd5T^h_&H*3qUZi}O%g1qVX9K|FnKZP0 z4$AH3mVQxX8;*Txvjl<))Rj*GM!DNnY{QC!IUv!^-GZP<+zO(5SBm7>-r}oa+J)j8 z1PKN|NX2~9;%k*VbP6%YBbx8^CzIe66@(@+HS<4+9a@y9yrl!yx^OFzDJ2we+M3F) zRk%SM(e~CYJey>$6iNXiWB&j-r7o3ST#!DcDg|t%x{K24<$F#~IO4vcDvC#t_ck@t z-dk7BST>-R)}j(dNQ}t!udsiIzY{o?(A~<^tB}2`Q9oC&{&%mB>5L}g^48^`y~r}) z0=p|alc(OL*2oE+-Hcb6IIo9%FRF{JKGN@bj~8Q&1s^Jj$K+Smn{m7Av_7>+V%)6_cURN>4RgaM+S2TN zxaw9W*DM-&=hW+kG633m^!@Wticqdv_WIDrdx7ll(NO`IaM- z&*q)7>vm7Lix-sT!a}5No#ZHN6BqzQp0uM$Px+0_z(@#ax!|&-tLh*@Bhro2KzY9# z=ED~0N{|~PdX?TX02r+aUX3`7E&f+V)5pv$S6Yc$mEj-{JEW3i&*b(7w_6(7ZOeyV zv~u7fL3eCAuL+qMZ7JnmQUr`((6;w?02`&-+q~Am8)y)K5}t&fLGRB>)h^$7%r$Ld z%S%`!zRSq9T4ZqAHyy+b5^BmysXY*JmD@!Zr#oR{n=Kl|>I9Ib-Z<~cTQ(yDX#s91 z@CtA{(sIk!A}rl${{Rh2+=gyj@Yzu086jJ}#WB6L)%3R4s$J=}*Ne4nM6EXNxp>?f(GE?G}V4(JiS0XAQBaS$_IWtEk!jqT(1j+gAi=|b7@ zcEg*AMk65d22TqVL)X>O3xl&Q!u^rw&`tp=+{owv2P zQrt@psIto3M4`t(m2E3CwLBBtR9Tc?-(6hTKG`gVsV?qXlv@P(i0G}wUc)syh1IH5 z+rBI1TNXnpZ+S$Nm2I2}j(N>?%208$iAAP!8m+I&INjIzqfD~|sZTVNCFPG$WF!GF zIi$nlOD`1$_gB(`C?K1LZ4~op2ZEqUJ9?Vx?==UI{nM6hi&mTd7R!d!Dsf7KDkcyT zKWgUhZtn%!E$q<7prnFcNj?*55$0Y{NJ#HqZb(zKcjA+3q-e&sX!B1CH|^X*qDZ#$ zG_JCf5;N1*u)Mrt?I>~7wynu1abRsEpGxvBoxM>jcGO%plGqG13QTrV0N?LI-6%d> zZD}fT4Sx`9+GD)+ZEhqu%G8Gk2}*I#eAgV~&D^-Vby&KK zc2xk}6rJ(XQ}J7Snc zg}dd{Bg9(?CQ_&v9<|$PXdokX%3a&#+5rUm*BxT(_A5UzXzQ$Q+Jcn@&*0PBdy97= z_SjHTzeo^7b0U>JjV+_HyR>Px3yMpjY0feYDlWRIUYn=9r4i;)NJx*p38!4VrL9Oe z2+4rrWh493rQbd+;^jvp!zw~bQ~9g&jU1dLj#B7ew*LT#;Adtb3LJgt3&!1J{vj!3 zDoiZmC;HcNG_*Z#9B{U^asZqUy(_rT(Rw^4l(!0xlm_Gj{V6{G0D`J;x@QXZr}FA6 zd-#SjwO}7WC=Z#rykCW-x4X^feZho}_pZkK;rn(OOD5mupHlqFaESY22i}(ayX?8K z0Sz!Pu#q|M??pe5-Sib{rrwJ-(QcJ^U`mt@3G|_tz-grU=2TbB8s+Pb5Z0S)>(0a% z7OWp?ShVVFEn{<5dy;p!RT%W9O7~)4ss+8Zy9$z`;Yo=xj8fN%SumxAyr_^8X`QRq z%_yNM`PY>)wI3*+^ViasYDsx+?se27ag;R6W+T+en#w2^{tVf;TYHt2EDtJ2Ap22P zPCV-OzT(oH6oFW=YLz&ExgB5%=`6JMzDCp#kdfw4ilSbT8Opmf)735ztbM<|H(s;g zpth2i7r?~D1#lR(LYppu-)|}?U+nDhoGGv;fEfKb0 zDAA4C?#U85Q7E-SM5uuuYUog;b91&UMUvKK6ZXX&ZDQamIUx5qG-?ifiHNI{g>Fzx z^EAsHfz5!uaGP>MlqN8dnpt;tU`PO-H}v~|y)d{#Dk>povFTX<0KnU&Ma6AbP?Y(P z+|Y4 zk6~Viil0<@`McTHu?7MHe=$e2jP#({VJS$Soq38mgt)vi($D^#zt+7&jkHN_2e(@A zj*@7Pl2`}k9BrZ2PJb0CMVo3LUDtRUN95wDl?iAZEKGkvHKHxwb~ZFsJ*#Bhlz;hV z7-zA{jZw?1hcM%c+t?TsG##EcDnLIlw6pF(HJ-~R##Zgupb$SMDhdQh8Shv)iTbne zn%0+~SIMfcSx8cC?OTual1Ji)a`NF|j}vJAR3s!-49SvnQMZ~Lm^h$yWnN=5+>`xl zK$zmL6Z=$vI?|6x6bO^WYJ^XqsR@8ct%7`~(uy?|N(^TeqC`zsWmpmunkFs`0~KVE z(rX}O5EUP4^q9q9P)CW{ycIb$fHRnZ(zKe!(l!eR{{WhTy2xM%=BJ)b5eX{YGsQr` z$*%|mR1zT286GS0*N|haWOFmtybK)BprC-Mo}#KgsGe&t5cZ1jUs1@Q9*x0>B;Zzo zLJbO+07>pDd_eWfy~w=a1MWJpmIiWLM)bU4LMCUb`x&!q`r7FmR;xTdE$sS^82h|f<8=wKe zlgv=J7e80kO=wh^M{bG;^AXxQRzK#dP6~;h^ab7F0sz~Ir7m0r96&U`ur4a9<)#cgB8~pq;k0mt{4Jv74J({+1yHD zCb|dPO1B*SjV@idJrBImYhg?QrizYC`cpegQbK1xZYo%}jFC<48BX4_=}i=ba%HXU z&eZ^Dlr$qe4l7pMF%i$LY=jM@#ENPp*@n=RkWy%Mlo^^PYLQW{gp+sJ6ZWShnH0VIKmxq|I#JgSEnH@3 zCAOe~Tp(jPniU%gZ{+6n1dDU^EaH=?^lev=AAV`JtOBQOlK@3?H*Sk^a(JF9Zd_SP zv}1dCSGg%b36sTh8iKaTKmk}LnBQN#w2;W|PAiMNSH>bLQt<4V(B$1j6MK0FHcAf}vBHT2*0g(oumJ#47 zMFD0G1#^0Rt-1<;ZDCzVrViZr2WkS8sKBRyRIo-gy`xJjb8%UZC|M#8r%G?8 z=uM?hHrXmjCumcc{B)-*T85H=s9C9KDH+|hO8bt~U-@-{A#Gf~!c`JnM~Da9O&jF< zBgv)L@)iF8^!geo@vM~8fH#fCe-vkg+SJn)_wKf#ouv{8?g`?$lxd~ZAX{Iu;*ub4 z#Qu1zHD>RG5p?Cg@HXSkRN~PdYugpGrY*~CxxGltrvUw_9-_mE3qq2JQO+j1^|b2@ z6qkSl^xz1k;@85Jwo8RjaGVd#GHN1ipBB{6Z7))XEW`Hhk~aVV2efshdiIQTvfcKS z!c`=AffAf^#eG{`($#UjO{kKRR6yxmmD6t}4mWa@p(R6{3<~soek>Er-ZjF1cgPAv5#zLcoAl|pz5&0BeS z;!sXZ0*ej(L>hLbx=vpgF4{{X~#j^>IH0WlFYez~jy*+e8IClk$aR~HJoaFh5I@AzIn zQfTu0jFqIyv*lP(;W_)(h5rC5;Qf5plG+yD6aY^%SOgMdGhV)%I8$*M>DrT)$U<69 z84t^1(#hyfQb`dqI8W++&yZ75SOkRQ>g+0gqS%00M2Xi{g$Fz z@Dj@^@RT7-<7n?+qkqHy0EsLvEUo;>H&)FAt;HnnRE+1>6JADTEK>c?N;Q%eeGcyB zw;XXyfRd)tncFeUpL!PItBaeK9uE?2)3pg6QI7PFRJ(PT1J;T%TtFf*wsHkAw!d4; zSOH-wX?vpu25~jO<;qPLj2t6jcDFw(WP~E!2|^t!*wD<(#QWy4w_q_~%XdT_`^m02OOTUue&0-=?R?LEe8T}}Ik!%K0{+ery*H6j4&LGRLN%cDoWg59`z z<{7zq+0XLTAZ=B%B&kh;MoGXbBk@f1{YJr?NV3ubQpb>+i>0M%B>JR$1@)zd&N#_iYPp-J3A z$VoDC1|mevni}_6eVequuV~DN-fS%+O>iYHBq|E_ZZbzR0G^_;)O>asZNYZpt?rbP zhg-I3!<=qUhN2XpIT$^})tr=i-}~@JF>mlz_RZTTLe*__>%oGHk#}y`QuiorNLokI zL>!H0CdCn@CW2LelCajs|5UK{M8o-0D|NDcu_S@%1vhV%WTD3Y3RD z&B!uGm;{0(Pcc@x_<^K2qWf1C{yNZ8aKj{}x6k~=sQ^IZCj-);lp_8g{1>UpJy`2a zaf>@x(HvFkF7Djg@?ezYklTqLN+t@RxYL1no)7Iz+JK;61O-l ztbrf|$uegKr7yYZ6@%m1%wB2fwrv&~ZTj3(Afa}wxB*0&9&?$9^sH_Wrc#$aYE;`` z58?+SFaXRzGrwf?RcCtThLY9YzL3@0tkU|>i;G9gEwqxO$XF%BlhSEL!k=M~wr?4@ zNgGPMBOyj53P}MZEnoFJ;+t3h0O4Iu>4#FMTtY&c1{9R@-|<|-T7KKc+jYXW zTvpU6L_`h_XdQRrFOpiscup0NrqtQ;=OBt!I=rI#!5FwLe_$s zxmBa+O}GJv0+i`W_iYk3$Q(phL#86#O7UAt%V{E9A3;2foc=2!c?cW`KJ`d-q2ntD=)1rZjzO( z$KsY)rqR}SE!8`ZGL?aY)34s1*xOsJlH(hM7#mLG_@f5yh)$PAk02>zuvfIEB&d*2 zXz4&|3wg#xm8-8ID;{)(?4P|gd2h1Qej~kP1i<@H*7pyzSNs${>5>*t1GPGpGBvM} z1NM55QY{&1vXCSPrd9s{rn7F#PaVC8XIEUfd_lpg{2eKaOZ{bZF#9CaXs+ z0b9yIE8-IoLKfP;J59@g%n*|U^GvSVN|h|^81ndvC-$V5Nqvb<B1gYZS>`V$|tMD5QFjTMoFe{v~uu_(lK7*4RPX@Y6T{*nhi6HE1X33O=%T^qzmIbQUD5=jMjf5yN~&Vt8aB;x;-O9)b-`1Ixd@Y zc_sluAwP0!vRC+Ktr4_4@xH&L{4#=f7Myo`${j#QC+wP7;mMcuh31Ac=JwiJP)C&{ z5;*H!4vk>#%Y_z9F6CMBwAz&)aa~=t_rTg(!t|{tR`{La68+rUu%Zt>e>8g}Qvv=U zYYPFrHN7)JU~r2jgQ^05{v&|r*i{#gjz6wz{x|;sLOk=A=|@BG$H982Y(IF%$ubC) zzCuUf*V44Mw5wzUs_@+{Df0%CymQ<@uaapk+-i>{-A3x-`Q6jK_e#&&c>7n=GNBb-$foG1#ODuvAF6h%pr6BXl%joqusGwWDmz>|^1Xp@YYH32Z1^p#TNt8s-8_*P-EIEKqG=9+J%Iy9FEl-12uRc zWP6&dj;4W%5jdC?C=enht`Z24Yuy2?5~~EK2B;I9)`0p|(2!`L9c9a@&zh86zXJwy zTim-F=c&adxNzs%xpL&BbmM`O_aQ}IXfs3jsWNwFuz6=DJwDVib`=UHdy4YqHh0Dv z4c-)apK8Tq5tCR4dC4c%z2Xy@npGkAv_izE)R~}PP83fxQk(=HqoD0VI<$~+NiB~7 zHOB|9r7d)Wf!3V9l&WM>*P7YerrLt3P!XOhs?+S#<$!q=-;N2KGm|Z0 zXi)@-8*xurKuShpyv?{u5yB365?|?S)f{S3aU!M1{9yMrK)x_R7>Sp-c2%g zEg<#B^Hxbrm@4|KFirujTS+8sHAD`LT1*+;Ki-(LabOmMX(xoem zKs4XMJ8(LIN^S1Y+{$G1G;ht95<*N0b;g8uW>26XnK1@|b!Z4sCo>olYfFa~L5$Fe z@fNcq1G=WB+6@RbR+X5<5$RmT;z^kbFb@OPyNy!aN^dFa&{rpX3rSuCLCN=|luD*@ zH_oMkK{1X1%_g~9mkpH&2^~qMTAR&{#Y1wZgwhh}d=Vw2brt7iZs_a96+mvX+Ehp- z#r95USA2?A?c=bDX4381+*6f*G|*Q!0byGb+Wg%NijNgrIum^goexP^As}>|)(B0? z_xaSEL?rV}($l0Xa7i$0Hs4LDDS0wa(*mZFFUiq+W?c=5Qjq`?-imqDCf*wbYX(Y* ziL3m>VF79)PeLXsUGRO`P=JXX8me4TE9Av`uYjbIr4=Zok_B=Xp`$5n#}U+o_B8ua zC8W8JDNpIx3y9X-gb-PNCAJ6N<{cnIXvM(x)3 z{IPCNX@yY?S~<7`TR4IhobQ^-RQ5Ya2TNVc>qG6l{izT(l#GGWyBkLW+jQKp;+`T$ zsG6pjo0KnQ&6<{i2GuPw28|B{IIz2Snoy7upd<55clbHo8#I?23q_(&flDq)7efmal`Y7cIilyvQq=KGnG{m?`#qgq&>MW$2G$Amk0ZvFG z&>CdZ(2g&(j=35=%G8yq+hCHMhZHe9R;sbO3kX?Z*2f+ciLQKKE#lwKskUC^2AuPK z#dVfTym6!!E-e)hOdyqGzZFQ)8CvX7bER7#6%?r|JjmQXYUI8n_;lT*s@rbZ|_H?zGR1?O$yvQ=lm(2}-e!^pfGWUrPS~ z%q^R7vw#)?i2K(gbMVgB6uS$QG~D@8rJ1XJEIHS7b2Xb6tOKIH<0-w$vQ)YJY*m?+!TS&2cOL)vG{!|hCx0XW>S&XxnZ3_U$d%sx!Uw_ zx?jWhO@utPsqNAVRzTW8^{&6+Tjw;O8V-eAIwBC02PgfhzM-S2p%%oY3sILgr-b*U zc4_%tKpPeR0Lf8QI5CV=vgG=f6ObE?^8HaRb(hs?nc&HEGt3`UTi_(YGAl7WLAuTSD>x zQ6stf(q@`pS?Tw-?pUVv?LDUfRoqB!*}$1f)wVf5sXSt9n{CZb*%p>|`qk=N+)@#7 zbP~eA5>gx=N!!9yBN#JI600ZvPf)!1q`upG3LLgd+ImLa!C_OtfTTMgF0kWC zaf_zJZc!K^Q6hQurH+eI)0WN)d)s>zg;|A!_>BUeI79I%3gU!v#^Em8wzh7be$_;{ z7W}9+g#*1LeFm!Jl3KEHi<8*ttxeV~-ndP>tgh(|s^O$GcO)S&QLxMt<(Ta>ZJ2%g z<{iAZTem2@X(~fsN|{lQ7EF0T=h}mP!*Z7|Q>usWO2D~4tR+HHBp)$5Vo2!~c5ZJR zv$tyE-76@#QWE%Zg(rJ^lCKYtm{cG214@_61K&dhTYSyU(rm5~YW2qRY~qqOr^DP_ z0AfzwC-<#dy;irTxpih=aZELC;NFnfb;58LJct{Vt_098+}*g+684#IZF1XaN*!Tk zZ6zi_SPC&BO!unYU0oqIcG^QJQ)yDz@Z1{|@(`b>l1N7GO*C4j#ir?5%WKcR&CAyJ zOMQ0tFDcf`ZX6@l&2pE^bm}*4Tp=v2t+I{M1Y$w!CZ!okE-C2hjW=(x zos%xD6LO~=LJR(wOZj}dTiN#ojw&=A$Zkny*+ zX$5LoPU5AUKm(fU*zli0w_)cza*!14LXw!|sPXMgh{@9s+S^_NrteL&>M0w;3Mwr} z>4^UT$)}bts?}jB3JcqY)Chz4r?$3iB}CZWTQGtEbxK^X9iR-)^`^D}6X*{VJAW(UkC{r+%8t;|2R`vlZnSsM zppdqk3rO)CCvg+d9@TptHLH100^wvCaX=6~jL}MVOOz`@l~~*CblX`AW?+=10Fxf( zle)`^`V^J^tfmch+Jcr@N-c_ZU>(PrQE_0qkcB#?Syb&x3xrcU=?ezEocgYApDIe* zQaVc2&#|Oe4%#%OZ?j~%x=N6vCOt`~_qr9$yWTa^u$2)B-gu16;${`FSbWUmTZ zfX;ZVl0r&NpbJYz(wP&`ffUx+qtzq>z)*@QHt13k8H7mwS)aWe)tT6tRCgx3aYx|t zvPrkID|YdYPHMLcGYOBurmb0$5*R^OZ!FK=x3~Ca_W9kdAt0F;l|_0GgKCk+cUe$(lSkbG5-LEeiHb3w7KzStmE#hKPzsnsx!zjD*lv` z=ln;~v|zM+5%7+eX8!>H0PgA*<@L{Sf=*+(r%GmOQTCr|~(zN?xKjdkX zYCh`{wGX(iU8Z)pqQ$klN?DbxAg#q8RjNt+*O_m_1A-5Jl);`pghvO~wC9p)o6(~f z$7vb0W8xjoDN82@clV~2wxBBj9CoTUuriPY_ofyhOHfagNa<5hiB3%F{v2qdWO;Hk zUs2JDgJ=eRo~CPu_;*hvB}5f*(!PhMv$ZK1`g4l(u*Q+gCu1;xbGo!keX~?;FrP}< zAVRZUbaS?O2m^|pwJ@%#2CR{a0@+>#W}ttTh{s-P3u)*pBxLD=6qN%aD>bJgXtEAK z#b&1CJkX)ktQaDrWJ#@vjMY{FCM!@;miPt@NdEw8_TFbBwO9<%7_tvO^>SmOtjPcx zx)P-yDoppF10qjos_mkaYOTA8N=JSx+gm`yPii#5e*nQMBg!jOwXkJC1I1XncHt=z z)aUlBx6jm|O>H2fMU;sGq6bqI12b9bTB*ileyrz#dSjl@J`#W=CG z3W*6h;%dy;pq>naBg@S^v9g1N#0i3$v>-}z3)8I+LDY_Bx{D?X0GRL10b_3jN@E1# zp4%V59u85ybw#`e-Vrpzip z6sFk+)YBHql9}5yc{83!A#MbY^fK;dK-(a66g|VlPSThh$)%3nl*tk%k0v6BbE{IP zu@l8Fd_XbNoW)?~qXV7LBPM`GQMNE}68-U^Q8Vz;qskQS+M zu-O}oPk62qRC!Rbxm+Hz*1n6W+of!r1Dg4+u+q@A;s$DYEg_*t8>uc7QiP=Ddr}pp z#f2$LgwIaCl=|~(9A5tM-iB-yk0U^IlA7~mn|o(;UlpU8b;lB-i+12EnG_tUwI3L` z7JrmL{i~|DLv5i!MH_&bCb?}w>K35LQ2^kp0;JPk%fd5}2Ah&-rLzw$x)+zV2=rDCZBiaUJ-S-!Q6*KBN2%z<4tj=@`raJJ}4>V=$F zJz=7%&fzO^=`E-E3g&CSvU^Tpl=$wYDv|VqKhmFyZ$~ty)`*`MO zr8nik+!Q~p7@rN~q~4cCAKwOTC zQt9o)pB_iiE$*%YCEJv5+aJVmerUAn47LI_JDYU@^z2YSu{HA}ai;i*P4k7-leDzj z3cnq`^je;^V2_n)#!wDH*dy-|R9Ptzcw1+9;;gU2%Q}iYBB+6HxDUC4XLh!@m*y`A*N5Hv41+D5#|R06(odX{+d7 zGK-{q<5u`3E|n%WiIC*^O>>U_0EZ7;ms+w*j=F^o02uBo?o2$Zloa74=2OCltyko< z=G;?{-G!-hy&Hl3t0g=88BeII?nmdhf5W@x)>=#Cy|{o9pkv;)NBDoN+O$x&(tvQ1 zK(D;1n;TZk_V(q#qqM_$SG6Bur>@?~SGN;r+lErx?UMuXR8!CLO}t+R=k~wC%j@(g z-4~XTgzX^BGO_;v1sSnM>xTlC;ei>7{S3{WwcV&wflVYrw)lHw_o;f_4Y1`2$ z{V^O#s{a76|Ii2_m#aqC=qtXs4| z1%6YxDsbeJ?^4Z5RGHI-w;i08q@wSNn?=f;G2)I%?={V-pZJHj9pDowMpc2D>vavb zp!}i@oa6rN{$>pK9Wmyjl?7g(26KyrE>L?Oh{|++0}Ai|Tgql$Qb=NUh52c%nYn8eiaSGa%sz?c$kPqUTI`yl1ORX;D)ge~|sl-4~Dw0oZ zU`=!SZRxqQa^SZrQg

@|6gbdk&uTP4&pRy#;AWwo_?SXbP0>>XHQw?41n}sIzwc ztL?bP%XBHy(u!3H1C>bHFfwM2)9zWgWI3fS-7hHr02;#AO-evU>D*Nwvops^<{rFC z5|(bf7g%v&qRVO`5&=<>8A!%DnsKM!CiM&z7t^JLhs!Bhw@FD5o(bGY=tUxJb~B1& znB{C9)Euv>c8rq0)m2dFL&)D@V=CM5Ss{?GluT3R6xi z>M4;th?;oCw@Hg8qPa8~c-^9fo7Q@b)BD^Gs9abH12VRq2Wrqp1_2#KXvNRZHdNqM ztJXzmV&!ODAua$xAo)lzJHeV?bEiLS)xsNf)i9O#hSe(%p<`(WX-t9)ZjsFusU@dx zo_kJH_YJKq?6pFI(v&HXHzfuH9%Iz?6=hGgOy#f5j`UWNjVk9;x?z+$6xE}P3rnml z0#jC^XNRUg|A5c7bzdr;x}XCgIQ=N)lk7FP10Q z?Lgn@?A*RwO~t*%xk(M&wg~XjN0bx6^L{I#)n97n)pr)jwk)g?<0wkNWx}J8Jc2R- z9Gc=>Rl|X&8@GSt!P?<%!40I$6@Vc0&LX&8E-jvvXKKeQS%G%mgU11;V@E#ZXkLn@&5pNeWPh^+Bg$p z(PG>U!d{gXb|!OHQfbst#W<(AJ)K%l%&W%mqp4eh5;MRRwavAv@hGz zfwH`Di=we^tCz@=;vnrG^%#$}Bh~&dWjkDc2=EmC9@xD_L(Tz%{4 zS`M(Q-w}3bQp!f%#uA^j)bYj{9az&-#GqUTmm@L=s4#SD@QENnN0AU`kX6 zHQIco!cts)B}ppk;$lrNaZ9*xw93%2^pJ@reGgB)bUX=qB(lGgh1yple+nP}i>R(pqKXY{W-~!6=_|%`{e(s9oW!Hyk_yp&a0IRe#|dv;w>9yDc>w zpA}+2`;aTMjOW#nPAP2QE}1A%@X8as3f#SDo_%0(ARk8jquY;D!%xQcMIUA z)G1JZ^rVl$tcmy{&QHne7oKjYIrl11{*@VElkKJd04St##VsR_MY>s!6-615C;h7F zd9>qt6laO9tN#G1ejBz@zDwa}4j>i2(lhuqWc+;E0=`S(y99`YhaR8*07W8?(@Abf z+~oZn%3cad0aAJankkauh-Kntf8{GbwL2f;W~2-!OVb$xpe`66b60QtL)BcEYQY1r zw<;&@RY}7a{{VmJ!lgg{pcd?0DB*|BfBqW3+PcjTPkqG=>^P8rnX)JwUyQX^;1gsF z@-56C-n$(GS=1dr#C4(HYe0W`d~n9zpY$}PKWOZ<3pdc(WhDYfO6+aeQc<1R^rZSl z`|CkG`w?c{fhvuNEKw(u)5wkFr z570isqy+&RRrsp-iuD!k>i%Owk&8m6HmVOet86LcW~d!dIZ?$(d0qbiYT7Y7RnS!y z59*IWHGdUQc>GkC6&POG>?>&91NbUEQy%pm0X;=(RJr=pJ&jQAGb&N{HK_*MC~ecq zoX4SxjjbdQ1cBb6L}ma!l|tryPfFH@O3V1^P7;9?;PTHA9@S8p%td+#rR2j%f~~}b zDN3K_HDurz098m6$oy1;6aq*SGG=OpB5K>#xx4^l&;~y2RWIe-**I(DP6&tiiDZf1Ol~wd91uu z3{7aXoW){VlO}7Goe0q`6=7I`H7YQZS%etL#QA-y;1E2cG!qrqqc;_U=}YcbqLT;p zK=!5!K?znmP!FJ#z^yKg9f>X!vAZ<6$4Dd|X`RYUC`B!CAgEU!?COnCn`_);(QP&s zJfd(FJ$S1pPgo~(nlV@C&6H^;aV14w@f7aN zq-~h|(q&tQ`V;PJb}kf?0Vx0vdaBVe>ei4rN%yN;D9Ir}{o<7=-Rbim#cO8a4!}E1 zNg!l;QRrIG+VR4cpyfRbNB;mdz-w+LN>Bnh&39J~2M}_jjLmV{#j|atMIe&-l|kvjAH_~8tAH!Rn@>*32drJ>BRwZ$WVnH z!T=}IfnCGOdve?fOhhJ)`NA6|NFEY+B{?RRnBTCz>966B1);QHsc;AHN^WnE%EMNu z1wiFPB8=2^NAHqb@WNXi3Sb|44%*eltpS@!DnD5uC;tBcilkdcX;#ZxA!<=ui%$7I zVwsA2YiQ|5mBhX3Q6PWm;MSHl7R-P(2S19gT1Hj)u7R7jt=Lj6nhb~;IVPy0?{1EH zH2Ox`TP>D>d6u0!a#EeKN57}Gro!2DK_CR41tLS}X~4hYl7CS#Ol@@%>>+I@U_MjU zn*9)_FS@gEDL}T71~(vcT%M4~M zP5ukxbaA%Ktty?4Tr2rRVrhYL?IgCFXIdpbQ?P&!zo@M*TevDep#}$(0sSa@OZ8hg zLB72PU?naR6ZSPfwCGmMQg0AeJW`dDkf8vI*!X4c7~Ex5Qk8idO7+uf-TP{n>VYWq_lSRMAPMtXw+lGLYM%VMz5c zEk#~bxQKwHe_Hjn$CctGVJ>ZxvT<2gW9)pp?-D^}GTOeNn27$hdu>(36X7igNmP+A zDM-{4Yy(#J+tjhc>4c8?iX~SrcxAGN!Hy7+LH46fh?-puzJ13ZY%cXMk(Pp%f%hW@ zh)%5CqSQ(fgEE86{&}u#VWhCXnYy@Wk>yHTGJf$w7L*_*Zz(V!z5pL%My8MxgpTcB z7`<}{bzW471+?rb9+8^A{{U9Me$Z!6X5!-EwQUYO??3uZ1#+#}gj%U@A+;(nrj|`> zYW=2^$aQI7AOM#DB>S9Ga+`Nx;SYnqV^h=}VWV*KEUf}m9eAowV38H;zl?}KDjv5b zfAdLHpJ1Mq^R_-AV54i+>bG-z1zbJi9(lmW+NBrGvv#F;jiGx9K@H@eS)tB-8DW9m zv++|lN%>nZnOymvc`co(JmqTVnuk6xV$tHX?{{#>3WBrhdv~rKcgnlBDORFM1L;ai z)=#`ZrGnt6!cx{`7$wA?#RVrp`cb`|^~c3a)Z9|vbHWKohS?Fp?M}3PQ5TPduXsyI z1h_Rc0ZTu!={{R(33XGB86xnWFU0rxK?p$~&KU|Q~PjQT&S}=0E zV$LzDiPP#A%Ds6^Tcs_fFchUGF()|d-jH9ug(KyfAcURj2NFBYc^9p;l$12<$WCQg zD(-&unO67rtOk_e!f;@Jx2+jU?TZM~p$*Nut*pMnklTw8i9R7nrB^oSZB8|G{{W2O zJC?i%JpiZcZMKl=M&zB}r6y~i)LZ$av2M=vEyV>n11p2qywsE39Sh`7;Hg9{^dU|* z0SI)AN$N!Pqip_Z#jRczx~I*&LK1@g!kSv00|5#UQb6Dt#wIA@;dOWVvq-dZ;Rs6Z z{1%e6xY#g4(oZ5x5Irie`-sXlX7B1Kd*U4?$@`Tdo0Keuf;Xl6QlUN>;5ylrL5T*B zySGiv(}BB-Tf3)i?yP*xfQJ}$l?N069#T+|Il(5D9X*GBCT8UIIro)S%v?a<<~_v5lSB zw6->WGB4Y;@EmEPPS-$e2FjN>l!B0`{Ufy>tOhRbo-ZM`tu7i&hR_qSVDb+jnG*&M zd)EV_=~s3M(zLx?Ci>x4&Knzhq^jI5<&~AL)GZ=X9c+-5DE&Y{=6aE@X(@8b<51e( zmvn@%&E>-2+wV`1ElH46g&nFUQmwNhgz@p~*`Yp8{h*fL)2@=`&9P0GWq`Y;5EReM zC2I-sQwDkG*wELi0V=ac%q-*dT532hy#wV2JE$JWjZ8t%v^ro>)Fo zr=NK*6cV{c-J}iy9Ek5+wyUArTRn94op!60IGw`btA!HkP5^;1j{T?&dr@88!!Ftf zK2?PloMjIw=RRFrslCU{69bb?+0^WAZEdX<982CN!s6DHhY8AHgXK8j`%uXRR#Fma zwLyOm-nU_+I;(v_4+S$jeqYbEGldcLM{d>chjj+7wH18Ow0Vf~hLACycF&fftF+uU z<9;)U@mpGhWhiZEUey7$ah!LaX&3lwV2i6Dr!0_mxrc7VkarM}qJxnKIIc&F?QCT< z+4a_gE3+-yHFVk%NO9*c?b3$?{UA>7usNc1{Xi|%TiV;a--eXsZSF28oa{(FM|x1s ziD0CSJ5qkdm%{e6l-VdWo}o!_ z;TGMql5$l2{{S$z^(%&%EVRqX1Z4^kq5lAXdgbr5cJ+r+{{W8rDoms@wIwYcg+>IG z4r%yu&D#>|8Z&N!vY~B$mA1Ck(L^O*)S4aJ@W3d!akt9B{u*JVBQb(vVuazy806u! zwkx#wTTiuHR@!j+$Aoyd1QMCXOkmezZ*1kW1gT4LC`kbbI{_oD0KzX-Y7?WS7s%b_ z0l<+1yjM4_xAA4&<882oY1!Sr)oF0HJqok?P)S2<50+QOw0F8y$REYlnp;-QouQUg zU3Je*pw|BY5&U4qg^6y#cK-kmGNj<30`dEi0Q*-BrRf^2^~#s2-YVwpDkU$LgKJ!I zyKV{fuA0l>b-tvVtCXtkO57A%1;9S|0=lu~$kjoA&%gTr0E3=b;TQKalCu8*HPzm5 zs_o0B7I5lS0FPMayFEYrKYrkiFh2siFnF?_@-jKRgtZiWnf(Vx(7;8@t9`-dEu{n>*oq?GTxpO~rP#G@ ziHUVVNB;n5fGg<_*|B~V;jwH{{J=?yL3i+dx)QZ8v6Ce#fnI{c^#1@wBh8fgEu40t zt6x5n9lU&xfZFO7-fTQi@8{3>g*l)qEkKF+b+o>nH<|+JOF( zHInVW!+kuG@w(Jjl0KHyLHxj_Sf}qptf={dp9GS3?&4x*x3_$`8v$@1%h0o}ckoTiq>MX>F0&lCSD$i*NBK;zyJTxU*>P z#>+JS0D6}UE4jbqHRbptO8940E~P85c{mUhAzka~pALLIa>ml_ycm)culBBANBEmw zy;w`P;Yd)(44`%Pucc_B@x+4r>r~?qC-5t)7CdWlvQm`aXG3Fe;i3y^1~{5zgirqf zZ2)@lLE5GGg}7tT3Sf}EuvT$h@kr-)XM!<^`x>g7l43tKq@If-*a{tPs7x&JIRcyQ zLX}bg1jp}Lox=epp;vAkc#8ZA%-p#iWzV>!d_t2XmC5oS#Y#$tDI@Jz3%5k=Tvd~L z=;2?Q8XA~FO1g^BRtddu{Xr+~TBh+w>Ou(itxM!uWXSJSUUqCgirf1@EO?$;WRx8|;sHhZ<{U~+4wt1e^ zv=Gl4q%LBX{KO7sD`dU|0&-}WDQ=Pkb6M{0X0!3gID%`zbC77#^aRbfiiIl&+OytG zcmikyvP_ZaDpCm~ObVa`0a~_`)`$rL1PYPCsVRVJuzS`v&l6tcV!W9o&1jTh&@wVI z0qa;RJZ6blfmsx0v5l87W+*3>gZ|XP&a|=oZm$8J%ti07%yA$lXhNR#yQj^H%_%D2 z$gY29QqnT;fu4TV@0J8i$GD=vi8Ta(6e4N3i_72zR`5*VR!VLUl<`G4dm0tRl+V(A z=^Ri~W-Bn24#sE;hd>6meaQsl?MbgLzzP$qsv54o)Ot8~X4R%L1_Gs&%)X=P(~9pblS!x(Wu7~*E8 z;pH5rK#^Ov(t<>U{i=^OliHnhNwEdnekdfB7&+@e3v?xCcJIY>PBfFqBzo~cI>dtp zG1H1=A7dUMrngDDy-Pkz5Y!>xV`PLX?BOKpvTjQ*rz?E=Vak zfOAY-E83_bGJ4W$v$UV)oK&RMmV#_qs4IAlrzR^Y_5V<%kzroz>PkXu#Dyc3P0s_HHZNZwg(VhUPJ6YfrF;-75@ z{{V+#Nf;^{YY!If+$k-fbAV{=0lO5jro4|ZJ5pl@ntN)`gp`t2;h2c#vt;yj=#S_t zRKA2Ka9J>Mfm$nMR0-snzg-yjWQIC$prS!Yo-h(-RZdAb2-F{qn7J!poU99vS)1Og=nIc zXy7#_(A1BdQ(%%ZHs{;}STU#FAwf;t7Z6e)C(IApx@)Ut7Zw}|Qf6_@0Bu`9hSs#H zB7CMIvU;J{$>y#u!d8_jv?M400KINVAB@&FPL*s6aSbgo{{SH-3F}vBeh-zmi?f~Q z3au9Vo*bn`#T8(v1pf5fe*XZVRVCN;H?wZx;H~$Mg!D;P(fBk-yG^?#0S*UHs0C>Y zpUo$7;c+)k?GuB^{MJn0xEpnCS*g{D30mIR2=BJEQ@+EwA==(u!ja+?Axc*U5M6X2~L z7zHIn9QQNLZteBNIFgkzLWq*$6bbE)Dq3;G?;!!@1*N^iE%@&IsCp$O1AKqAII&aP0>Nlp zlL~Mr$~(<3;M!l3$#YXu5{pFm4ZL?P`-INa@#RlnN@S~L+aU@{kfw@uxcb3TmOVRr zR5In2IGZ{H0B#t7JI?Hy2L4~kxobyUR3R+GAR0I`GG@m9;38ZrRh;rEuTcItM z(v77kQn#d%U=T()>5AIHR`-uGNkY&*m8683Amo#PBv+8dwe8_Tn{kI3aKLf3wxk|o zJu7E4{3FCWAd*&;f}W-oJcCI78CdV4?BBO=+5k&THFV^*ge@yln?WF%=_Ya3n_J$S zTU$YGW|Hz0qK^yyYXfl!DkNq|Bu5-qDR}cQI@^qvEE1-GQP@1hnB##5KGe?E>AgLh zg((XVTa_VyQqBQNXXzs*CpB25V#@fP?Te>VQ)2qzX5^(Wmk}Eu82~C}AxRye=dD>< zb56F^#lEDwb8HYDEfly`cWy{pOc0eBt{eM?Dghnq?PQaog%dQ&1`ILyxz6b;(|IJWAC3a>_ilM)%0OYgN?Q z-EDDhv@{gkh(0n<;!v=y+<2+N!Hh&x0d(qa9^8vZ{ATWa zIIo^+UlKcYsy%p{W_D_KAr7>KtOo)ul-q@+!BSQft?|Hw_N(gO8|l_4MVrnk8l|gf zwr>3yTHym?mb3zuK*E6sbj>`WDRr?R<5u)_4tz&s!cFbKw@u<2DzpCpF=c86NhGBT z3k6ftdWn(wubo|gh#DoMib4yaOD`!%Z_F7|grx)-N`gt?NGTv-ispVW_{R6*FyDsS zG?vx@dfM4v!CLq$+W?@RqD^yF)+xAJ3yM;dtak(L=~adZMRJ=czN2*&L3)qG*P64< zxo*=eA)uh21Qx(2r~sbSJAV=C_E$-`ZS8FUo|aI6#vjAWUSBYW)HJ z8)*8fw8j;p-6q{BN<+4XP;IANJi=12H+_h&&mRcsCGADDGTAP;pho9BH6(=&q=DAI zQE4i1s|MJ*(!5o+l$Qx@7h%MXcO=OKcN|v_lCk|tilk2yl7+Wy=o?CO^se7pwr!Wn zOuV~C0^rPV0J?A&oHJ{`*KR-jv)5b6h%&ws^nD~r1hp1Sv?_5G#vo;7grC@5Q-wYDy+aUc`V8M5yG@n_s&r zg)e{a^&0#+clA9epXQI-RRYahBUt#m}$!dwipz^rh`Bx13JaTy{_jklnt1tfpUer8dOyl*Z%Uy??IRzLK4* z@X|RT03yD2_r_O+zxahF=|wpUcs7!*z(JvJJ}X<}{w1awK_hc^@~Hc9_@`Omqi9d1 zCeGsDSW1G6YX(&qRG6dx0L%+*Zg7FyIsDhlfBK%<-M{!1l`TrZQbe5g25F6h_u87g zK-J@f{{Xa7x4s6_ZAlk@;vd4z z5OJ}m)j$3pT9e0<{{X2~JZrXdsJPllApU5CEGPn$?IxbH@b|(AQ1O4o4~I&il>}Hm zvY7OWDH)um}2AM;3@$Ax`b*+a!Q1Hnn9whg1lz4TbLrMToWMRbrplFJqZJ`rLu*~usE8O zJ2V%FC+Y|9S~R`FO#=*DSRCP;15ozq0KlyR_i{}yLg<#< z$fiK6H^o6Ch~l$Tk;#guh&sg+BS{CK{8dz-35*)ZxCF&!rwN?EtZg20;E&dSHJ8pP z2nIm(tX^|*IZ^kaR_KCBAo^CbQVUh`(>xEnRtvpC<6RSRaJ zibcDEQa0vJ(amaG07@6^6e5)*{{SbHQ4P4X@TGmLD)wbzK`sL-LW+72L>kmM3Q0~Y z2H)sg2mX_{_M@Bdoymlr(L-K?Z`ni{qDCjnRSZEpO!^)xRG~wYKqUv6M-XU+nkSmo zrY9uL6o!(XlR<12kQ$lhD!u`u!V-E#P`4dv=w3h}sH=ej4HBe9U_=@NCuR#1QQ^p$ ztx1Wj)sq60WL4#eITPzjpHa?9qi$Cu(&egoub+|Z#1%;-`_+(u1lEuV$**k%BQsM< zi#ZvqgPI*{#w%w8145BF6kLUh(UQO;6ZoNBY{8G~M9P36K%c!)&AWWx(rJ4Qw8XbQ zP@qZTYo5JwW=ZY$?_E9IDnR~T(Ok}?gZDYEOpz}KvzWbf%7R4VGfI}|DuMtenBOwF zB|WoCOKRQ*+U7gsM$I^q2Qn#zuOneNp)Ku!5()Mrif?ITPXzHe6NZxb>j)`hlYC=8#lrmS0pr1^|`(?&P4Gjwm^CupHvZ6aWi z?OQL)!Iu%&9}mz%=gVNh(*HB9uOmK>;}8Y3jh3hU5|sQ8CHMr7tB@A_)Q{3g};HyoCTh z)H0iB0|g+|Wsd!oQaRAdiNZ;WZ(x-q!jdrqKK13e+9zbI6M~l2gBV9GM2VEm3W>qqKt&#d(=!E9maTm$5zOs8qcD zAmr_&SK5M=xS|RMJjN?);_B8&l&3pKb4c!N*6f#D2u~wC*EiIcM{Ht@m8>nf4XHbc zjwyTk>&Y;eh%@?VzmadlfS^w^SfH)YSqO z;tcSd{{UK|Qcy`*gpc%-LOF4@Wob&c3;`U{arqmT^fK|Tok|M-0Oj{|MP?S$ZuyE8 zob9BJ6~0yC74GCGprHlItw}Svd_@%KekqP_{1q&g+X*{K1yVZKJA9>=6hdV2C{8JW z)k>6~;Hl+E8NeT!X3E3rmji7q41$8JeXCuCE+;0@T47JxzR}<)H-qas7eWm{LmY z{#nl~p`mdE`%N=bM%Wc>QshQxM|EZ!b8C&;7WRODhG*|ZB-X{`{Rvm3J83F%({j|A zNo~MP{(4rdGi|sbcJC$aqi|DUK5I5wE7;w!=0jkU>A2=7YjD+@aeDRAk_q~@ zsXvM=u(GAN?fH$_@SNB{_aY5IT0UV5ZTBBwDij=01oZpS-BVyqdO^BfvjL%TV8ZSk zD+gm?Eu@I`is)_ZukD-Psad$f98{qJ9xCI!j+LgIiw0TT;@(Uhs#1$(BmV%!CX8b9 z#%)NpE-kJ%DSmHYI*!y7Z?Rn8pfx23SBRjrAt%a}`E%1m@6RXN zwq3YNN6fq7D1XZfGNbPw(zCa1Ngu-qSV^6v87I6{LkxF8u4Jfe92 z0OG5zU0SEa@Iq~^nJECORJRXN8RnTVw$!&8y18cD4t%B$PDNEVET}0=_)a6lovF$J z=&tkm`YLQLuyhhe}t=u8x>NdaTR|rnx@)!mqq0EErO>XU6XHqu*068}) zM)jh`**8uj(Mkj%BOGQXm)xayH=)(Crwd3h>eb<_DD?nhBdIk?uKOn$TKjqxn^ZoP zHp}HVDH~EAWT+#P^+`C!2>a01`c1=&ahuj$dfl=U<$?hG;GE)%S^`j#n{8o5M`PuM zp|;G4^Ed?d;*dLb?w@R=c&&JfUy+u~d8Q>xP?B-arD}CUM%9URJ9g|Ba@DzQt8LNp z?x>Xs`cx-8{wvnYHkO_>&xWVb=kW{91*l{clql^!kxo?YtIVja#;%@1ycbrG6dYPi zn4iy*Qxl}PbAWONxM5l?hPu(n-Rq=ejD zETpC708-A-aEY85BZ{hxcE?*cx4(Yk-L{-k?c7p)G?gpwrKal-mQDpuj2{F6sp*|TWTD*(ymURaKP-nG!W zT6v{yNDAv96a6t zuMLhoKw(P&{=%2(T6>olEupqc>rs-DK%Z{rm$2K9x|A}4r4s-oNw2hjho1*oT{fSY zxUyC1#M|54J`=yEB6g9UV!h0cJ~lou{{Vk;<}u1%RNhRV@9cIT;eWx`3`4DHhWV^G z5Yp0>AyOm|utfWc{T*Yb>DIbmiDjWJ0ar?Pp)xv8O6@PTHxl{{rS+RvYXv?X%L`i$fMp>L453-z;=E`@w&f$M zCQhQd5bCy!YEbRXt^1B7Y(@2rsBy;3Oe}I!{`94KJ>|L_x4JrY(@m>#7xL;!+E4Nl z2vh;o^G&Zcd)C(5_srQTrx&m~jbJ#1f70V&L@Ngh0@wGxB)wRP7Z61)PKc~lN%cNdf|rvmi|SeVmLBaRE{`f zUv+;Wc}pX-bz0BH?pqDEdIjHx*b#Mfpnw25$9U-!rN54D^;gPmnQ*E)wgP6jwO;Ew z`-{6ecB5#A4&d$8&`Jd4swCz+Q(ZShw|F1(El4bZxlN@GxL4^QS2xsrw9b`^JbvNU zx$!H>+SUHFwX6>pV$#V|u3RMKCFYYKv8%=z7PznP{n^gt2mb)G{=#i5{7Kbr zf>Eb@J@CGTxXMc!d3|G_)ZypWE338mgW`LH1zhX*s7X)I!W(D-?2%1v{shzY6d}UR zz6k`NlpZ@8vG_<>CvKiV%n%ncE2B2$!o>3;b z3;zI#{wcS3w#siV71hs_LdUVk71Zi_exYo_${Tp}mauE)`j)Tp4YR9Fwxxc`QZkdb z_c5BZr@?l}Mk-!|mGM`{jlz(^kNSVg5~UBg1Nzl}{iA$qYq(x%>?t5g4rxv&{{YcA zG^L-7J}R*wv#s8=20~8XFYQIO{w#cA(erCdc5@F~w4wWo=lZIzoBm0bSl^}nP5%J> ztLlp}sA$?l$=U%6LyZ{hcXXnd`2No8F8n8^IWS?Xl4sbI)G_`r{6?gNURr3Hib=?R zzzHAvO)|Im$4Ay2{{Z*j!8%gbey1%|TTvwDBvJaU{{a4r{$dV4F@Mb4PuKKogRx%; zG|(d@FDDuA72oKL=+_%m&X(<}q+~3AWl~)$PWVaW?zgnnG_%M9_@M*dyIns`)7wm1 z>L-9k@JIF)Tyc|or~d#5M&tSZqf0jJ8W4nBHU~g4{b_wf7FBj0D2V`3Ml>+ z!!A!FbX0t`(?MUuSFY7={`J_CUWseuES67|4>##J_p1gLuzV#nq5hRm^faYKY5xG2 zl_U@6K?16^$RvQjxBF4L1z${a{{SAy^0?|#*(+34P3_qK0GO%7e-Qy+)@dpqS_#^v zBiTlXMx$~zj}c29lmY(Lxi6Df;4ycpG2>d>sPu$v{lyC7_G@49SzCYq08QyX{wjs) z1mR7#zp)imzQW1g%7p&_DuDk0=7V+=zDn%1Q5!DaKs_F3pSZZgOKy%V7QG!%K9e*{R{BsqNdy3D+6HKTH z&{9bzsHI?z^e}kDjGCrOOwZ=Ehi9B}NrOTtt7#NYXQ$$*rss}n0s>^Jb2V_>h=>D< z*+}hDj%!(rlGfM*HL_b*L0ZU|0%EKTlhUz>X^wi$6odo}f+~!O>qNEzPE|8O1lg4o zk1ZO}-GU&{sU4`sQ34HX1q7ifNrR8?TL2*X)f18@HDt&hQ6O}tM2QHIfmI>_j%w*r zNWsr)^7)D;245NGhI4~OIU7~!K)R?se$=*7CMsTIij;|*{?*_CB$*hTiroVPt|>

<%F&%M%o;uZ45rGs~WWf?d)E1-x6cj^XF`C6l z#%Y3@fyY>(mm)-Gr>!0dfk*{uBa`-^-B197%^djJr5VZQlw1=A44I0Ym2^B_!ERT~ zaY!$mkf?%VtuVQ9Kq(~hb4lKGg&oI=;hdS#4#jUXyvz)alp)i?q=}~KP*H$EuRuG# zp*^W0tsN<}FJjG2kU{HCY?)GH24;BaS)*t;@5rWhpHa-fjMXv6Vq}oU(KtS$nl^&v+!L0BE}Rg0u^v>NT$69WZRfw%`F%6rfY zbV-9Xk#NbA`_rk|yimn=J3-shm)u_9`&MqrO1#27>1)YebHMLU$&19Bi?r@AAVw)` z*Cr0t1lCteQb&-;BQ%}+l(##hbV-_5Di0bBarBTRW8Q>%>7*4(XVPgU?y`i)OnxGh zKdr6F+a02(%`wzj(A?ec1{O!4pu-6&nWUoS;+ZQ62DWE&#LKQ?hZy$5_am~Qy)AlF$fwaQqN{FwR zYfrY32?|od9V_el#*Mk9Db*9n`m5$z=9-jBQ;%a@9z*U=;!KlWO`E{ZqiaEYRQ0i^F={E#By;t=kz2H0$Co$mp!1{A;4cw>Fr&r*|cq;y(@8N zgrJg52&+e~V6|qJn~at7LE&D8j&_b*Xtn?=!EI?fdjnA3o*FE`y)BWn1&z4&rPj=N(i}&Lv~ZOXFiDnDvs&WUb;wh-S!M@q@LbT!5(;oHzHmhwDXYm^5#@RX9D&q_VbGyJ`x z5X-6(VNWajbgqT%KGNAGHts)kH*$bXQVab#LB$kXy$3s%pgz>iKaj$9!4TW-JcGGN zSs2>@e)MB4KHKFFthSB2iSY`j@rnZ8!NT%^?Y_+`MQhe;CBPrS@nQ3e^+E!Ln=~qQ2YE9Vd?L}-XkjpL= zCS50SQq@HivvF>lac6L)B>w;oR+g3TJjGt$!*OMxrMT*_5(oFA2A=h|h8?z2*dyxk z2|u{a6?LH;qwRy+=&7@U@=~{!sZWeIxYucBG!BzZ;zz@|?JR;L`hxC|Fnl%>t? zg#Q4gG6HAbw0_ZKFqN%3b1!-20Sk3$27Jm=i4r?gxSBGiBh*c*njb8Nh1uGKHsX|5 z1Y_5M>0H*iq1;{}#dwXSri3Ld?UJcEDgOX-HQe8}g4&q27U3lU0bmq>lh#H_6u!?$ z@R@yvEz5;&Z&QkP5Fliza%&8psQ5F+*f3Sn zNj!Io>xM|mxuJNtzqOpjmZGcs&zW^IBWF_y>6+@%soL5!7V?tP5Sa>6z!k`^MZ2+i zOk8l|M=`+bT@mr&mUnc9mys-R{Wj02FZ@W*ZGwrt zDpwm&($r_ufCX05{7ca8pGp&Uwj5Rgb(JAA(o_kLxUZ8jqhA5FO)JAtkV&mBG~`)? zrtkP_S&w?^{{Y1C{{T(5{J+7=A1}%MRF~ne`#$h56LNsL3z&rhAl4VE4%@8(N`Ow$ z5)XghHO_oBaHC+$Yf;)0Nx9?iGQA=r5qhO2?Ki-VKX)fA03bV9{h$gQa z#gR#RIkMJVZZFxxd>f&iTJK_3@(Be%vYo`oNanvze}z8>CsF|ov2Uxgl z6gK(3{j0w;mQ1wXj}|d_@g)_r=o)T~a{GI9>qQ_o(3aXt8&U-IL4nU;1yS8C^3vPX zblc0O5U^ZRFCeLd(M6?sw)&*{RHI&LZKl=qIdgK@DQSz)ZLXU{kE9vi2caX{l53w3 zSf$y%yt~p}VJcEpb7|tC!_gs0Nf8|+R+KTdxBZSt%2%OQv43Y^`@+iV(U*zyBH@KL zx#*&lGk$j_y8Dfl+o4WY*@+ifLVlH@H0 zJ*RK&Ol|a^0zOAhV)uX4pc9U8LBab`h6y)vH1owOM=~sJt{)42^xmmP65YM3)Svy? z6z@sWE?RV|-q{v5$;ff{L7a3)KVWF5HJ7x71!^{|+OJ?NvaRv|0Hh^Ae$~$EKjH~- z{l7MwWSI#~t)}TGtdd~+Rlb%uyOXc(N1W!VFZccqw%x1Sx74SuuWy<`!aS`sBeFoB zX|2`%ceTRGH1se;V%bhF)>XI(K7zRtu)^{{Rt6Q2SR1kBwJ* zCD#|Ure)Lw%?S3m^Uaf|h7=Ff0hLeYp)Aqu6j)s!Txa69+dynwHWR{$J7@1zU&V+? zgk2~*)FB_MTvg}Ak6##cm$sj+I6rem4~h#4T5h!VE|Pf>w3;s8Q}`zRBx^b?&61}M zG)VyWFlIgKlE~+-NJRw?M=ZOQUGCbleBVmB9_Otec{@eb=W0D`?afPbw|6gjcy zEeGh?(6SN=qiX|=EyM~sw z)%>dKRb8X4d4fuZLY&g^VNT3idBpuLIgW(QY%iRQ!iTRkF8oP>=y6#d49DiOZ5ohw z%`&g~jmNhY2g6!6wz(&vJW|-sb6!Xjl!8BcDqkY;4O1%v04#TdS(K;`GDrrpONu~{ zqCVvQ)I)bl0B#$I>?r-L1yYb|;Uo+i&^+4kq^TtL5m!q>FhqK0w-Baa8A=4b?xox|3&N(6gVQzwemA_9eEV#!4m z6Un&DZuGAKiG0VENc%~xP16n_+Mg-Nq#7sU#qNIeZ@^C=|dQSDLUgMms>Dl8-n znW+m27)h#cPh9=#tJ}8*u*ND!nEF*hB92pFny-d472;9p*eOg_N+Jx_?)91l)$KT} z!x9(N8IhV1&b8+rGHV6AW2FJ88sHN&qUAVOywgWs=cuL6l%AbMB!@y9z^(w~(mSb1 zC+vOerMVbXc0bmT-Zjhw{i}*-=!2#zFYQpO3$w=KSzIPJ5kM+rX1S@OwmTj(c91|O zB#L)r`c!9kC$(}`!NRx-2O!r$WSEt6n!Y5+WTUXMbubB5CNa%9ZN%*v2Dz&>91tU< z(=<4sk>-#3Ytw|O98NYG{lFhezZ4$gwM@wsnUg+Zed=3Dkr+Kq3$`+i3~d_RKbC33 zO!)+=ms+Q3BABJUhY3AT6+2RYX=SChGDXWrq7pl+KecX`QWfM{lnLf5YaKe= zf?Nn4qndQfb~e6Zjmj8_t3v~QvJ%T$K)PMK&pA6*Ie-&iLei7+?Fj(z42tEO*Bfif z(i7hzVvM)e{6z$+W2#~WbwV-5yE&YkwsiWL4zgelNUxu3I(S(mfF>)oZsc4w>V0WG zt!AYlkOH{-RP#m64+f%quKvxoONt;80mXAyX<1sJ1cGNAerveZtkjaJ`izVU=M$s4 z@{aT-Cm``&U(?+q(Tvqw5uv1&sVE9sfSKe`P1&}Ay=XvOU@MbNmcurQBm{y7J*jYc zx0Vu;Cw56S)S#laOOuyGvW2g4$VW))LoV4wk`Sry6zbO3#81@;?b?qFAq~h#KBk^9 zZ?Z0F9HMmRoyjl{dh@z+Sx=q{qoqB3suTeOwFUCU)b3D}^^r>`A=J)cY!{p@obR%-PjC(PMa?xPfz>itd?B`%eA z2^gj8JAqrzhLAcAwLhs{6#9QBVAcAY2hJ8_l^p*7wGn4}m)wV5LVatHYWb#G^C+n) z9H_-+=fo{Kpe+=Zz(~k%Kbm!u-HY_Eqq|a?-U5^q;p4Rlb!>-1yhW&k%nVl~&bad- zUK49@tMx5hQ7voNZa3!G2$-rWXSr_-h55YjmigPy%KGNuis(Dm$Di zQja8z0ad!yTT^Onoq;C`AV{fkkHJ*s@=;oqxGD}MwL-D$Sv{}#i6NAhw(wLiJ*jT% zX2RB%e0L^5i5agZ$?aXS{3SM-f(nX2`&79$w2LUE+2wpyc9fPa-}9|e^vEB1tbQw8411U;kY6X+`D+zJZ$`9yEb3XMtYodw# zg4-#lfBfRn*raAX&suKMY%MmWs~bnsa6kZaQ>JKbH179WE)bZNCy%kF3(;<$T8)ZW zvp~m;u$6uj-kvd+qE2h3D1Ob`gq1MbEtm%+bM_q6GSf&(a`{?%ql5Xbi@;J4NO`9` zB*L5t1gQH}61Gs8QWm#~`cj|*ea~7FO?EsT^kZtuRD~klvJ^zZsk>H%EucK(TSOhD zTR=YipS5RVkPxQ>jG+k|lwwKy)$Q%=E?h3D)vW^{p%MQ8;^KnPt+`re=yh#43zv=A zp#+pF+y@*{oN^!!S`FP;*@bR-u&}Mc%a@9uS@g!y-jsYt@eQqtd^Vd!iiV+L@j=2u+#pAIrgBkKgA9DO&!HRJy-D>T-8Awxk~W?A?qjWNZJK4qlWU>9 zd35m-wibp~wT|vBUQy~OeJf7Avk6MWe=daWZMK!LBfsXZ^|P%Xn7_Ew+Cz)lhML^9 z$9?BBNf^013u-z;h3M;3VdidEV{pLQ-^(nzw#>=%pCWV0jLcUy)xFE*HhN2!TAk^Z z)U+QH!;{O75)uSYoA4@*+gjeHE-zH12RzNZr6d49m;xkvQzkAhk!qVeKMD5sovKkn z89?g^9V7XTHCx%0q+P8v+W59_KC9h9Egu(PDhL4Ma`+$Ki%o*>)TVA7VZA-%?{p-P zrGQ{3cW(Xaw)V|@f|4!+jP43?C~{KaIfWj)aZ;^H)#sGQif-;)!os7}r6M>O7_AuJ zLb+@57i*&?{kN_fw*i9bx3xg>ptdD)8F*o^6<0;b1Fzw@MNK{Uan# z5Ia+cS9~sPEZA)CQyX1KRuVZr!XQyL_RgUzX7$$9sozSS@HY}ro)bL+QZqgAR!f$N za;=;$rSNiH)SN4BY??q(E9K`fR7e~o8c4r^>{+*U%aJ#Ck12Z+pE44X50s7x6BYHH zO3LQlmdooeUEVyR+CpG7*pgK}W3i!^@Y?~@+FGg|aCo=34msy(7}}I0`Hpkao~0MX z8Dv##_^aOoeB1kN+ke7Ri)aNwAQ8q$&IsbUzl*=ar(Cl@OTZW*DcdTP^D)yOy?&W{ zKwhJ$A-0NbZS0TIs3}MT03PII`qw{w@bh}6r>-$*+l%nF{JW?D32cB8OdjCkpB$$b zcd)s_YM;UR*WmQ{8cJ>MLW8U+K|y0_2k%U8lmZAsPZ>PZKZ^bzUWkWv1*PrVN`gq- zRGB-Kx2ynU*DS`W&C&{x?(m>vX^+~3<0-y%+24uAxTnyL)%RM}-EG21RtWT?wY@{a zw^;)_apnSpyk#xkwChrYs~fu3tY}wS)}Whqn^9nY%ym*^Pad4tQsUZgqB3$xq;|jH zR*vnZ-Ot3&6{MEb$N~sUN-(0K{G?)fSL$swHnw)Lr#83RV`{@&cPSu^t^$Y6KLpphURr-O_TM+b`fW})-WJ!IK;a#RXfeV+{uB}oQI zfFyt?*R56(b8c-OMr9TDrf#Cc>F~B$-P$;yq=wyEwyYTiB$2oNp$EMweN5e5xxLL_ zPP^)NrPiKD7{}B)&o%4R-dd?j)im3ON>mc9Le|+$EU5h+DpF79gFv%=dvf45Z#8fv zf`<~emI&tu*X>Z8sc-%M!a^?X4K1yoN)&CZjlfU;0C}5KxiNt#2135n;@;KU&96M= zyJ~GIm0a3}){#7PSM6Ml=BaLl?!QZc*3XzVYU662IYEQ(O<30SCZ&tZr`mj^?QCkI zNQ@n>0!gc3?_CLzHWRA+O>v~U>u7nKk@TdrlPW&5GfEiOz9!UM4p~_0m#A@3v*s9O zw4S?C2OM>-Ygo9|wHxx@pL=@aVi4LGd4+&v@>S36MCtlx#O!#6P>n?pw$jCPfj_>SLLg6oZ@*+jyEofwWp=7rL1b*r@{R@T-!gpdn=HWEkP zKc#eP(wko8UtKo05}EMc+Cd`_GuYDP>DD(cDW=(W-_I*86%X7RgU00VWmM9QdkekM zJ|(hbh3VF|jP8P>nSp)5S0{x(z@Fl50i06Pt>pN+EpK&cb-1;Sy=pe zt$Z@%TRh#}tVe@zjo`;(dsQKh5!BUxzXV*Q_ZMdYdu+70Qt^iXjf(Q0$22f$DFl#! zuwp>SPie2Q4~_o-!~Xz_Q{en(q&=m+Qa|EZ0ZE?o&t55hqy7#q{g*F)!>wl1;wvfg zw6{WC!IA{ePV}RLWzlq6u|Z`0{r28Zz;v!pp1Zlv;3M3gvpr2UO4 z)HK`O4&`?G-J`m5;E+=Ddr1-e(F9QzOob3c85lAM{MQjFIVIM$?)}LPCc@ z$eQ%FKYH0FMH3U^s9+hBF;+JOoum>yYOpGbB`Niz zY!r(^RuAc##ImXW844b_sV+o`s_*{*!C1@iS+GErNUcJ~AWZt!9w{f2KDDY`NeYqq z;$c8;oP&TZ1qM-m2^33@?xlOi)XHc{BH< zwenFWXt$DQB>kdUzJCBk9rKcgs91t9zf!uDRbstchwOIO}Yp?@|bDt%wJc)}gDGn%rC>PbFERk*B2jN)nj#uPaVMbGYNTG~A}d zd{8R2M1o5#XZeb7D2rV|^{5_G-=$`|L~xNNW=PFJwJK0khXbJNMP2L~qq5R<0-%v8 zBy);>ur3=ZZ7EWxpsqhfVbzqvb|M5$CpFyKB@DM4!jEM2uE<5WoXyiYd)+T#D_9fG zO>w&57U4~!%8+CvAF95UsuUC}=_i97D}&XUYF4Gh5_1(Cu~Awn2ls6G^X;zb!WXzs zU>ennXjn>8fPu_`{b;R9;SEl}o`af3yVWgl^1@ON2{T@NX*+E0;@Tdr>AO*}8&-P^ ziq$6R^c~3v2D!iUBr>h6qjLL<(XHwiDoF7ZlzJ6pS4?EOAmYa;w@L{v?O&l4b*Vlc z2_*14(#EcmW6Y%y*ing4SPEKI=m^C+Yv7$W1#;7l0#^wU7^QA9h>`(5ywm5K@f1N( z2^}Vc326l2FWgjIZIp|ldJ{W#Hz0Qvxkm5d?&mSmqUzyAAuWzR)U+w-D%_BL=x!r- zv6aIuveEMm$>0iC;0F@5iBu~lF)-j{`xQ{(JREW?0rNeLYd zOBz6Ndx0Ps0N_PDO_~@;4=;P3ohlHfl0Z_#_5!!j`48668+II^tdc*N;)!QVNKBVh zK;r;Wq5E(l3Ek>*6=88`GV<^_39gu9QAa$c-#}@ww_F7hr4Ex;R*Honi+f_D)a`*q zCsMU5{{V$RAab%Ok$m&pb}L{!N}yHNJaX)doa56IfAsr+c#rL8Jb&7bv(v&z+qvU7 z+L#n`PX7Q9mqzo%^O`XZ+%0=jpA8=TQ!Ef$1IsoG@@xG?45af4ioweOw-OK%;E9zU zw8^^!se7FYS>wzkO&Y@pN{CZ=5fYjY7cCi2h=AEV2}?~PdYG%*TKS1Pl9_`&Yoa#SFGD8@9l zR!{L17J8&&tfw7G^R@yNl6Qj@inRsX4YqbWm(@W)58km&-Ey8=AfO}Coevq3-6waRG zNGTp-wt*Xc6r)IQJ}+8T;Mv*+tx}lX;u2JP!g>lC*Ic{1OKfTO0Rc(@%Yreq-~%U_ z=qY5O9*Rfm$~$LCe|F7=)ElK6rwkPcC_+a>j#RIFSIs^laqFw+8nSuRJ-aR$aemm4 zne^hmqCOsJ^oR8z#k&C>R#KHZ;{8Yw1d=&4Gk*^UA9;5VT{CWlPsy^v5Tbem2dM6A zah)17gin>Vq3TyohVB|$Z4u{M)N}Ug53P4P7sJdlTS}IvTt}EG;k_uw=xrhikLH@{ z?X-lg_lp+ITZV8D;VLQ|$w=UO*0)y;Gi-#ve%-KAh()cgoNgX;^^LtFsWqb(-oq@a zsT_vAxJYTo5;u8U-MIRwp#z$}mGHuum4&JGElFGIAt4PU`ka+|3hJ7RZND*vPT z9E2qntfx5YNusUwTXa0?){o(`+7zXr!k#;fg06r-$5TORs#B?5u9m}W+c~u^T?7*2 zfSjQUFe9lydM!GIkIl7e%V%pF?mS<|lF(n5u@AKJ@~ zANi^{@ks5SPcc{1*|$^Hj%o|JR)=g?yl?YeygsF5VIXll=9)5mn_Y_FxYHloT|BnZ z=Y~>mZg$(WCv&c?a8^4cVCSHv7OY#lsA~=Bic7}n0jAvBJKrk_Tb{852<=m+w|jHq zYr4(BWy|+eEbY}HO|}S12>{@b4m#62cEy&I)zNG+mEYTG_Nq{qWQ75?F&<(`_Npbx zs~1taIe&}39YxNX<;AicE(IHQwq&6tTT(>yDI}WtXT?9^Z-Da`7V1+?tpz9uSCo;@ z7|d7c0G6-5B!1f9H07nUD|N*wP?V`C*~j4Gx!Z8X^`?sT!(mI*tsN*x8(~Rv$U<|@ z(~57RmrQ(qOyl0q%G$1g`wT57YJ+M6XfWhYAtoeyOjpqU5z(zL#kH!%!vSrOr^$ks zIZiuIGhJ82+8eLfxGdO3>u)741}DM_K_p65%2t!qMIrbW)pi$*y{abV$NcGVzE0;9 zPZD|o)Y7QB;?9?mN-dvX_)(SV+7cSPai) z(@}AL<3zdg%1Kd7eMGdV1c~TGgE_8o%}Cj`r}*nb)?Pc7jF&Q!+Jpo;4= zxM5Zn+HLv~ZEk!tTiiNUv?xeVhVjlg+Nql8bKwM$=kVnw+X>VDD|+*4?d}4e@QCmo zOO3iy20+~*Nd)o_ASszW90~-FY*wjL@7_AJy3MzWOR8aD zsj}yMkKv#1Qsups>QrwnSasG_1wjCidjr&aR3#}(U6$h*nYD0KzXwrsQrRR)YjVB8 zh$GUTHFI#?s+*fTq#Py3w9gGu_Y*$Htzg!#Yi!VAq!VsX0+}cqRnOJ=;(}Y2$!Q-d zrEJ#t((I3VbVq)X#>~DJW08=laxq`kzU+eWkd{Qz`QSx6m1p zT*jrT-Yqi4=H!9`R?&fx%pSg#R&^a$!$R(?-nl?X@e~3={{0OFkwwzczMZY?>_zCe zWW0;pYle;sZrT+(oc{nic^|Q$RA^n#DB0+H(9@JHebeq(s*H=ljaD5kz36J`PAE)^GnSlxU(a8LgLnzB*2VE0m*R#JVdoz*RzVS#YMyjw-^;!MYCK&36iAw=eL znyb$ws#{-@YZ`rM>@`pDzg}w9dWXZ!vGGQd2b$wca4kq2%GofWW0n zrq6Udk(21En^>o~Hy8f@0Lb?^>Ppc%H~6Wnw{Dv+h}gFHj>KT1@dI>p^LtlIrRe_v z1N>G?nqQCb?fK=R;VqM&(Wur#~l#h$E^fWJh#oB};)@BWw6G^aq- z>|A4K4oGnm+$jX&tMbrvBTI<3wlePOU*~1iFIS zz=b732=}Bv3~HCwC~Xbg7n(8TT8aD-$KJckOP22&9xG|wQ}io=U3iu`D}f0)uhcDU zAe;l9vs)nv$)cBHz)=WMVSSv)KG9e$rQ0X?$Wj68JAw8i``1qvgtW*BGO38{Yet=0 zVp&m0=~!=(js98L_w{hGuEi2dd(9VaqCf9L~~GMYP~1|jEZ?Vlg?_XDsBo?K^?1U#b!q| z5KWPFxjP?EiMy;xtpf>KWrdXw<)IJ^;KXqis%0inN>8DtPtyZm1;N@tP$@l#t&&U= z990u?91!sP)UCB7654{&d4rKgz5!*#_02&*AoD#c5Tc?)RnA0Gjo37D2k%k{gPe|Q zU6M~xSqX^#)MyLdg&&uuiq(0LZ0x4o2t^Mmn*Y zQ*!RmVAqW%dN2UEcb`!duHooUz7M4la{17jEDkHHP>k@rBfuy?Ob+{pryehFh*`hyw#6rY;uSlTtW!tO6jc-V6r$?cN|+zL9f;^zcc_G=U7`dU=k7HkrFkc$$fpnK zt~yRhB+0-PRD3=T7r37DQI{vsW5srJO1gQ{Mqr#`rIfe|h~)ZLP5w@X!H}u%RQVL8 zLUie?>zfB?l{^NS>z+C3`j5lDlZU?HFZE&>O2Vz71QnFJp?ztP2N z>tw)^XPw6s7-m#^vEn10jN-O@f zU7`)ZCEp`OQM8<&Q&fy;>{lh|+Q#8(QBAr+5|Q7Mg;cUxon(oDOQTooJeidpety2M|CqO>$bjy2@mdBw{P3VbGSaPDfER1^a#~ zWgrn143XrDMbYyc>jf;5pqa-siTfW6?ox_&$?}Trbz~_U!b}ol0=S)3mk3{ow+yGL zuQFU>&q@u+0IAz+b)CzZJ;hbcMVW*F9p;qW?r(R7CVx>#%?jFX3?)ZuA|wHtc+EAW zG_m_f#jOAYCQd7uwzo@OWvmi9lSZRaVZSP3L79rOlOW)R=(=%I zrxu|Yj?^=ctxBH}K|Z}{TJ;62k^$IGB7uJOWWqV_X;@-Eb5LoJ4)Wsmb1+-JRU%wM?cn~)Q{ z@~R}?2E@ATEbvLiI+%|av!pMjN+oW`noOJ;t2Z|4+!Uu20VKhJ{{VXCnRQQ+qJsyD zF{c}CBr9Ms2hBz@Y!Zm%J{GO5fw!oL6)Cq1o#p`jAYy`I;Uz!9>5Ng0K3pTk208+H zs;OUOoM^HR-z!a|HVosEX!fqyXpbGjN$5GD!SN2ucJg?EM6sr}$t!g!Qiles7Sqs@ zsvBEf0N&d4zQLn#n96;FD>%{8s% zYf_a1iIGLH*5XM?N=I75RMoWoDX^>Cmf11Fj!)W=#?aB7Fh?!g{{Zf&xMKo#$J&x! zI+m0HLN|qCWmV<;Sl#OOd_SXV8k342^Gji-{{Z(Zr`m_Q(EcXX(v|9$T7Sd#WMo?s zBDrUcKAF=A{PZrn)h+Et`+|bK(tx7?4{GOiFN$xk)3@Q(SWq ztZQe+bjw@t>33F7%_y%7t$Ps|!ia)LHQBtBYR-i%w^o#mja%Eb;j)w5lPXd7HPnCf z{i}|L8ApV{iMUzaK8a8J*Gs2cD%#y`>B$O8vR`oqN3j0X(DA1?OiB3=j^cdY-{5Cg zx#w)$m2g-80Dfv(_(uJ^fNb2fu-JD8-2u&`(F8#}SJTsNa)#Y?)Lt!wjJoHReIjrv zRnDbtYYS2qvY`PjI3xqqsNhlm0K<%S*sIK(l~BJ;(OS^hLzfp0T^C3%T)yw|U03=M z!c#O>r*yPfrAcHo^ThZKY#_!!S&a0ci}o&UxNhvUxyz~kGSM7_o-y8#KdY&=Ts5Y! z@h%O8=heB_IWZ}i9)g+ihP?|FERN4=R|?aTv|j4(El!cOi&X~N&qWdDJ!BKsm0D^K zTB*;L)EsGQSW9Sav~O&7P5>YE5@=OvcPiMoowl_Du2D?_83n)t!+^Erlag(^z%w>U&eU+a;taPD$)mM6F)fsMNkEaZD=NHv7UH zO2Z38AgBo&lY#C%D{DPD=2Ud;6{oH*93<{hDscef6sd;Cq|q+?Fx{~hkG$RU?51v% z7E}^*DF>|a%~m&7wf#i3bi|`k)is3|b*rZuw%fFpL?uU}QUnC1ds7Qt4by7>0L!JT zdqm%YC@ihT zWNkt~JBj_OkjFXpPI)5_)BgVe+05y>N6fG-+;eS;KuIX_c#2O!{+Rx>`tIFJbp=QP zhEf}Cut+4C+JFa^1Q|834X_HF{{WJMwWf*MTvYWv6B9Aou(!QS%#~g^ZC!mU2mpZy z7*QRDNC&B^!ygq-vNFiqvpahi?k+|9z}#G{DT{l55CpRkoMtGyrlBidAgyh{dayB`AF)I;4{%ZzLvrOyG3IWu_sty?7d4`=_oKxD@Ap z^C^zhizK3{6;O-27wXqcURqqga<$u~3^bLX1qmCVnCaeqX-~_!rr&(iPCB&s4j?pA zHtteLF7D*u`%%~SOVn=#?Vp4a+E#~K=1`?|2a)MX?)O@-v^Tgg()_K20;WI`LQhm5 zS~0dhlF;Rro_6h~t!{#ma>_~?adO(>Ai$mp=9%fMPN!nSb}LP_>K3i`_l+hHujJ$v zAoT6ca^L3G?V{1t0sjCFo$R=QDOe-+igMlKFS~DkytQ=s4L%YIP_BTUVDVJq$?~E5 zH-XzAmsI_w;#5$kUmM#BAE~vDXN>f$-))0#r&?Q!5e_#78&pZ#jt?~#S8Xct7+R3Z zkbvc~jE((B{?XExD)GBZ#oC7!)9OKS4LO83nIclJ=KlbCE)5do4lP@GDK_^CQjjsY zrE)ui%=^<@U*oW%%gbTpC}jh1^d>TrD0QaY_d;JOON&S-xMh1)9>W!jceggKI-9k4 z>PFBO2biRo6CC~PoyDhRF4lt`b;K1W`72uEf}QQTkL5fX0jpNtLRW50xK11JRKep)2g(z0Z6k)$+_ia-P@r504YyQ^{wojEh#SBy5+logXCF={*)z>meL$b;r3UR z1*i7>Q7N=VvXbqwDz_p+PGi!Fcet8J09>>@sw1k?-kjJ{R$yx=R--nn4H9%QYe6tFzPscWp>X(eAO_wd#sY`BfcSnNj? z(-|o5+05eQ(cfEI=$ftDmiEQ1Ci3o2$~XbHe325`zd@75aK9J)4}b8L=@$$swicp9 z{*yd$kxlg5y&GR={J!uD1-PkgqGcgbg*Ks(W}aDmQG4+E{{Z@Hv4} z=%lD}2YPu%IU{oZf8XgXMt=lBvXV6m^5HTRa_UNt**k=zX$nXqJuA>SO65NGav#v71QbW=(udNr~eiPEO0R6&+@U@#vP)_Q{Ym;s_O8Ot)rXO~&EzbeQ0Um9Ke4Zz*)Zj? z1tcxJ5UT8SrN|Q!1Wk71$nWk%Wgmug1xvOb72|M&f(VRy6Ht((kN6^&Q9Da3sb=lw zljT|1F|~ehe>IBNY@$`QcB>pIqau6lHQyOpIUH!AZrRlokh@1ekwXTte%02AR05CY zNtyd}qf-y3W2L)<0fn|;k96}`?6|Zi!gF&&r!ntMNC|-@bA!itit!|LrUSvb%cH_q z^NOpMYit#5@e%yVp-<4)@&bisdQ|TqN=zPywO_-=II2e9dP`_zjoZ3WNe=+v?rPo( zZc6ReP)|d}VwgxXP*x^ngIfMTm%%GS(S;^GdR7V~k|Y`>7bP$k3qkKz@S1cfO%Q^4 z2NPPaqg!2pBLZq9XEk>kTUb~iOiW_1QZS-UM`~VzNnFkPCOs&DqiF=zjV!ih1`Q5_iLF7bv}z7Y(&iKjRoy%q@lM&?C)9PJ zQmU&i5nB+b zz?wnKh=DZ#lkZyWF((y5-sF%oJt!E>K#~mCfSxcagcM8~_lRRWRxxpcYUtqh6h7&a z4;7$P0w!j#do>{-aaow?DmV0*s8o)%jArUVPDh)i4pV{$1AuFDB|^R7lffs@n!pB1 zbHLAP-K5~wty?IJf%&4_wC+E8#x#BC2dRorc>X(LXq$_O1k7}#S9b*NlQrQ@(d)t> z_b>A%J!yKM1w>b#QCa%b2c=*eB}An2(zx4f=?i6HM0cZG+Cq%PQudHR=8H)vp0#L4 zR!&c4p4taIQoF^-QR+-lt6YOT(&y2-a8IR9VTgkC?%Y1r&fYXtx=60A`(4tbIrN|& z(}|K~{i~6t&ZtQvhqguD-&G_o(ln9^ zWweZeUqI5WJTwic7&(exVQQ2VjiO+2?_CwJN(_;SuH0DCIOjJe7?#3Qkryi!+`41!`n zG|u_q22LXelC5TGB+_HV(|AebNb6QN8*&W`xT^pYRzeiDD?PeWseBot%6V&(&$+C^ zcl}^dDYjC{{f!jHpC3}E(kk#xkd*RF5kVPx^QjS1uIbQk4lv=$=gsm&#Ulzy_U3 zdNPN|eWjCIylcfJYUVkuuC>-11*>&|jFjLC=XD)HVJdA3+B%SF3yvvf6DUvWCzDy^ z;;JJoe7ibZjd=22d3+bKa1$(pB|1r4!C5)|g9al8}+=b5Xf5lu}1) zq+h>q0Z`fj87ikMP*ZRf=IT3F32Cji!i#88Q-nwah!xZ6cShh9EnJC#$Q4BIM5L=` z8jXr}sY@~loL3dAFoG2cSsQwa`fc66jFSZ*N8Hyhsp)=G04pD=xw7MZqWH2U+48Hz zkfq8%%yCGrN|Looh#rEw{X<1j-6;{c*FABiApq}yI>mCuNge8OXyzPn4=Q&opGv*B zaXDCl(oZyd-VL!Kw!)k|b($ClZiov=WwVbhQ7@Jwe+Z+F`N%E+uE{l}yD$pfN|c_P zh^?D#t9euvIsC+NSgmStdGe02$)YZduCDy52uVl;`DQ6ImkWLZH^I-9CMnm+hdjh@ z^&-5xAm1oRKBl(cVNZ0ZL2*Jxvj7Y-jkRendrN ztPV*Fo3wMmJ5Zn_1Psv@nj(qu9Fn3&N^78M%#cY+P}x26`0GVBrjTJ^pR`rui`yYA zjfqZ>^6(0f-$D;Ht6C#$CR~+$xu)uMM5vM{i%{j8)0p{i$rW>1yAZGs@twO zN>UO>p@^l8rPoL*B+eo_8Z`_pGK8e3xgwc1`U8qV$sm-aN|O_gDb1@XN&=JuRnP%Q z*m0XRe-TQ3-J(#BF&|-8dj9_PVGJb}44$q^*1tfWE2cENBP7|;{L2&_8>er%eZz72vpb}K)*rxt$$X!q^ zpL42QA$M*{6_Apn`%{H6;v}!gkOwE9#cya(%X{2b=LntUFeB2TIJc+KNJ_T52(i;F ztU*yigNG!Zf|&%_AWr6#u8V-9_L@fiq{%@`h)c>mZd88VYb3m_sZ)$4Aap_B{b<3$ z8c6}EOKpv|1(k1cN>R?`s1L;qr|lVV7j-MeE^wPy0s>E}kM$MGYPYW6DNWfe#I~d7 zA$vjp0AcN1ey^$7X$?62_0G$53}o#-z-30GEXg{GEetNLv*{X>_P3{gqpGRoG<=rr z61~6m8S@(AHG9oGl-(>O1H!IVCPDr{c*hmVd_Ab@zY=N*Q#Q|{)nXcLM3n(JA^-xq z@$m9&+-Z7)TkdCTHg?OBoZzid@A2bEs&1GI0fQG}Fq#2&2rQ*4X-m1e zAR#HiBcSx|YoTW47QrP@>y3?w+4O~uquROdQth1;7_9`rcB|txT*Q4E|!+O zFbjwZQCO1OatA-mU}BNQzhkGO7`1tjr9FpMQnyr+Ah=Ek{*WkV-ndKcIM^tQ)TwF) zNkkd@CauT z@&UJXF8=@!VMq!IP$T-+AFbcFW|u=MKQ4i^q>xmW)!6kEezS2bYF4RImWO=k%!Lz^ z_wGF_oI@(S2~vU_AsY;Sr1A&Yin4qIu&t(M?{sy$vZk zTZJe~%5^H+DO+kAf=Ki;n#rv-OzkL60nDmMrr#>H+4zjGlBAO19XQ$;Sd zCp8;Z{8jiafHskxiW$#vYU|V#@*7Kqz_`MqVM7P)>s;dPZ*A0+vb}{ORF_5p?OhFn zTbK%Q)>IV|m4cEC@w=@aM7Y;Q+rKZ{2495&qLP1*_xAO#H>Tesm2sz1SB1amYCXf7n?4H(O%2q|;uJ-MceqHLEIyG0)fer`bqBv7{}$4R(l zAt&{On%dPj=W5iE z1H=-Q42UAS9T!wKyI5`2p_D85YM(m_AEf^PGHcO`hDdR1U)bfTW0j+hZBRMkQS_di z^PkN&v|JSqWK2`NTS{xMk0MDK5)Q0?Bids~tmQ%*>AD%iBB zZ6Q%GM??PA=jF!nKb7)&804}?@^?Bl>1q;u=`aY2?X<@bp}B=ffJi)w`Hr6YL=ZS4 zGhJ?%d{9tQqtlwc9Es_WoIVesFrc(|2D5RpWB{r36gHoC;_^T-bi?ySb zfF%TEDPjZ;lzqM1hygB-9UY0ts4d0>#Xw9>W8SWtT@$MaE!#gP-*ZkLQn>!2hdwp3 zvAHSfB2s@91eB2q6{A-xOr=UBALbG%tqVm;TX+(n8Cms^XjfQUQd~*f(Mc0DYm9}G zTLlF2l$jJLdB$cJxjjOwSIAf35kf%qtV&GN1$YmiAyZ52cr-h#loA#(-lo&59W>06 z0z{HN;=O=*nu&<)DnC)})6$U*M#;KTQ41a-dm7D5wDM1cy`=O#Yc-Gs4y7JJ&q`~) z#I2rnrL}@4I`^#l7)kZdN+lO~+VXrrI`yetvR)gl%1$TRjcZ1?><9q{aXl&lJPMU0 zp-@v5C_$hyEQ5`y6I-ST#bhubR>@XRVb85#6B~G_OpyYw0bK~GX}H9}tze>1B|z?q zB@nR-iRl8e3CYaX=nykn!$c_sB6j1wSvmBrd6++X@IpxEX&q}OWMrT_@l=8V={+lT zH{4A@PCU5xpo|9K!S7g*CI;`gqFhf-wUACTS(wT&M|$u`Jns7n(MS+QT_~UxqNzzZ z38^iTb0T{R*|y2+G)rvO4FM+9`qrtnqr-Tsr2v%!$4bTkOF;vHiXpHE6}fFc%Tc$m zHGpz=jt~qUOwu>ja(Scfu0b@V*KkisubZ5a?!hvFwJAmhD^yzp(9|}+FbAz1!iOf3 z2{5-tNw*Lui08F=Ozla*G;oNJD>Wo-<}0Zri2y0GYKGLP888hRP$mh@XoiwOpS04_ zkjp}a*#&)DzQ(aupEDb$-kzzHfyw&{$!)+nk?Vs)>7dwz*by2V;%Vk{+Ft%?TgZ7}9 zT&Q}}i(s9m37=HeBNPVNvtWeGaalRU%+U}8#zkPYh*9lMqE5xPEfsOi0rJXG;+wjp zl4Q_Z(mz*PM+IZXPKvQ&Pt;TgVN4mZ9;9>?n=ZDK1xhEfMA6NZ8s zsw76vdHZnTAdlLy8gPPgXV#mkC`#w*lUc3Q0RbtVohdjkWjTBR3owZp{is)zhSGQv zVoqqM?hbe!#**GSAo+*W)YRpZhv^wc`vkhV+@b=_Ds|wHqD&9ouX6a6bu-A~m#GT4 zJC=4LxzyaBWn&6N{K6Hu0#)7x3w5a>v=k;J7(QB;*R>=eNlJQ|iKP#(dXy5N2ceTz zeX z*_3z)1!_tY(LLx(yYjM7+P9gGzb#;287UGmWO&Hap8UVVWwM0P?`_p!`D^=Ka zi3m_3NCiebiJ7j?NZyp7tdyk2Cy`t;TB8=r2?z^wvtnW^pwq9eE(%IoDZqfE6!B3= zl&zmgw1py4l*q*Op|8@?K?^cKoYs0#TWrh31ss?)Th_c{PVY=}PTsVL{^u*HpAe}n znI|xCE0nyk@j`-(W2Y6~+}NjPLF1}v1@)p^1Pt+lD~?J`>x7c%;NG@vg(!d$Vq^+j z?Z_BuJEOlf&}u9dD1ZRz6qwqjvQ$c^xv1Od>Frn|Q{ov=&uWCS;+6pe&`}L2g+{;) z5ZV+;Cu)63qP7QgRSc4V0*Le#WNF)!ROH90qEh25r*f7Gx`Du|e9ErK4f#@f)l^r> zCgfY?n^@WccK5B4%U0+FASnLTC=IqkWbQo33jUS1h7ti$kbg+&RZ&Z#ZcH47+~mkX zJV$DjTEe$2N&~Q_Ot1h2L{Fw7yj0U;kfk1#N})Ld2Gps9kUd3shQfx?D-bJp2~Y%* zlpo%Wu+TLPQ3_pyDN6_vyO+pqrpI+CoKsh%O*#|0`ddi~i9Nr)TEkDd)f80AYAF-B zdq^LeaHc*T&{AA+XW~6TBucW2R?PG~st$3@EYvB zVC8)VC7UY*DLyaFK0ixxQa|-pWkJM~A*3HHq~QI{Z=`4!*YF#VI4%^D1v&Y>1L2Z< zqgK@05+y-MZ>D{H1$0I&uY^idugE@^@TKmcg*$B0ZCC&mJmA^@^*w2wyMGRRI7tdt zwSPbJQjg(-?-5+isqt;zxaQl3LYhedwh<^o01<)KhAwVM2WmhZ15T79`DFg-cJy@b z{8g!HD^qPPyDLIS*>0Vu?yvTs!q8Mn8_!Z?l4(142XF|_w@N9ygj*wXipk=ilVZ|c z4q6S*@c{sSkST*VDlq`ZPg+}~T3za)YD6-^L6xk;N2#WsJ%xglmSXMEg0&AI{^#vZ zzEG?)4uq8_Xo<-F)$4-c3X`<}05@?!CslDu+(0CV8Kz9yt?PANY6}e859Y}}!dQz=%$6hwpf%}!KDtm;1{G)?|PBk@~@8L2w+ouqYzO)^d zl%#@-W|ZFSy8L;z*B2S`sGW*YaoCSg)!wstHsO~wZ9(*uwQSJZmcY!ot|CTby)Lr& zF6UUh7uw~+Z@Ly1TS|7a(U|}bezo4&wWjHX!LexyJC*?4w|z>urv}B##+!ceCf?lw zT4$oYR(Um)9Fs}^0H69dG@Z07u+yBeWc<3(Y=WJtbRFwh^qwfDM1%6{SBk0L@l>mb z!HiYz?iPl8M^cn93?waMa?fCq)7~j-T9xyk{&h`SGEa~t(3MBu`ctH$fpU>^Y|o6;r=evlH-xvpU+C!+tF-V(=3*YCgEt(H$}b<|h+VO(yHH>QY-em20|B!?w%u*7M3JGvX_K!=`!1 z5@M6R@l*Gsa_!Q@!Ao*s1o8^HnCL5xw6wZ$g=(6jqR~gll#&$kqks=gQ{59!y>%(N zaR_baJm^%NtRtQeDl}GUOHtAf(u`Yj*=Mx-%a)wI)!YrUv&sU9$3T(EG0@gldK-6d zl5LcR(4Q@~z$A{U^{%Z}$b^E27IyhOMFM?Rqs00SC>y7e%YQC~lD7|(1y1sLCYj>B zwk&eBq+Z#Jc5Gi=%Yd+n2|^PgOYJ^_u?35bJ;z)H7Va!52_qyA)#`gtmv>AwHeIZE z7OH^IRyTT%v;+6X=|#=J+qRJnf7r%$~n$Y@5V9_$g?i zq&A=c5E42M#YQr;&W5x><#Y=iYqXZq7dHiQ@-fht>p&hgTUJ|n&w5Q44yLE4f4Jxx4P zNNy}%vb9?UM~H=wq?ify9gP`ggrU@w6f7wOCvQXQD8Z&FR!DH9!-S#$>ohI%YPZwW z<17XL01+W5NQFefKW=Dk_EIleH%-$niAvNKCw@C~N7}OE#XBIiws)kF9j269T0X0n z10?M$N=kqtF^u|**HG1~+C8!zCFhhA4up|B| zG&vycGwn}&2U#R-$PfxgKWdaX7B;+?SMw1OP-*%bm4jH?q$I-N^d0MUCu+RN9`Twb zw$ZfA9)N+FtlVwZ;Z2aRc?!W4sWgOc*+Aef0|aCaDjm5<1quX(PC(~1Zxn$JKqsg? zP>wv1Ov&w<4bhSdw{@bDhZc5@4{u5y?%Xu|6#^U>Gef07rBFc!Clp%_q4onWqyx%T z8L2G=4e#5rkp>Zr-Q`S#e2{e5@ZM_rconO@j`(? zg#$d(i)-_A*tl&awelCY`6({r*Rk)Vu8M^sW=nGhGWSDNHCNp!MX}Kc)FVo`PxHZg~t3xUW|p zk;g8>ymJz$!{d5k zc}eRPkzq)jN>5md)MU{JOoJe5103c05mPkD!j?50WhCmb8 z);<^%Qqj~+Q^W7g0|KqEqH$UwAvLdu92u*icZ$JBnh}FrrU2rhw7@f3?Bm*N1o8>s zMQD&sRR|*|irOF;tO7O+#&I+HFfk$07Ji|X# zDRsC>89A?zm8N|fpw@0Pku=GNq*4ZvwL(u6($^pnIjUl8NhHTDtf+ZSYqY3lg z?l^jxs-}C#rCCYcHKT3G1azs}Y^sVOITfN&JqfMbWX31h(MY!=n8&3$QU;6}X;aCj ztjI{7l?>aQWQpxWganTBR!NB3GjZscF+jZP$uvu-Fk+}JN{o;X-YL?Tp`C>hA6%r9~PBgYQ=8omjz7?(H>$zHP>aOUX`2%}TCBd2{R9 zsGFi(c3|UjMkCgRxJW{9Wc8w5e1J^P?rB}arW8uJ#X@pLqJb_Q37Js`(z)x~fOiNg zBfT4ac2biEJl8dQdXc$GTo|r7Na&1hlHIxP)vGEE@Fty2$VQS3MZ=*aG*E%nN*SWqwsP`Eh_aZ-JWx$| zkFUxj6S-9G($!%D##vB~`#}Eydg+C|5tBidI#Wz-PzV_V36g82X6oBdLJNTi^1@2I z%0AW2HFi*cf+9ascok$u;NhA6v%*7aZYV6cchgl zoB(6JeC>LxHsfPnXCir@{weN@d~PFoDM=!ltY?GBGpRs zyrmJ)QK(wT+JKhP>DS)1T6c&{05}5zu_JCwfJ^`lI#r}ChBjJgAJn746`~e{k|cB! z#VB>qfK`x`_Nmh?bsvZ>lWC*dc*JCr^rQosKiaCNT4M3EGk*U71SMf99ZAJ7(zJW+ zWu?1j%38EXo!ht9v6LtIi9FP`Z=rlLvTof+#9F#fnGGv#rMc{-h~7u?rhUaM)%8oy zh#f-po39VI$Z9Gyo*(uUnR3iyugIQgd% zTeeEU=^kZ|$El?{j-{yT~KJ+NH-mRKA$#E0k){+qWgfC^nNNBR{@Yy#1 z05z&9HkN2PQq>#3^R-opWun6zhknRs}nIP1CeMUDh2JcWE9=X2?+l zaXqSU{w8Y8rCVqX_zjfGpIUOSYUW1wR#v|<)ya&Xzckuf>_u`KKdoI~Cu@ofTP6r) zumW-4y-l*#>zJLsiRbM@*lDldJQ!gMD&htY+*fQymGE0h7r__$(FbL^Nhxa|sME&Z z>?Elc)3lbf`#FA{DfC zq-$ClH+vh~*szyF&cSH}U>~dB(yw)@uJPThsJ7Y?bJHCG=}DcxI74IxL6sF^XfVy@ zu2dO6Eoy>F)|0>=?@?_*Qbm&G*%!K-4m#(E?4_YAQUrq;u5lNGUVYas*-^4&!dB^- zJ;30c*7x=azI_%}Dcfq93ONN{_0j4701ux97EC9>vbIk9msNzQ_7T_8xZI^VG?CRF z5skKUI%b@voLlM%@Pwr*h}?hPx+eY?`HOYx7l19MPzpg3RA&Hp%`mgE8qT7WG=wvz zS%bHEfRxA7eNO_6*X~@tb(Wbzt=I$jEdlhTexf_`O&m&{_5IkdD{WEFgt>WefVR^6 z$b_=VGTRC1DgJI|nv08jD|IgE;FG+ZH5T;r{Jo$qIm?R&D?QcNS3hxj ztFTffEuijka}qP<1Bj-KmOc9oIgXd{Pl(ITtx9OAYHr=MWT*~Gztcfm-K?!VuHp$R z7%|vZ2(ePwRokgkekwq06u}7+G3i7%w#|nfZ*g`103tf=MF)~|(0D>U3tO@%4qRNi zH>+8DR``dD6~?V{{Wnl65%`%6r5bUaE4fAM1bFx zJEY>gES!0>y(vgqh9@Q|-G-Lcih)W(m-@3mYIQ5+r#@v}n; zY@k2`X$12;RrLEuJO^&BN?HoTd%AnmYd;3rOq+ywnBpdL%~ll2$tq5)^6;;DHKVpp-nfR{cWU+!;vB(LU=Y5C ztuwH(ZFxzXmeb6^6Hw|Dt6v%AZ(Au)Qb3Yp(uWtsNlSJqV@_()={7oxo)d~k@ZaDp zb~wKz6c(`ZNb*UEr8g>TTw#lb!MI?dI|jm)w4uqK@mIFglG0Y%mg~(U%1H)A zWRzaSlWleu`z@_zJZnpo4ESpB%Dus$l=EpM>_}MsdmzuL6?NVFug>s4LAAwpD>`&E?dPC;5>N$VWY7s`PtlQKyo(vix7 zm!?tYVs@yJ>p?B;z~IGXrI<_}m1fr=QwC}dO_xzLZNiRoCnAh#X;>qGW`R)xNv|^O zDgmFgRIQ?lIvpXv^HCml#yx5oZ~}AXRt+sh!rD?)k2D%Ex|Jw6lz^3vK0TfqtwZnL$<&O}9bHdf}!?GM>3 zmQ+HJxB#RR$J@1fSTb{5m;A5)08cw4I6gnf>ommSG5{x$*wdEKq7Ku6Kv|C+q`uNX zazBV8lRVQ#R(AKEhP@=H=WL;EP;*rCBp6q?t=2>tp_LiiH0hJP*NZki)q%?`#$(F)t??L}MOEvz3pI)lwG zN~J_{1rX9fJQ$@-u8h)sj2CUa3Ub*}KT=H%wnj|)R?MkQs!@YR6e8$Cp)QVx8K=~R z=mob6XPg5SM_EJZmVBxorCn=BSyD+$xKJafHImR$K?H(0tetjbub?uO2@2`cwMsam zypKv>5Jv=7F1A`x2ll0F;AG0A`HgBE2Z2M(pDjvfy(mrmLpa55)9OGWUSN6kq0ydt zRrdtVZi4Cd8kUmc0-I8S^{)h|7$b_!fd+C6N$p1?YuR28dOeHKKGJa&DM1B9jCAy( zd>yKeCy5nrmOFT@RAlTIgPPC|BpEe-2qt^gCn7jJ)(s-2NF6J{k8J-1rG$8X>G-O5$W~!7D6_}bM!|5aOQik@EBQ=51=F%fPQ2_!mSr0A% z{MM;-<`M_BV4e+-C(HM$D90H!3w=go6^UHqrC^?Q&8jiQVq8~473^|zRVfPSNvvb% z7dgkQ($@qN03UNjzVXbFK`GfLay!?{Hkt1t5}C?~9%z9~E#WE0N(sJu(QPDlgVgt} z1wyvQY#d0UlD_fypjuIr1_mo|=tn9u>0M}uKvq@hR8znMLN4SCfgZC#xp##|2WrjG zXeQ$55AJ(Xw=T?>DbI5Tu(`Tdo6JX+F-u&0N{-@v>y~pV1!IO@4iPE<`WkU+kU-B^ zu2$OMjnF{FJGQ$5C1hYx$0cL6@^p03?acGS+ zGDJ^!?OKG&MD6<;3AcvBB28+SWC1Gs)5VKcY+3UWIID(|kYoc1nuc?>YRe~TfFh`g zv0dp*D2?356p?O%uQEXNBAZ_m3YBt6u5#`2lK_qdMs3*f=woc^LZpwyHEQ5(85!(r zm$!LKR4BypPc2e{*fE-{C8Ba$3U=ZFB1ts4$GMY5KBxpI%SxSer9;dsIT<5w7kDr$ zcF8=5^q>&zqsx3!nS}h z*17VAFG5@Ug(yf8r~)gMzEajyRHSfv&r&j|6;8H@Rx?s_^R>2UZ=hLlqDZY`;;=3C&aOT3L z5QLA?jLl;0%Y{U!%8$~N8G+iRP!r|y_MZLgYj#j2%o2H7r(B+qI*KCB&DWQ?SW=Ea z=kHyPnxt9@TF|00bHJ`mdaG@uqR=Hj>OFm`sMMTLR`sbV>;_FTq)n?ni~eer4T(^m z(?o9tAQ+hMHS_x$!ly@u2*46~itF_CDM|W|Jen`=V$~A!A$chi#RGC%Ww=&M#CE5O zZM}*-4YZo=j}dArf+D%)7K=hj^dfP4i6}t?o??ReGNly}CV8h0q7Kxg40Niv+7OaS ziRwA2#ePa(z@2u%DV4y+ClgDa*)YmVgklm^C(sUSp=(UK)*em0i)vY%%5Xi+GEez$ z!wD$U(%z$~r0re8q_b{kI|u7C$4+KyiY={m@+*>B>`KSNNVK*_qw!;|*xD^J#j{D> zadLmspVBfia6f7jQ`atUEfuHf%#9~UfId>(`JEg~s~=H6${{pc%-w2NFI`?OPCDt8 z+&XtDd6S-l&|p+s+LxU0w2N<^f0E()PGx;dpFEJjUqnP44vVJ4sSp_=)vuw}~N5*)pNE2RT>iPu@;= zrCOH1eXBMt))8r71}$y;ss8}hu2z2o(v2u?U-$n2MorK3j27mTq&NwUdi|_mgs{=| zBN8}-kG(F`_51BnAvdn>Wdq2=QkGDC6FK)ZxB%D*hy#G0DymNYe-yu3PfJEvlH2Un z50=p>l3+x04gmrQuIXY}6Cj?z9HXXVbNuB_YX$W$RIOs)0I1SEAQK9th_`{q0+Zd(@C zt%$R-O0d#;6WvkN@x@U$J-&&)F|@YIU){CxjWpf5Y_zsOh@}xkG!YWTz}XO)GJ92D*#OwE778~5SJ9P5|Ql&tXvJOwQfOFhTurAiDF$3RBQ^nif3`H0?F*QsoS`0BJdKYTypP`RQ9~_qNDDvanK9 zYTS=C*&kO>4`_<#bp1NxR=Y!1O|5RNKx2Rh^sM7w zxQXBFHu!%|H+M*FfP7LxQ<#J%WRg!9igcpg{`*ZZr54Q%&OgwetDYk~&{wR68%}}n z`--@ARQnyJ{pOAN!kxkn!&V&v|EeYl`OdGz(|kidQ$xf zrRo~>x?C!8Aw^3hbP@jmdW&zQRw*D2MUrn^V33`-aa~E|inX<)cMl@jq5LI6Hjbp9 z`&JLUUAVcpP?Wf(DFd=eF-*Q6adv+*slVk-njx}B)ZxNYo|7l-P*D4e{ZF*t;%!hS z$J< zV{$;2HUi-M!pKTdgwJ|yXJG3pD}=1MJpPg1ncHcthFXTgQUFr3%@B)sE*w!$k%%%# z;8o(1uabPr+1jwA@3>HPus1g%N7{nN+(=ddpMOfz zO>JszPc2zdQd}|Dzv75tNpDctcN92*w4Nl;Hx~-BaZO+};`~4lGeWAxa=y)1ZY`Rd zcOZp~3{7Kv>u$D}odC~3Fu(o<*dZZgKwrI8Ta@C`VNE)TNj>{ew=M@Q(uWdANr@fl zTS);Ksg&FyQ?V}fgupZr@$FUoH3u8&g`5hrdz%}ZaHw}ROOyuzTPi46Z}ZboMnFJ1UhTSA4J( z3SnIUGc}R9S-~WytmdWLRqRnhRHTlBKfMMCNl;KCF~vp8dnp<=BpApT?OE;#%*{^R z6BU}HYPTf&(mE`amhrc7Qk5B*p~^<(Mh#sP!2w?NC)q{NQ)-m~2Wjg@FzMN~Q?_T? zfNzo0ymX=vLe`?W(xJzC?Xx}rG-vcje-M-Zf{6l5Hv_Z3utXEb^;GgER0p^;FmCKNvP zv{p{e(r8@1ji?4H%)*ia;Nt=JZf8c7Cd=7(y*36FlKy%O02 zku;+PHiC20t!xwiouksSX+VM^h`~tRnCV3dN)(qv8qB=1J{BFD7RlC~Xbdn%UY!aB=sj z35*jJsuTy*CcL(`iBHma9jFAL0B~lmfiqP~PeVj7+~l8105`Wxlh_EOSxy1%Rkmal z<1}g+F~wwK$jVAr=1;9kk+ez<zcoY~@OcA!saB=HehNMwQXan?`)F46WT8+s77Kj8_w!k3KrKFCMYc(S{pcL#M zXOw|e+xPUX%6!HQRd#yTF@Z@H;GE#r+iHXX#bAQ11Wi#W+x278i9ednLMs@+fG`C| zrAq5LtP|s}+@m=rIuS$s(o_k+C$(U;l2m4W{{Y&DN`Zhdd)LPD+4PD!9JKCmW{zdf z<|#e8=d4l9xMPAVq6a+2sB{2inHA%3PB%xrEf+gP0&6wh4yvnjqI44F&;k<@DSP*4 zGZE`lUBXO~Q}#3)j->9BH76nBiGA>JlOB|@*Gf#Qp3_#jxBwU_5y>=?>hgC%lj~A! zydpP6=tPo0#0WHNI*Q}^vg|6ES({P1Vij+!(MRrBe z&cmB@piI?Xxf=-vO<>YWj1yLIkWj4Vx)sp|g!lI$K*5j)TIKI9b08#Qx;yq#H*L;1 zu5I=fqx8UmT-h9~q-fH{;YDMQNUo02)vX|Pq;?jtK_p_j8&D8{Nlbg@jw^kOCQZsn z2XKl?`=BZYMKrv406;P(DGSRgTp)F(lLdw_@O<6m>}=cF6U?q<6(IK$%w*&BmNm$twHMPueSPQmKwerJb`zk;`2%Dlr%Ia;Qiq1XEYiK~$K7 z6sqOqqDfY16W!U+V(AS6kaIYtk6aK=)s&f;6>F=k5<1egxJrzSo@$YHY7XDx6O+Wu z(G3Kog+L}c(ti}B81}0Rg#rd(_3d2{q;t(38%}_sB#9H{=8hL(fE2PZ)Q&5iV&lR9 z&zo*52QC()D+OwtPfF-gQAaz8&^fL&gZwY!DsZACP>yPs7fXd?53u|SQiYWXQs#Se zMYIa1YERhHbCc*!G&FAEEv8ny`U)Y3S8=sUJ5YE8#LyP@igbmjz@|(DxAZ|=KvBu3 zTNR<@f)c`KsRBvGGhg`;*pK(YCNLq3U^Q|E`uV;C0 zsBcV4mE6|ErAU=Rk80wY)fT0sB!C70O0rEjwA2fZB?%=<1A)ygduB^|8L7EH!$1l+ z9W(y`ooj1N7+f~HtJj6Kl0n?HV5_i5>P<9OzP_3~X+_e#X5l4v zQauOWm%67eUtPuG+;!!QsaYUQ?erd%SCUkLCI@d`{`4M7UvK?|W53iWTGJiQ6){pQoLJ>7XWVcHi+>LalrRK+Pb@KF2mtuq3vY+ z)%*oFW`imOXE?z1HCvyEnbX?&UxiQ`A4=~Cm2Uq4`D3^GjdQBrO5AZuamN+8N?9bR z_dV*$3m<^t)d)Eje1c2<}G}m&3mY zwAHcJ{w8zfwC2pE1dYy~z;*VkYntoY?OABP}V%wreb#yW#JtC_G#ysW!01M=d zo0H=ZJ$G8U_>)l%tv@MX!HZi{jfv<<>Y_T+-3LcyHm6nBkm5APj5NR=d$j}fAP?)? zG(W@I8&9;oM^p+%nQlhyq!l`@h(z*F0*uu4Wjg-=%v~WVbRZ_)-9JkWjtAqD)caK+ z+`jpLzw{)MUu+F;Z(h53H+q;yjFGn5(WMg82|WkflgTu$<qG4Jf4&d#-vgd zr7W%0!7zEOS$L=^OM^YlSL!Ar#*kT;l?a`utX6>`q-;U*_4T1XXsndCJ4Xh2sVXWy z7&9bf44NsYuv0v>?u0glDraa&=DPGV{{Yc?Lw7DGcDsC!_;iqef}iKduM^E>q3PD2 z4{53Rm*NzSh|Qg`BXz@=fgQV^mC0P|mpay>_4WIK%ZF5V6yP9E5}!Z_6zQn0QtX@i zoglkJKX&+SkX*QZP>?w?Vt+N%-7UQZg=pGxlD$FF4gJb9;er1E+X@|f(E4hJ!rD;3 zc`>S9Ad7UImXZ3L?>NL(Hm{8fOggQ>F6vh*i$trc&(z}m2lb;B{`>*#BJ$}PJ(BHM zRNp??53C6AoBmKo>Oc0bX_i#r0U*nY07VYFhaHtXr8u=oQq*LgOw-SXEsgH0Y{`(6 zfa1?&idQ7vA=Fw%-xH(dEv$5=1O^!nJV4I%fALv-I?%MX)Za)}=IvLollq|2yZuR8 zo~?ayf_yhzP;7&ZqXL~UkjKLMo1SgXmfHe4sDb^BXuDu~iOV-`mfA`Lg#^K)^)!5~ zy^fj=?Wg80Co;2u53tYPwbEEk&HH6!cuD>$)OY}A`%o^ttLxXU6&sy!xd6;3p!`K8 z)v^81HNt%BN(AH#4C1YB$A|v_IVn4_lSLQ~I(qKG?^rgH?ZQ+*Q6>PbNo*?1T6hg^ zCMIBZ6v?JsxIrpL#E+x`VurV}@h$*>Qre|nhMKu^;2~ZY)Cq$zMO*9-FA&Q+ z6r=U$>7pC90`#a76A-dLrDvrP;$2)lxl721%^skZt zgrz%)-~l{PuG}R~B&4Y=By9mAH&vXvQd(1aCB+E``9&$bbh`2q$sq`XiW)Nfj>G8- z3RHb0EBXq}E;i!Svm}G{5x}6@R>x9>8Bh{uy=s+}2f&b$oaUazNUGdT!kSLRkb)F( z$(lCZ)TK8JtR*ThN=!u3+eK}83V;I0Fi%Rhs!AR}!PG*8~@FQZ3Y*Z(hBBv?(GV4axmrF@g zYCMlD{{Twe#TQct9Zb@#Lgltr0u1AHgT-t9WZlJ}T1+^Uptj)PR`aDtdo9D*ys1#%WT z$n^HCP@)MupWcdXSduC>Q7SZU<<@6uBzBtFR}4G|c3XC2kShqEYO0YSfidYvmZ^-E z&nn%-kwPakO;6s~9JqV?A$|MVdwO}Oo0<}rT2|o3* zlF5>yq@M90)r&2`D0vak6Pk|}Qq1G126`E*p>a9ptd>tgnD?)2Q0mzosVR@6RI3Ww zaW$xT9&l(uw`ubo#PJmgaWX*Ue>Ee}@!vxRyvvc0Gfr#|QNiMpTB#}^eL1Jva)%-S z6BM4qC8Md*z%lr)>q=har9G31;xxbx!#hblcdpY*%5m*puZyG0%eHmaV8V9f(-xH} zIZ*VPSz@BL(+MJvK~e;SjQ2I`$D@hsV;0rpAhx0jPtt2H3EhQttWtJ>L5bqEc!eZ8 zB$7f(K&p0y>5@nTy=3C|CMWi(Nd|d7^(7(<4AjN}Q+`~Eg!xRTr+Oh-5}~yAtYlUR zXp`j~YDXm_kYc=_@7}d-Axb2e&T9hb(UcNNjl8m73BZ?{ffydy;H?+aQ z>_vDTv$Ran<7Lu$+Th}P(5qyjWGobs#AF)HS4K|zGDZZZi)9iCJq*?$ME0W_FfrP& zEHQ#cF%)We#V%=aMl|GunEO^UR%{1Kw!>8m5KQq_1i_O;p300X0&CN>=kHW8KUI5h zh?${d@ik6(tB_;futLm0Nj0YIP%`|MDv<NIgLmCgPGZI2G~B@_jF$=0FBQmG-UtM5GWYa$gDL zkWU6VtqbV@fNII;gFy~G0|qE3A2Icj?NoN*F&Q581J?ow6M-@*S)&^@a_iLn=tpje z0tlI*oqT7L?kE>-fXYVF4>3VHIwK1&b#a3#Cp{@e!|pD2oO<(GJ8+mXPcy{?x7!Dj zPjf@Fp+=0$K}RRvm{?c^sXez=4542oZNfnS4wc!D8j0u1ZL?yak;IxFWW{bt39Jd6bgMBDqE)a- z#d%i%B~_6`it#|}Rg9fzn@NCOuX2?j8cy#drW5_?wbFtDk8D>xb#nmciiF(2_Jeqlr%6;+*13?lmwE3Nh64=N-Bi2p1)}bEADHN)YOy(; zT-2Yr3JrkTA~AzZ+Cn7WhvQH}Lc)~*deZmxH9n;VPlS~YdYW9tv+lSY#vp0(TB8xMt% zvex0d-Bj%k+?Z2rdL$evQ}!JxW~cEDuAOGP4~KWLXhBNekK()z44=}UPHA?RquuH$ z`McLmm4bZATFBiV=L7oJTw%r6Z02%r_B$J0NWHl&c#gQ@ywfC~`icIO%-LJoG%ht| z%GLBTpQ&H^dRH0$0M+)^WLtbJ&{ByTubhQ@_+$I4@msa2%&y^O*Op0Il`1ME@jbn$ z%Zl%83jY9P8?4_ws3f#NX3!io-Xsn;cQlnPB{74FV2Av5LQmqXt2D;vNg$D25>2~0 zpy^N|S}Kw-Dpb%Zl_+DJON?dS&SzvjIJpTaUr8=&sbE+u()HiCMqS*xP2ccK z`1Brvqi>0m5z`#gIccbYMXksMEhS`^B*%EHmY@tIbfcU^0+{B#Yp1z(@*B5r#94%- zPhVe8T9f=1Ko<7iI>1Uw)Nm3LGHbN4&~)#I+G|kwlg{&kZLHCiI!6Ti9!(TlwD>05 zd&o`hwCjFGXUf!^DPP!iisc-7=TWa_ZrKLJ$m}{PnCXtuA$Y z)~vMF%HMFnF@*O&X|99fEl$%wuttNTsZOP%{{Rs7$?@LKRnVb52UD8KeMaXlm-|M( zD};M6*Y&nqEg^&!t*z6x&e}GHlhtbOqXXJ&j#b;YZmftZQlyo2sypIbxV5awQOQ!g z)v`)$8-)Evr7VtbT^58C7v!p%O}v+hQu}nMOR89vz)0vi(B3(5WVmXla@7dQ=_V;+ z{swgW4;Lv;K!A*KHPw74q+8ru+-rLCiM7z|vQ~5As2K?ZsXTq@H^RRQx$!0c0EXhc z3l;|Zm(_%!B1zBK(VrD-jcf0>XQiXS(6sdw+BFz;K*x-s)RGK(S6)7@T^zI9;}K89 zn)S!UI-0JQp90+jWv#Fh8bCN6{pr`j8aoLOYWl$m)2vcfM%~^b@wEVVptL<1dbR5o ztiUQDty`i8Q|u{^#9FgA1+6nn{{Z`KklL940Pi{aoIafgr5OFsoQ3IdiK|aN@kX|% zZpvFpGU|cyve4o^0HxZ-pfyX?z7P=12H0<+TL=1{PrVg>qoU}R_8@_IsKd7;N!r|) zPu?;;tC}|nB{E|f5fM!(w71&?VZC|_01~NJl{uQ}^p}6iz7N#(WThy$zKd&azEVH= z{MQk=cTORufjQd)tm3-=0F2yCg|3mJFr>wuzlh>N$Wr*9u%(Mze+H829POztKe0Yk zCjw6u(mb+06PFK_-9-tB$p{KFT=j}}rA0)j^X8|Tl1hFZ_hg&*&~60N)hBt z%A+Lv)=z2(Tf5v+icla5RGZC=7ZgHIpqf>2dZM9~K&{Tvk{&;*%CQ^IUey-ML3r*X zY7$ng*ERDoD^Vw}HC1W`+fH%CkPFIyPeDr=Huf24>sh=)x4yCf0LFh6y@k?lm{G(P z#DiZm)HRnEraF|L;(ItX9it?kB13O5d@Pp^GaD zK=R~KD8fc*YXif;k|~Qsk%|^Ms!8t^>5zL4YGCGTClS{?Nvvd%RqY`3Gz+N+3O(yr zRkM=-^`Mt1e`~7MbhFtq|fIpA2kN@!ppyz^*Er$UVnGDppx58ilO{ z?+`ntt>a*2N}yQ1Py{URo_ALA)4-O9Li&MWU7IDqK#JY8&lI&OaS9;u+)G|3)h5y&{24%~$#iRaRdWbSPB zP&MpCb ztKq1EB9)+`rOm+9Y~6~mYhr6u4Wp%Hu$d*2k+hm2mc;iJWch)bx(44`Gy-F2#b*#y zdYae?#Z^g&pafv7171smsHs#DS?!=XcJcuMGu9}CDkxOs5{N#PsxvSEfhH+NK^{G59>!=w1c|Q&6?#@R z1o)Dp92&Ohp}~Py3m|u_YBvg=D#36lMLH5Q*Vd@3uml5G6c$XA9CxTI9cvieIf}AXyVKNjLU@GzBNeM@_pD=Wp}{2Ldp;YpDM4%9O5mB`1h6^GPo5 zNKYg{j8ytZRAXZobtsuoKl-RAHA9~;1CD7E_hxXZ$68+X;Q|z+o~f;}stFHnHC>|$ zkva2prB@on1!g*rH6^*diArS8eknc6>spjTurcXH43!SxeF=%Lt9(6Xg4#?JHLnoFagu(v=$VtAN}W*Z44!5yNhXw)DMlv0EY;=2@W z9Ds}D1cGXqNHVI?Y5?H-P|heuMG7&06bZ;a_^j7(045R$={(SCaXE^}o%z9{xeCDb z8=_KRWahbxr<69CdLK2De3m+*BwSnjrxJ zU><3$uFO)$680Hnet@ksjj|Mz%xCtd2|^f55hNacXr(Gom&@x*-;iW1syQc`=+st- zTaZF88%a6Eo?;{i!am^6(AgdDFxaDkpqHpPhUdSC#SV2x<Ct`w0Pf4VYrBewI zdc}0NY@}twJ-ula(v!Jd0605hnlbquF*A(eTa1~irLdU<2{c&}RRsO2ft3}AgYV5c z(Tt0VQ6Ne{;Pj%{nB~T@`7afRmCq^{vCpji8hcz|{dN zASA&4lv7MdDo~Otk&za@O0?AX6UhQ;m9tHuXf7nW3ELZTeSqes!pZB^mh9Wz36(6E zN=I^OkkzcT6`Or2v})SH0U^~QT5CV0BP&$nu&cv+eGzVkw(Q&KC~osxi*}p?z_UmE z=lx@%dJ=f`t-gO!_>IQeVWNf*Wu_2MhS1_q^DFnImYO2}0Ew)8V|ZE)A2OX(sl^_L6I*I$P5Itc6?o{{ZlM;&hLO3I6~y zSNWc(gn}*7pf>Xxm5_lwGNag6JF4r~+T)((xdcpQMv>%iQRpfOb`fakU;^rZG0e>qYU{Ue4+yL_o=;_qX||r?=3uE( zU;)>qH72PFn@3Ex$Aqbj@+x%w1?%?;jG=^?Dk+Kfo(HL>8eM=(Chu2Qgw8Ooq;?(u z0BW5_Q&sTrSso3j7)Uu^s*h7i^p|g-te2!s)~Lpu!M4$e{{U6N1QI%g`g)q>AGixK zCzH?hrWa#qJ5L}9r3y@|9V%0sy%mHILn@Tu=lW3)Kp`S~Ru;@1*(MKROdGdT&Ia2? z^(z1lRp@;wVAy2n6`N;oE)dfRDsY$sX&`&oR^zOE8D_R^i*IXgpbg3r80#n3h!uC= zXW+{;p|FSbx4=*!N?A$$N7uD+fz{kNw{9F;j;TbgN><`YB$Ga**Gfwgx8L?T>E+j> zR{^Va=R>(@OTnI#vObY7scI`eGuy?VHHkxwIGl|mr*4kO@e1=}{1ui^{FeCsj)01naV+$f0qNv3={15|3Q~%(7^yYrry*l(5R)MN#W1kb-PA7HwY6-x z^O#O1IFt68s>m(?*pUE1{{YVQ-s$as^f$xjQ+**??x2+-$qM129(N%25gxTnc|6w%g|Wh&JlNz03g@clrX{wtJar8=~RRmAc}T zN{Wk$$up9odz1YuveB*4plRn|e%f-v0oSL`B`d_SkLd3s&E#f6d!&yqO_f)ha8y1Sc)E-qYC&pfXY z#j7Pmo@u6}rD>WI8gdi^_oP2`M??a4B=_XeZ8!KpNu~K>ty55YQ@eiY@=D$$kv~p& z9@MZhrKpbiK7x#GAH&|DWbi9y7&2fOsW#;7s!BZvd@-h`?^?2C;1z`^l*oY@HIKwM zgHrg0{q!UyvigwR86hY;ME*Z|d+?z7XK!x=B--1oq>hFOAH8yE8$eMdMletQcbaXe z{^Zu=TrBy+V{zLex_k6Z=8I{-Zqs!3mGk%&&RR3MX$nXsOGrM%{{Y&Z>sJbESk+dN zr3*`T*l(x`CRn;fdq{fIkEGtStdb!BAJ?p)H>b1sQ_? z_oCWw!_PrdcH3nYBo4VHa!1weK`p{mtibd%V3HRk5LH2_0Z78OYRjG`;IE`7lUS|t zry>#n88cKO(j71Y3bfOR9LiEak)nUIW-OiJo1#k zrABLXh5_26Y|;#;GE+Q)Bl_0tp-8qVoE1R_r4!*LN)QYL#e2f?fD<00nx4vNUd_wR zIHM%W_T^nD*WBK{#bEycZ@pDkhbICmTVX{&97P876s?vZw$egXm8)`qs)j<$f(NB; zw)Y$a!0IaPutAV^lis4;luC@-x7(ED_M#Bh;TWykY%iK2ESQk0P^Ji$2s{tPZV*Tj z81$l$*E0v+wM!tF+mqUWK@!|^ks~xhTZznoaTH2z=bTjdJ2N=zStE2{CmG|K&BrtY}n!0J2J4yGp)TVpMPIO0{;1BwaMC1mYU^~Bbf0ZBVX2Q`>p;R-pZvhmi;N)H4_ ztw3+2%@ec~oz+VpVGfS+(af!@|b!rl1 z0iD$Fj?Q_@L#;aA75mp^r8cC5ou?p8a=LIKLVMR|r3l;q08T5`@#2x@GJWoEAg{PP&(u*z16b>uXk4GyZUr7B^)C#I0UsQwYBCM4FCUIUEB`1T{oKI%$ zluRg%q(MFEs`8Z%+#dDh@m5!u0y*pHThP_A69z#e?^^60UpM4dDLE5cCJ((JO8^I` z?^jGntiXegYEm;4$XYc?#wv?Y+GC)_TntrIiI}aR?tzK^m0bS--?eI{NO3kr64E3!S^y3DKbSqY5?Y6>|Hg_QP9V;S}k~>9i+r_B_sYpMh){Zt5g0{TK zKcrK4v42jO2I4-UHKKTtQ0hREIOmFifIu{Zn@&i?3h~J2@+%=ft2GKxkqIybV4hGU z5OW47WfBz}(L0I7Wa#o5z$l-1^sSPjL=ccW%?edG>sz8YCnJMc#_1pl2R_t8Y7lcV zL18ETJ*#BBuw;x@HQ0254Ax2!k;P_*bIn|?MPLw<<^!RvPtrYTUE)q_KCM)FU z_&$>o5?)+#cVmTDJ9%(U{s<=?ZY!a`-ceZp%7_$)G`m2 zdQ+#YpkNLrmA=dZ0H;snBIN>bKYGMjEz)2EKD25XD^?RCo$2~!^`ypQpsE-HPS6XA zNl+ur(ARUK{5KF|kO=jov~3i5RG>hSCM&hmw8XZMdhwLvnk$hBgR6Km6eq(M2>(`;iy+E41q%_X#Kc%x`h`&U_~TS`)P0T`H}jxD~%%F@*YB6HWJ3iwe% zw!rr^=fMdC;81V0g9op*b^Sld%VGx*r5`C$s?9RI6$9-?xv?TtHr6Y=LIH?TKD4SS zZo=f+sw%5+5@f`B3JuU3n8j7=}f^IfxXZGj!iPXx(Ny!fFN`;T;1xF=vEd4 zWJLy&DH~$j>#3c)Nj+kcI+E%0Q->HcfuHqavG62-Qv$tSvOOc;69S8)WSpEn3T~>M$}U zx3ajux<4)Oy@|66I}O3gjQZyz(yaZSjqsa_ds%f_uCbHy?Wib&%&tk<96==dRbw0f z0CaQ7qZXLez9eihr&}EhMo8SDAu2rOYUG%XU{K2VgG2aPDz*4=glgJ?C&jyPLkK)U zSL!~32fZNpgW`=#;s%Qs4x-OZ3YP3`i4Dj4&nq+D2c;xKWDSY(r6PF16z?id*5A+g zG;y}JHNDktwOh46sa!dD`5#i?B_py@00|y~uWHGA%Mc@_1ll$Uk)FNirmo;^Y#+4< zw<-ma_7_Ns#(5^LAje}z+Dsw-y^z}5!wj)Uu(=463 zbHlbsQNbjfbKbiKnY8$A1>3~@&GAXv#_!z61NzXXSGLj{`7ei3mo8EQRdf1MxX4fQ zXRj2Pt_qxWelyEE)UJHpNj1`vimzlhA+!3W&C|)azF8zD5Rfp~^cAaGfKtea+8~&& zbx){~l7#-KHFoaEP71dAnho09GE1a7yt>_kgELuMfT1LEVw0)WDNcT(xI!m)%KO{{TulliP#MDEO0BZ&trZwUc3FOID!cc>PWv zQ`6EtX`Sop(KJzRrGNX^S4ev0d!V&%I81j}Zh~u=X`-muk+CrV2%41Z~qs4;~n#csUov8sa%m}M^kdp=?rkhBnuFPHvBmg-{o|G!mt*J^#=4NQc z_sJ2NsIs$F<;!ea0OPI^5Q0FUw&*sJ9Q+xfvvqLo_M)V>r)lX^*9$$s}~Hh+f0Cx)6IN(XM_j%nA$7s%2roA_%?R3gv_Lj|Jbq@N-A+eE$HD!$s0lc>9WdNovm31;^byB`|1xPgMT^ zJiQL3DrU{J_;#(5PlnXU{{Yl^G+w=Mn>%*?AG0YbZ9&V|0m|G0LOq1%*A%(54Ujp- z2KcThDRTRmW{qT<%|Wp4E!qXgdU+BFQ6Fkub8-G_cJRrxR}N&pEP&R_`krFu#~CQjR6%OtO09RQ-)LQsrmhYgbHB*hky zwL+GmkTalLHl=w&+>Zokl)$QgDvGL>M2?Q9lT!E7doy2t#UF zk;V`96teNb;M=y8l>Yz>tfX)^8KZw_zxxzrpx}^BIf{}<1wNCS&e_n~2vLd^_exZ# zBPod|gHxgJvo}^yg-3a%Ze0i=Ml*`tPj5*RnoV$`3Zi+ekT>W&kn#@M!Q^6+TsWfn z3(9bq9sAM#BaN~^jp`s|^sF|g%^flc%{QS;(ArX;G{`s=sX~+v?2-7YHrWCcNs~C@ zk7d2&CvM<-aY)89X^_RT)~O3{!1k)`1OR(QFnIu-pvq7-#7$5#2<`4E9ngJ(w%ABJ zI(Mx?Bn)v{h76s#nz(3#Q+7Hq7Rpv2Pgym80Dz>n0iI0|mf%6nT`kFhf$7qPA(PM= zEy2hYsv6rzA)`~^&q^FGqbESfK>|kJXoLixL+MlCV*qg(s)Q-9WEx&HI$;QJ7?t-m zqFaF@n#rTf;}s7Ky0Y=6(=0mib%k!C0k4@BDH&CX@b0^AdqNcu%Jg%McgJw zI3j6dr)tVoibhdiqdSKD#GJ?imn;-2JDzEQc1a4)tq%KhByA(!qa~uzbPn3%={T+T zjEqn!l#&KE^zT+nvxNgS7Fld4jmB{Unug&yfG3!&#^NME9;UZyNdUkxCTT`m4Ypq$ zskN%r%8n^pNlAz~>xyS;l_bOhJxwYz=yzyTAdZnw?9S!N`Q$}&)`DP6NSfiAHFR9ZAMHZSIBmcP&+S$V-b{W@C<}Xy(YR0EYkZOcob;jtW3Zfa zS~RlJ$KcSw&r$XSR>(2Q{MH9%jwLHfl+id4M9>9LNaTvq$4hFI}$T*tDFa*yrS_g>C z&LXi{BO6T6q3v!E~rdV4ULP!Oc7Ks^O{~3qtMlfP~>3N2w->tl25gGzkmQ4??D9-Fn@Tc z-eyXK{%aWdRoi(unoDreC0(nlxV9NFnoj++6=J?kpF@o1owtCYOYSW&e4t~cbnaVZ z;+NdpVN)Y9BBdmC#zz@?#?U~V8JaG~F$^Js)!FH~a2*6eGHQ`dM6yniI(2|R5Rf3w zX1WF#+K?iB>elRpA_3?)rY$g{0WdSr=DQ;Ui;-(ar0@nK6z<(SK=z=mn<+l#b4;65 zB%a+XrV*ktWnEMzI6I9cx?B^Prgtuq2nVe3NbZR=F_`dgoIfn+{vOzVtJ%5J#m`#awY1tQO+c!9LWlBCQOe@?!g=Azw(~ zP%3P=PQX28h*~mAgr1cs&p3*jP4+d?(XExC3k6#TO6e^&G?B=PM%%$L3K=pe)D;{i zYp);LbII};++8L}l4GHz?%XKkkSkvVkV=kdXPZoBsN~r(UDN7RnU6OVy-QN06GvR7 zAQA^!O?7mZfr3Hyn&p#gY`hhR1vZgO?=MmkLOXFw*LJrC6_08S;-Z|WXV#VE)g)PA z`73dBN{>?+pd5W_O0uMn^wpM}^ON_gcqL=c)9pqxv7Ays3v4KoKw7tF*0OP);3NY~ z{3Mi#7@@9`cu2&}SFtSTuFydtB1D;>RKw2`;h{?>1SCy0x;rEec{H)-UI{2|!k8EW zMIzAD3~W}ElXB};$?mR5_Nuo2k!lEehxG?PNs!tn)N*SyY0lo<6tLRfbpC)Of!?|c zPlOoL8BOoSSBSG@!9!!}RCEwGDEkQYs=+xwwF=~;+cU7#bsa=}u9t4jyGQCpvH}JS z32^3}e~{_ABM7?qy7ja==3BH!nQn6so>ei)flDs_Gw>Nzx5E~{;^s*KUWS&q3eHG?Cm5tfC?neK*2Kqsvkp!_#;t2&Wv-p2ProlrR=kO2CtsB=@& zJ`PH)wC%c%s39u_%r<}iVH|(bn(4;}Ewh?Zts{Lq;oDs{)ZctS`*!T4;3-*By8+6N zOk%Orz9Dl{N6Oit*=e#R6Z}U9Ws%eUHP2jXZ|XM&;{EkL@WKQHrLp{}PXK=PExe}= z22VMwb4yVn!M(A%9ZLuaYy|WbP_jug??a&MN@w#|3kU64NX;5aSCu#%f$LXosjPy; z?do~0{6wWEr>QjPU)Yt9kU)&rPw>W$%U15|n&De|eYgCu23DiM1CpXd54|qdDUSe1era6$u+OU`rCTL=|!4U-j!#IR@6w6ByS_XZom%asT8Y|BHM3@ zGS#)~jb`~*cWLm%LktB2d&dz2G{VbI3*7a>i)^&1 zk->`1%1*{fMQA$1Wu3aOZ&SArgp3ktrhrj+{l>4W3DflEi>?HmD1+rd9=umEb8zOV zc7V{9%8=VA20)bXSZqR174K^ob|b){0m7LN>*h*UxHt zgVz^pDsknND0C5kk=OwOwq@kBf>Z;?ce&m!!Y+X7&UE&I~bXN}-s5lKNzCg$`4QV*1UQC*&n@c2hS zR=aS3bvv~NbjDSkhxy3o$}(rE70i4YqW=J&>fiBBp^FP|;@rNaZAwy^Cq29Ug=O(C z#XtQ;s;cp%xYd(s3|KO>!9yUJC;dtwsQsz2Cj{pI0N@jsH^CW)6xmo)i*aUmDO{~Y zl1V+ullQ0kI{yI4G^Oh|2uiM8J_U*u$y`q0pXCE4xy=gI&0|(()g-{`8LB-wH|(>W*o?opzgRr!E{JZP_4PI0(Qq)b) z@fWm{p48V}x={E}MH2~C%E$+k2tV&!qMb7sRcQhq&hz0#o%hZl@oKP{>4QI!T(+UC zB;G7Y0T7HH$raFi6!23z$xwjRlGXbk`~GX@7wd61PA_QlsZ;spsJ}ude{t9%_r*_@ zSd7UOBCeENf<{DAojK(|KmhxQqRM4*1QXRYG3?O03pMWyE*7C8YM}(FQ$;`xcyk2x zrLfSoX(iPVdO-B5mi))puX;E(Qh0&hr@)l(Bzo4rVM{=)rc;!H2YNY%!(ga@Ka>uY z4ckc|3{e3c1V_Cn$rNlE=C?~zp82mOv$V!&hLCZ@(FGZX)BdluTrmIwXVQvDBm<09 z($E-{8Wd!}#ko8lGgb^h?oTvuYZDw+o*oGhW735MG=O{~(yWvLGBZYaQC0Bc{{V{B zL9-M$Aiz1|p-cmq{pgem(Jg|L2}*saQIv(dHlHt+JJzWwJP4v)w<1JA_odDf(d)hjd%kkGp+Lc6F`p?NZT&~3aF(xCk$8t_uGX9qQ#u%Vh=jDjmqA2=qK zp-Kbk1JqWjS@)#~VSCtnR%&$Tk?mO(`E$>`VdCVIB72HqE{)3Oa! zoyq2h18T_#T@9OHW;bR&)V|fQ2M}qi3)=#^anZ{mow@`cC=t?(Z7Bl+BZ#0{ zbY@Kwmw_=c?Ol+fOk&)K;)_FMq|jE1-5DguT5f>|C0GzDu~tS*UvP*{J5pQJZIDb5 zB-cZDu!NE8HOgvNa7<$~CmmQ|_9?nlk^$*7upp?&Fh?e|xVj}u2~o-5R&6LPT&Unr zt#Gw$x^!$27#?b?jVJ|14II;Q07xME*PUd+Dp1EX%NRMF{lVKjpS@s(7@X4wSV>y8 zf>qL})v5M0R2`d^jcpKer4Bd-nzW@PVKeJOFu+2RRAxA%Qm{A}u7s!b(HC9eOvW6N{syH#z??pH8;})qGI_<(4XeV(%mF(nTmzWH?fR8>EAuLqI;VpmB=T_sW$CA}bJrp_v$*P)4zO!6jQp z?u6A;*hnL2k|T;aO~51spX)~HItAvm;zKC#nLqu_y!wIqjc@oG(n3ZpS90=Ow{32^ zur{RwzZKqTUkh7l)@M-orsbY)Ju_+j5#JWj&S?N{eo`bwy$;%wihBcpIfEl1Z3V5EtlCV19Q;-d z7y&qUK|wV#@{?~CGAJ0FPFyH}mo7nWvn$(Q5KK9l5S^=QU^lbRLsEjh9_lFAlC{zq z;KSDFVem2`lITnzDV?jmFM|9;G_j`3T0#Z;je_EhC;s{!_gFd2wdvk{m#0W!BXks^ z1-Ef&XoMJZ|MN9u2l^h@v_PY2ThF9cyp?}ueeBTBxRStU+7thHm|s=Da!x$U@idm4 zNka=}nGK2BXS2NzjN=vlQEz!AHir$mp<9;_0ijhNt2%9jj~^EYIpOUep|K=lCyybR zPGipO6gqQQ8;`zQkq5+tcOFd$*7QotE-e4E@^dQ78iNE)SK(<7dK+{La6K%aaY9IH zby-~)cpSJ5Vk~5>zP!c<&r+)^ir%GGlWRXrPW8_Sgq^k;^IYzRv zT@4r+dUWNY2Gxyg;#t0o&%LV2xqE}$+hiai0rV}K^HOz>EZOTc$4kp26TB5ri&Vx>w{?X3Q z&jgOswu5~y1+$85mk}^_VpaUJO0g_59XtXPpaYrEi(vRw|C~=0_W>|JA@hpxq25 z(PVE|PyTw|#ZZn2xIpmgtQ<+Ro4O~yy3~Fx@V&K+9Vc(%AG6!sNB;wSq*|Ne*)IXu zlI|Q;9KYHMjIF5G?psEB!1a|X8-Z1y zZENAkJ1)}#izR36?!UbDKfvtGvE2Rc2e-&CoEYrxFJz>Jtw*6B#UGhuvpj%evcQ>TP>U_v#*S~ddZ zkD)xn{5JyDw>GlB^wv`PEwi|5dJOJVkiSs^CtL_t$fmDc|_`*mQDIKZQ=I8 z@wMJm?zO=V;h7)@xv1lEE5Q!XZ%4u$G8qB=Ct#TVdyZD<$+l;HrV<>+-q~=+L$*y! z>&#mz9=U}!>uw7J@1?eoQeJ{rhn1zj9Y5K6Sw{DqjP1r@=Guh?$|!G@#N;yIr;a@FlBr&O>ycdk zSzVJN^4$)Qe~3SF3iRS3@psmTB4%>zjq!FNPrEr*{ZI9%w~mD$DD{^sh7(sC3?9;8 zeX{h?!N$7_Rl6wW2I+Z7D}iuFgb90N;ed}%KAV;+e`U3r zEI-j-%`p@BD%2l$DPQupgPRIjU{5G6cEJFZLKvii{lLGf9b~imXt=W%%ni!RQ38RMT)J-F>%G1g(tjm(^pb zpzmq7u#<^N+e7EOZ&2_=!zn8F)B&>-t8Nqb#*S8T8)ZlNa31FstP zGru-oA*h_`FMnz{lnhX}-A)iKU_wkGaubuW9I(*hM(b-uAcWk<59y`n z{3dGI^E59V;#=DwQp-p_OS(XW3i@m20_P_Um9Z`j=M{ca<0By{sI|YlijBULhHq~i zrN7(qSNiXH2u(;!fld<8%QJ4k4^9_^`VfWjmkJY;?Waye&cFT&WOMP{^F#r8q<+s@ zMPXQdSv9J;Q2>h_96?f6c^Qv3gYjvbZKHQvp4dgN8ihwqW`)Wc^+cU zly#0s1;jD?57`WzuB(taldl5#+K5P4ae3=jxQKyhqoxQYt3Q;?>%alRiXPYMfdTtz z1LYKrn*x%^`5ALZ^|E0;4F&p{xI|2XK=~6a`|J7drS5hn=}CHZ$}_-JfSF(0DUwhw z_@iyFRT+HN4JEEh1MY5Qq7%OSKLFA!0FIsoV6qeBTNvZO$=5e^yWqIY4|waU|GrDj z9XXoE44cs}Y;D+RUcA@if=rSrfjE_LAKtxoNRhw57MgRE_Iu{lljfkof_)3b!FCX5 zlVPI$22;Bw=-sfH`Qph%k7^;*xr|Xbn_{SPMa3(tcgC&aWyI!DKrC2>pC+r%r|ch= zOw1s#2J=i8eF@neA?QE<7Dut;hY`RrD|9bv6zN z%rFUdU1x-%A6RR)wSh3N&A>AbG!!s*gmI}OIEBL?{-f@Ej*j1sfI4Wz-n35wrL|`c z85*Dry!7#z{}1^~q`OQZcH#P5>8NaaOubJA|KNenITO3Bsky9~9lKwhM~uO>$O+yQ z=}6Wz%zG2&df|`(kbu)DX@gyA@RwpV?rB+;FS@@kno^Yb& z#WPNhfZ?cv#TuNcrEPTv+rAN0Y77M2nyWf)J$3(;L*fj5}ymn%5X#`-njTblTTLjZ_QiZ1?YVjvb&syX9kv#ZXQR$6#}vh}*b zz3qIX4=}-2Uc^Q~+*7P2OyIIJO*>JJHAE-XD?N%*S>L)e2H7APR@;=b_gC>yPo#n3$}H*xoe z@kkk2Q;8$fSn!vdqX=OsP96Dk(|?%NI$e)uLxnC;dARPR8!Hka{DODg}{iq{e#EL*C&4gY}3YP&%${6Kw0e;T~Jr_`P?+xiWx4!G-?fW!_&HYu? zybHU&8Be;#T&A-JwR7lwru(z)(A1lnk=oOXZdEqzjxU<4_yp3ozHzr6BK+IaZ%4lF z4`}L+&kDL9a~P_WV5R)6%&0b()-Kmm!P)5^J|e|A0#f|EYXg@pGOC*hSbM@rl_A`* zuS)#u2Yj@WwHjTMqvisWjTZQ8Sty0*POq|2Y_LW@lK!W z@2;A*+jK|6Jf1x#XnNY8sgDL}nHd(}^c~CH$M`J2(`Tktwg?7JLqY}U@T(mDxK2pr zr&*g>j|FXo^DZlpZ)H9{DI2Ii|Bfa%8P;_@ui_Q0<^J@bNHPdi_&1p#DIQVO34Fv- z^@sO*55qsH2>tz^gze}zQBJE9^xf4hXQ{OmsJsM;pdMpb9B+F$w&JeLwabghE~^uE zweby3qsDjCowGV9W0Cpqe+|hk$BOjOy2Z!m+qPwN*FdiWFV3&SC4!lT z!@l3dkK$~wXU~c!z1&)Yw^75l|6cA>`O;2(jrkXUZ`)cc_(AIz!re&~oBOE$0s44x z`n)q874qiR?fUa`uMa#&bZbz`*mp=8mzD{%5#9nw;O=vQg+Mn~oIYYEFJH&YH^vY@ zTObzSsfGT~D+8K+1D+JUU=7>5%E-)um#U@hmY2crfIoW;9?!R~z4sz=#YSt`UdYuw z8NrK#k3s^d#O+r19BiJ&5w(~H$E6|4oQQhP53^o&B`W?X+?>OM^yDrFoJ87D9oKUyRqTa<7v?W+n_UwaeogQ`a3KeGHHwKsjks0T^I#GpF4W=yu z=KOt;mgMZzF>L;ZwS9!6o=RP}3?gHwN`Z&yp8Zrm7q<%ZD}@qJyQ}qQaCf^hYSW-6 zMoJpxUYqVGC#5TXejtt+3iSY(s7=gDm-`ZUr3!yWzL}`S8k0>pSa();=$E4@4{$9m zV?q0Ei!%D>uCGXo7$D5#y2Ef( z;Q6S~7#Jah`aNx!VCzrU$j6AK*YId$G~3)EsDQg<{lTV!FQ;-v=t9~51(rgS4V@+g zZ~*zMWcnHJj@JO$SRRI{zG9Wg_4nIG8&raj7R60Ww5v;5i@oI(M{(e5u>oCr=SS|P zs#j~Czi*Gq@_8v&@Es(&@A(`W-ui6T5hhSgOvF%CEs&}GpT9GVJ=uts5%EkpACs0-}eQo^mbdVpb{x+E$JuCjU%AOQxUnM{UT9E;=h&^C6y12uxq--LcM zICR9^+PFaxzFV-YtyZ1qGZa;Y0OMz1*{5-l$uofoW092dd10|!K^vK;bxSf zyw%u={OvSLK^GW=lniB=e4UVz(*gyG!mfG(P(H6_&O|aP-*&q>e8z;~&eWP(-~{IJ zDHGU8r494hB$aH8U3NmMRMU^rxya8h0Axz(Yl}dapJ0_;Yw2KH<4{u9oB$@{tKuo6 zwC=NC;a5}ylU+Iy3ba#94w|)!^r!hi3g?vPXvKyZ*DsR-t`x%58Qo4TM=2v~tD$75 z86OB(P$O@KpL#hjLE`rH>vrRMxqBosxx4T&93jX`vR01#=C z3jr7vkcv^3w{Y;x6JcvNQ&V0J?W}GdC=+$Wc$ET8xUY?gbyC&n#kIe8$#mAaSI(s| zG}}||4#XL3lWuSenuajNAqC7p8x=?Hf{g9|aAU9E161pKn^wwsj-^}kwkMRB$o#c8 zC9qFNO-!D=Ha?8vQb*`fTaUb4Cq2|boKpsQE1OjL{@a?|E^1r3DN8NuP3@0V)N0M1 z;Fa372;1#kdAm$lbh$)~>7|g8J?GG0iVi{@Hwq^Kj!?;JLhJ@TrH0*+D@fu=%dt=# zwwBZNR41SWj5;oTqdNFnPr<96ZMl=cU6XO6%HP@fHLP0g0eO-bk!=tq-CEP7Z!+iD zgldmDmTwnw56cHZHYV5O+|2LYz7|utXmi~K^XtCeQ6+yM<@O7P+S4=3UA?3DM0t>r zjuHz@t@`A7U-x{dVcdMp2Vh7p59$-?i^{Ijzj~&7)f*Fh$M8kt8Ag)P;ZH8re=7M>NMDP9;Xub37PUO=EGLPEhwBw|kB z^kWj8;FFe%WNTBL@@uwE`)?pG375&Q6Ue>>pxJU)#i|XjrQ8rJCTSqkfyz)badwU` zqj`=Z4()&=PDzx`{v-{Xdbbcg_60weCtFW3Kx45P2aYnUE_nkJ1I|p)9yrLs>kbLJXiXl7EFizxi8|5pQ`!ot^Ci0)gTALtB&6Qw4vTNbPd8zd%r$k&Klmt?ne z9g{wRKgQ&=IZH1RcLHvQiTmK)2t1nWw7R%ZK%#Ehauf|lPpTk(rw^=_TM~@CYPa7i z4QA)8S5DP++zgGp>9bS>JI0Pe^8u3#tAN9Ed#g>sq6Xdh^$_ZMIh=AN?Jr89+)MTDpQ9YAm|Oa#(=s z!vENUD)Fw&mEDWy%`st@58HqeGIIWh;?Xs@OGPcGqF5d$UpdI>C9CXg>gGO1>gvl^ z0lEpceWzNVB&D+=V)h|Xnn({*=XG|T0FcGz<&~)ct)b#+yop1}wwBQzQZcwNC-CYZ znFBJw4Zq^eRZXomS*d+XDk>fJnZPTz`=y1*vMFYQM|0qI=ZmV9ceOrSi*$=*MIGIo zy5%(g>+XL^r_X;GOHXWfEQt*-hpD`=4?z6fAC*E5{14E^{$L9vJW#ukUS*wiPcpYQ zM2{tzNqoaadN3$Z_!r9GruF0QnPVr%qonE&+K=czY&zc#gbY_nXTWj4{Vf*HObo{6 zpudTy@}p&W2Z*2T0FpD#{R<8{OLRMlXYpj)!$c^w(NRm@u3)1{Y4OIP`MK%@-P&$~ z2@aXTC+E)u6rVPo@<5XN>;{g60^s~CG+zW)RHgSBqO6wkY+!N0 zx0$Ut$bmo|68m10KA6C8g7nWZh?5&tcSbebj92*!Sf}}C)jBN+{L*cTWx2FhCNTxj z*IcKS!XD4d0x14ZsQq?aPdxOrItM7EM<$TKZO=N`;ni@@N^c`2PIlgrr>z@R!%l2$ z+ONOA^-0HP{_HYzG-eE65h-;T5%GFG1=iB$>Lb}?${1w7HgI$@td-NQC6uBvkB%F7 zu*UAJ%j@324;ao}$a_pbl+ zW2PqNc2#WLz3Y0klTxLaI5Wd0^G$+ob`Z?EPs%v+wUA?4qQ`5>^s->#8`bj0>*_q3 zyl-w-X>&t4R`nXZhqLL2T1F3>=Mwjkv{Hib&zcZ=x)wZ04aK^w^f8}z^?tn}O;Ff$ zmvl*>Bpo9xi~u}Xz6r1h3%blJ?>;@|v}UsLAe==ad#uULrLMnFlSgz=9yL;7l2{o? zWng!vAQy1zwVJ6<5@N)kC)ZjFeeiqu`%_985;w_P%J{bA6{y zbKR-zO|3q3DUdvL4)HFV06e{*X$Ec?k6dlXZHd889hg8xX*=?N?!0rh>I=BKaVDl* zTy<|)6XL)8QoU4l440V~XD%!;%`BpucX_>c*tY^3*>QY`z?sPyirz(!A|w9 zh}Rf2tHNag3dZw|!HJ3Resy;cbZaB&_ABGoxPn~ag3=YhnqU5$U+UI}@0Cv9CG4Bb z%oG9Aq>h`1^4jGl+^^v553SFyDgAt4Jl~0r-6m^(taRYtPUlzR5I`I=%$pLhr~-&V z&QKnL0(wLOv%loDe+jvx{pe9TY{;v6d4^wWra7u=;-gy(VL_{Q(I##`x*e!I0AY-_Dw$Cr)i;6i3h`T1p)~%zV@EUw7(nSRjCcLm3~%T+q#!Z-V|n%K{PcS2=*ru9b! zOG-6Ul5XM#5n7P;_fySfL|0DTUFPrNz%gHGuZo_fnoy5Z!$|vRgbGKvK`i*gEkKLh zu9Rv`Hwx5GbYCcKP29%x%ksyjl)o+vet&?MAo+Piig##S@_Gj`mEToyAI5~_XiB$I zi1NC6#%)UTMFwY7BWpEAVvXOm!r190SC6y~xMtA8vawiez4&USawe=Y!XzyRPrnbn zpE#LKFctF2whdfb7=ATe!X_V3LSJ+IbQFEQ;H-Y#?v((8-jP6~p}!VOt(`w;GRAz_ zGp>liA|(LfGFg#hwS-k@^MCYoH(J~5K~cHPP%|_3x7XD+Y0+J~s|cy5}6l%)iUsxTK>VjoolE zEKQ1`pJ~%)k+i^T;@{KCi~GHDWl2IyLi$M-TFw|%u#B=4mo?yNb2$ly*zn&LGWC-! zX0{FzN`V6+l*hI+1ya6_zA$>OhAU)9q}t5q(^Z%N2+GD@*J^_U7^OJAN{99*UU_z8 zv&hj^?v>M-tGE0ap*9KmiB#;`b5oXoq2MSvcee{|zS)|coWa3hmv2oR}E3@)jKX|K=C}pnu=zy$c_C;l#%g zb8jlrE*x~o6|-#kbFSoX#X>PiCt~+hoxjOILADIrtG5-@MX&$S+LDcQ~VEJ^63#lM$pUu_>=; zM?`A^K23iX`|DjAP;FXkxnUokH+R!L4;^oIOJD2x`vv@i(wHm0kDu>3UGA#g_+8Aj zMP)_*asOzm-1)`skHVokY3*J_fW3D_1oyVVf-8gfR$E`J4=Y-EJG`mKU3v>I9lT%gBH{9P(R#ZSB^YBq^ftM!J4f-* zF{Jw5){}P^7vCkeDN98&%RepYoIR~0+;wd=;S_YAE8u0uUHQ&mxt;JOlEqcN%W&Ct zEjjG9@xW79n9TEaS$&JY+s&tg%{S#*k83&vxX^v9mpo(UJAf^5rsY}xld`zv7?llM z5$k77Y2{GiNf82u_^~z-d1L($FVi;!JQk@jpR07Thq&Lc2cE$?!+OyN+RC(Ib-)GrtGd zC2*$i<+%fZCnG2HquZx+_EqJeUXS7pE?&(C(Chw-zTfAl1GBEZeum>kpXZ=$AxI%@ zj`8aD8-HgDhx-cm`*=qmvXQSN(xd|F>4ck*b;~Dira7H9|Lr@X2{}y)FEnNv8rgy1 zIcO4S*REh6O+c4j-p!cSSQoHpSD^JC<<8EuA;xcf@@f+w2OG&PsI-`oIAA5KL7dd= zKCeWge4AjBSM0*>;aYwf!IQaB&-3xLx8{KYEIY6~g2fy$3t>aOJk7U76`oLSk^K!j z5X@O^aiD^R=HEt%oPH5zm^|!%c(>RhbiU3>f@3qBD{#?SSldyUIOPyJy5yyIOqFbj zPz|FF>;%bhYUWbL^D(7^vEmZu6&@?;gqYI*&^KTXHREZ5yAF1_2g(wynn1j4IOOyo zxH5bXQ|a7>Sm4+0MhFBIJWk6(RDI-Lt!pi2W#l|w0Pp1*@svFy$27<5J46@1QY=px zL?_xU&xRm6m0#P^T(7#y+W)ukAe_rrx7a@m12=vmOyp{Dd|k#YYyc2|XSPdQHC zC9jeB$a*q_*vA@%yx@r;PFBL2;ggzb$#o;6+fh@6Z>JouuBO}!6*W@Y^TMFy48fRy|CR&o=`z8&FiFo-n3)x0osVB44$Inr6-eWCyBIaH%_VY6# z)aGY6*+j?5^k@}djOXc2eYsr zkERB{00MJY=@m=Orv0{0GxD~)e5N0@*b!M?ca>im@)3M`>Bg;EmEat}7v-t*<1jX7 zsY*GbYo4XLs*j94QF?B`e7qbrp0OpVu~e0js-g(cHb2raJcP421A45~7&Yl7l(88|_C#pdyveT_Uy;AI( zS$q`#=jtNx6DRr}{5`OWn-cY~OrfhscxslU%G5F@zvYT-#qGbftdBnaBMRU6vX&6^ zw2wRcLAxcp(&;zfB~j9qy(#jJ87WG~`(`?z!QPtc`OG&$6K(mKm}!~P+-x(~Tv?zP zhILx^pjg*R+}u1?M{0I$Tv|+VAn3d|H_W74 z@6eN)nKd#~icf%uh<$+kTq9fu0YA`jb?G*iAfqn&*Xjc6##{2rTp+l?WQjV4ZHYE) zndgapa|$Ft)Z>7R;is#)wjCgm87B8~2Q$SA?ca#sVmV$VHDi<$l$_ztMVvOZfNkAB zbMM8hs7{)hHqze%(InbfXFBcU?9B@|yyMseyIgYc``65$RGlB{-H17TCF6g9cm03& zisz2^Xw*mWa;9YHa$oq44fmo$r_#Sl59?P)Wd;;Z+LNzsPl$r*U!lVPNW8TPy*~;b zb4`ek2TNWKgDU=e{O~4T=*t0qVJ{;b>1$<>(^^cb-6Vye*1*27=V;}Tyc#XlKC@*S zW4UT2eg-PCZPd+w-Q>>(^#C@>@W9kXJDA(3j(S>(YK$<9+V_()JhHJJ)6G(I{Zh zD~Pj8QOH3*o9xTpERW=C>!TGs?+$#uNZ4%V8TwuqGgERQ55yNm^}L5^YEJg+^>EFqw7PRH|V@9oiQMmEkU*U$INkEc*vLIT(lY; z^5g>AG1X|y?%a;lfX59u)XC$HU=ew4Tzs;wDOXpu?r=1R83|pm;80 zUehi&PqpcX&B&ig*LFYq^)k0PnkS5}>et`ervH%gRI#sG6;`uwh6pWYrsQeY{{VF| z=eq3vIO~#4_50avf4^yC*#Fsd>VD1ju<3y~eO3rGXUi@6@$Z6Md zuJ;0wtIEgSSM<@O()*c>#SsY2`!t#Mf3lBF5=T~x0&Td&TfHJkB1*C2{7a9e7h!3R zFBj$sj=xH=Go^_O$v+FkGHY4H4$d*I$lY8Mrux@P4@7!#U>dX}G_Wjg?(URP;J5&; z2_;^dnu*wQW(>fR=OP^y$4gZfJ|^7`=PJi&0H|S87`;2oSl!CS* z@h9xh&XRc6CIGBo6bK7*mp5wa$mK7_IUqS8+gjTynm6_$2wV&~tv9FnJIMH&U~9o= zcG7<4?GecI2gt#b;4d7JS6x03Az$Jj!CX~9shHec8*Ppa?W$gWaukhD72X@nLdZ>quv?aIVrcxD|-XMp7xG#ERY)ZqN3vl3d)5gV%USuYIJL=YTWnq zlJ)9UoIS*$xGcYY|BOXQu@&>;`21&5io^FNtP%tzrw$7x52eXfX*_z#E-d^Iw$WCa zgp&eeM0~lMyvOSV_Zk8XYl&|_EZv$MIcBin4pEz4=$ z>IiL70g?sErfxOB3bQ>il;O7i53+r|>9Sn<6cR=1%&pSa!L$*Y>aFt3Qr_FUn6ZpJ z-l-g^t#w%J+&g~3aBvng9imk@W}`Hc+Zf};kGNAh4;@w4cX*1b&H5z1UivYD8||S}t z{+q)Hj+qqqhH!SaeSBmw85xr(nfRtJkS%()Q9z--XxL(*%-B)bX;n_a$REqhv+vA9 zo{N3Fyd|Fa^0fbd@GMS=Q(>e`2bzTKW42?Fs9KN;^`tYl6Mr5HyJwjE_AWPGf*5W|s6f!J566vG`}whfK=int1B zlEdCNkMbhTMaNJ+v778Vw#gM;^RJI8JFG* z@utG=1;jpSjB+@kpBC%R?U@a6Y?b`w=&&YhnWp7-sjP6q8+%20iB4JMjnt+{W^TmS zuJ_mHqWkYI?BeJgCQl5t0MoXY+6uiM{tP23Sl^*|poXhA-y&aaZ_N&3zP(JkADg71gymfE%l8Rf|1db{6mPGzK2 zRoDA%6wGH|Y}(Jm+Zl}I+>P{*FRCz;8%G$`WzSl@t~H(0KlCswzf|cCEhwnnEW8Ku zeRS|T^)WUPgll0kNJ;6LGwp*pI90n|Ny+>39%##7dev_+o#nn}&|g@ar82+)J~MLI z_1zbQ%JHG(OkJxK^}KfzY$4<>Pdzi{)unOWoy+%6O5fDFLhqjjNBs}*SNHG4`S@f_ z41b^$LKZ=kCiB;gP@~S@`=s|b;Kh62$4gT4A1+a;+y7V6D}+WWtE;T*R8*|?~^PloDmcyqDc0Jmu=4%dIete-JE*- z9!8uCrPX10lWndA6Fj~A?$EN~I4@l-vj9j_WNi`{r$}E2d41U1|4A2)Tb9d17L(I$ zW}&kRA>2!(etCs}e3y@!0Y=?pDAoRQ$7m7_S*#1+A^Mg?S-w?Hw1V_9<<8RYcxhopUx^i-K@7b}C75S^X|&)7OKs%D70D+P4NhPeC> zQ21rJ=dZ@LbMKTBJ*DC%NW{5V=gFzgDEZTQV4MaZ+|)hrsgI%&x3|sGE>HPPyU@3! z8`p(bQylvN^73v~RgmY~XMPIET;A0!XFSD?0;;%Slvk{RvA@-M#_u$;40!oF9qbti zx3W}wD*zQK90_Dv=jEnr>|<7Icrq!7mB@^DDoYLUw0&232_&0}U@5niudV#; z!%OwdM1ve*k5v*a;4)9(FN*Dh<56o`bJs6wOdNpFFnsr57&&QnP9^>|_d!tmkG`Wt z{7$!~NXPrdpHD_E4_y^D>aAgoe2&pxWcO!$?z!!Gf>f&ND;c@n;&;Z!*Q9k}_h%<~ zo-|iX-8=SsdCWo7mX5Bb@wIM~IwXX2gowta3zXr-h`&=gWk4CZ#&h4bD@i2Uq@X>I z&88H9%KywgjDai~`_QFV8xaVc>5@pzYidJC&0-8qTBTlITEjigDGZtL-+Ql=IQz*vJ zG;eV9Sv3|lBIjD=Rg6?}OcdL$2MAf>z}YvrXHr2AgODZb`P#_39|`#!y;H%XOTrS! z#}MaM z0Sv2lG1)uUtC-1V8_VfWYT)SH&ggJ;nh7u5e!>}V<}TiIfd({LF{)O!%b!M4iAF(+ zQhAtRlQ1=*Vx?Ob3+E-yWEeIl*u!oB9o;&=>hO;41n&LQx{H1Y$$ewRZxa!y88w0m z+sp?8l6SB2fwl%d&ydT%=z?#I>y=By^!5zx}wevC*^GPc?caQ(B+R(%ZQOTLF}G;*&;fw}tzZ zBMQp?HJT<;%onGxDQ0AEa?#kZ$ZQ3=pq{Le0z<-~;^qCoN5Qqz%EFv-6Iaze=RA{Y z$gxgf0|iCHsXjIwk0q1q!TqFI$3F<_sA1%l&K=o1#;sY4yZ*ygoM13SHbK;K*kd?S z@|IDZ)O&QY;NPHP+Z#wnqVz*;pJt!effHLJaGWSQ>_t zY-{L%$Lb9{f^h#u+A%*RUAEe&e-vev=B*R>jH^9#lcCJrUdLR8%Q`Z`@^y|h! zKnUs7h3YR6`VLCWVZJuDg(f#@o(=+n7<4Ej=m->bjvArN=rmKb;Ys;FbFce#OEQNd4IxUO{9Kyr(-6AV!mxu_OH| ziS5ZIfAha_b}aASaRRoJ@nYPfu{{4STOwD23UTsW;CvFSs5Eq29}5`O#l$X*UY#|O zu}d5drQpGOQBBZ!V_Qu#?T#B<^_7SQRv9|x5q4qvXmh45K~Id~o|48D#EDLXVK)fv z+_xkg^k*#cRhZkwJ?i0b@bXaR9PN}c2-|h0Ofo~87LuTfSXI?Fjm`KFUfxT8dr?ST zp_N}$Pa)yLNrU{b@OZ|ZjF(x}bwn+sd=}G~q*iYE*+%WH%9bhhqq%7ca(_CzfA#Sy5%_`G$TfE&rcb%k~ zbAcZ(P{3dr=t!*8Q#ivr!bJLb@~E)D_c57m@vSF}lV>0|V<$wdln0T~Mx*eI=SuC? zsg(gNOopR2Uz$d?DC~bhXY##J=%va?_SZ!3swERxeA)-o>E~wXyMmv34rme;#z`di zPA!EXiwinJUP_Adnt`Esaiwb!^gJ7WNb&X-%~y>GOUy9d4RmLYgw7SO11v>FUdc6c zo}xwYjvv#{3iP6+z65b!UvJR{15Qehj&Cf z$-adn`=jAB7>9%K633@*8gvrwKm>~)JxTz9G!0&=+C>QES)Rb3MX!x{m(u^(Y>i-J z7u3hyKMN>QW<@~;IXBTR4?&;eW9<#jI7S^>U()H^<&pl-@!9nentpJPtL-ML?b(lv zW03Q!(#m%#f2~Sb`Fgt_@T}XMzBcaq9`W-cz3XG73K)?Er`$A{+jYYGB?kk(xLLm& z5+(~=aXDFO=&U!YOKQm)+Y@wnN-g_hF1WSw;KW*@Bku)TUd8wpGA7c z^;*(>J9kzXJ?X{U9e5dY2URYUu?o$ADE)4I+vHD|$N$OB?t>%@Z3WSvxR1x3+Zfm& z{q2f-#6LQ#Sbw&xv~P+hn7=!dDUYLy&TgFcGj;b()g0LCyJ~Xo*W=3~6IDMjG>#Og zf<<^n%kOa7H?#QJn`^RYA_rM=R@En^(vL?{S~-NU+GW&ZkHe)T0`os>E7d8z%+^nZXZT657`Ta05>Yx+RQW6OUs$%8)F z@29?*=&3NK2di(VFxU)}6{=#ztFQZ+qWjf_ho+UU6)D^n2d~~w&)r3~!>9zyOp4Dt zaYBSg%FkGevdTM6ME~xDGA<)JaYDm4!f-LshPFe*pSDU4zD;*pQ-OtPe5X%6KL#_0 zf0A@KASd!b0M~l4!b6P6>3(5+Dm$BGxJ&w1&#wsh3&3N-9cgl%ZLpoPdc_d0lXb0x zR+28gI84D&w+g=`Sawn_v={6SE2<+@3yXffg4Y*13@qmL?z1WW@H$|!k+%j=11NcZ z3;|1Vfsrv)`ws7W6rPm!*#+4=gIB!%Z9-i*amO?nN7-wsYr>Qy| zni_`o*F29sAK3sUOVGm+sX~SPXm~lqa_#FC;{tT5Dj41o7lyO^P*YdW13WIJz#w@V zJ(P<>#o?~#X-b7`UxqR_qzY@Zh8axNyvGhKJfJ-ZBvfw_KPtOTc_m=icaQnJvO4ZZ zT(`K%0FWT`Y;un)>=xI(x#TJDN2y|KP8SvLNak2J=-yT|%vN=pBg^uE;}#nKnS{)* z)01==D+1JN@v-*?5L-*G-dUz5v0%??YWK{QN>pYnrPPH;)OMA~)>^Ip$Q0tSq~6@} zNuorU7_>NDFgLf{wo!sS2mVa^%ueq~>?k(CU8)YeepQ{xskrnYW<>}%rSM6{95eSc zp|wr}7vZG;=+aq(aM|W?pG=E?AKBiBXhAbtE*$L2s@R}z2}gV&dko`X`CIcwIQ}-b zi9dtoi^nFswJbQmaH^LMf`+myECg#*T@u@UvykDS)~I0 zbN!>gr_ac!wTX$1puL0VN^}6$r$QC!6rVAK5He#_evBQ*aww)`w>75;|5sB+-qyhI zubf#2;bGaNqJ{Qa;P{B3?HL4Dfb}|eSh9A)6m1X}0Tc2$DOS%mXo?y80O(q_Ek5oXSi(K-KHd5#G!;Dv;i#HSg|Ho(MGP&-OyJm(V<(j*;HEe5`nYopO#9VSqE-BYyNo*9${eGL=N$G=J zi`*|EAE`t_7j)65?{B~VU}xv-?7Uyk=kxIZtLdIGuK8BgI$4VW(|R%7)Tur6*e~Ce$#TP$33NI1 z0w@x1B!zo_a4)C_2FrJe@CEMvo;@f(Lor=GL3C9!9-qSx=bNwuM5;e(8%h`t=wHY=mjZ~+WnjwA6Gij+ozLJqUga^CQI`QmXTv>nk z%+)c!C29?QmjLUVdu^Pm)OT;1Ey7CG)Sk0}Cv8h1%0F@~+;EUEUDC;;sB#6R)^o-O znhyhQB8o(#EImf?_8#)9!tH_o#)h6$p%K5?JTQ39Ho{lTdQplRieuGBh^tZ5@ z=v?j?cO-TDOxFf@nFxwh)Vfq&UCsBa{v5VnK?H3k#FI?>*CnMwUD<+S?vc~?+LRKr z9&S0rn-<2QgAz0p#~_WK>_Z<;t-MUG)Z}FYuYS)4qmBrm<4VBMMfM3GploiD3T6>D zr)nwLqiC&NgALJIc@h;&a<{LDJk94;xd*@Psv)y_eOa1G)L`O5``$|2n zj%R^?PJdB}ta^F1{$xPb_$~1dx!Zi`B1c=1CpS_a&ci>yZ?m-;yG(g9qeLX_6r7MG z2ib$k2Vmw&s}2dv9M}ucjRt=cA~n&S_+4u=-*N7~!4^`UphYFvd1**5Z>R6oNY$0s`XuD)3W|TEzDVs-Iiyegda->l zdQgNSJ|$Yd-h=@Mg1>87_O-V888@O}6OEL2_6!^boWlbpu$dd5DcMgKECf(3!kC<( zA(_hi^FlWz#re{zRJ3m|n;miHh_49zmKQc2g^)5CUOVSAq-1xcF^ek`N*I z{UNrWvYlB{&}2E--UZMkN;yw&+rAvrVVzEIOj_WdD>#AacAa6|{U6}Lb;YvR^=OmW zkYGhDR0G4q-2Titbk=FD?xW>rTy&bb&iH26_H9(`u8aGr9>Oa6?x$nj;amgQ<3x3q zm7|v3Ip>nlH=x&dlg5}TH&a_H)KwjLjYippKn=t&_jq%g38(_h=5Wp|SZ(#!qyFAH zIJ7rV2>=M^zUvk}Je(^fqSm>=j{#WOjet|2XM*59r+Hl(Md^4}JhBy(J>)^2XNFy` zvS4^ma-Tx`^Ej&FAoj>UbD$U^=Gd=H0mC-$Sikt5?lYO5Suh2B+QgB~)qvv5Q`O5I zxn^*#^buu*8c4SOO(->r^Tk9yHBmD|jn-^zu-%A)GrscL$N?`vCt^8$^C12)IS8DM zjnFDTdszgYGJ5M{`9N@%2*sh6+t{9}MjkWrJ1@KOrKJU+A7M~ekAv=zT$ zrBu@D7K-HNzBYOeevv4b^H3BHql(&T{==dfYG~RkTrH8_XX!6?T+ikUFG3Y{XJhH4IPKYN6ZR*Aoa-!XIAg2KcyF zYCt?C*Noi?9UzTR$q?^~OJ?$SWxs_FFxYKzgIXwZ;FjUlw@6`{=WouuZ`J4K`40|- zrp|d$ppO>+1*rDt|Mj~dMaUgC7dg8y;^(s|vMj~>zZQ(Fg9yKytDc=%eYK-yBkV?E zd1<+Mqee~+%g=5DtB(mrGrfFmkBd_2_$FT`6FD$K_&h!b)z(l}U}%y<#@JHD%+npb)TLHd7~*JkToO^;9OG z*9M7N;4)vz95P!j2d0FeXWLRWWY(HqH-6D4%pSRTbXtLL@u9nVh_Sk&!14Bs)#+H2 zz&CXmbuNDBw>x#sQ>tey4vN3x1&QuyIBD&9HbfWsET!BXC$+91vUlCD<(^urMF%md zcfH{)pBasi#cUCaCJO0$gvwKk!}G>sQa^Yyz?!NWeCNF?jwEGhy`Z4(HYqaB;8`{> zZXT#5q&(m8RB#8M`aKdki{DUrA9V>oXI(P2YI7WK$~uL?G~ zl?Do~FggLG*#@tTO6sBE(O*j7>7voYnVP-XdGWR`NlFgt5hAbezo268jqy`VwK3*= zI{_iD0m&?jyFqX7miG+!-Cu=d4Hqv3CMrgLiZ3L?IC;AJB=zB?ZBzpagF#jl0D@J&hzXa~g0RgAq)G}(181De+4vd_SR^$i#g-)wKv#1h60)6*9`gUVzB{M9o}Y9)|(;ZleG zIURSjfQl@aI%lVKY8EJP#SE*FwTl-xI=PM0=B^SzlUsC?5_)ZqH1}=+;GljTJ<$7p zK(z9GVQV(*FM+7Ik$iW$@Y!r4uV?s1V>#aQ2D;qWVSylNV=_+aBkEWS!NL{@_Tcdf z(Q?TO>p1^(FKQ%9FP5^-r$(Nv!!BYJYg0;o7s~;FDxofGQKR$|jZ}%*PQJ9#bHS+o zabPo*C|TAjWk41N5lm|8Fp1crZpJj$8drBDu&oQ$@0C4l0V8#)K(xM zwIRv6ec9)$!Q(sHFC|aMe2RF8%hZ*v@xI&iQOo9RmHVwFrP;QrgDSNWc2m#qk$4i0QKB<2B%d{{iN5 zJstzTR4YK4A5Ro4h6$}lb2ZaSA0DyoLw%OC*tj2T1kQy*vKMSdP#gE08Xdc9 zw#Z>=F|op@05pH+kQ*^PEI&oVW-%=y=ta3#&NJ)=0j@#pz9(ER6dn$N6lRxh&KAuG zY9wuQi9B0%y`)Hs{s5NPQevg-K4?uY-0Pn05=E#QeCTcV@l|I!hcvG65{C>yTce|g zvb-977|dtpD4X@nV&n43_P5Fy{Cg|V_TTVhyRP$uh*m=Z4Lw4Fhx^E)f2{ojL6rDx z`+&B3?)FNKvzM99<8z~-3fC$m%0eCV_+*gzUGb3XnE;N!&XZeOf`+)nR}+9F{%+x4 zdO!knOz97#JR|m?c&0y{D8%ynt+5iHjrz*0b$mQFtyrCMic}o2Uh_Y|ade1{$9}a& zRug)ex9_>yl^qYE5%#L1*UF${o(p_79bCN$Gm#)w?~I!(fMO(ohG%Rl@RUkp#y{778U5a>H{c%3iB-;mLI{vH ziK{nB9ldrcL~p10U@7DXOa9HQW5WLcT|$eQuC3e4LbSHd8ZBK25mhug`RQ2ao$!AW z0|S}MCp!T+eD8pF)VD&n5?>@FchlC;ELmqN=%K31_qE3_Z+pK;3FV>NTP+UtJ6$Q@ zz^@=42>6a<4ur=MtjkXRA3zrwR|o4fw-Q!No<^LH)H?xR$ID&xTj>X4*!@%M<|70Y znaHce91+G04$!M_YOGvD!!X!tqtr3$=f@-R|zcS^VdJ0M-4j z8z1?L7XaY&jJ&`@7mGv8#wQ3-OaPX)W8v{z~k^&H9967!_!}WN3+Tw z@Om*%QC(k2yb*DN)tl<$L6GYP$q5$z_v_!>xwfPE%0H#f2dj1&_7q1{PxQ1hY2hQ1 zv4w4>3d+I?l4NnsCH$&f4g&Yh4q@v6U~Ba5m!H2IR2B^J^SXIQ(Qm+Rf+nQG72ciS zmqdMaO%Pm4=rFAOoJr1dbxV{qbDFuC>OaBN$K#1a%eE3ecL^C_ZHi=~#NizG*X?W{ zNA8`^fYJY#(!*g7*rt0omlLfN8A_n~yd+=VnyH<9?PZ3*9 ztFvr7IJ;ns2ySa33DcH*Dye1-6{Mm)0~u+(D-L6SarjXa{cf zqlT>5IXlJNwL4^#QeVmw3`LDkQLXJz1Qa?}3-}TM467$>n9+=@_n7d2&twsH=FG*g z-S|xn!8>i;q^Zm5Rj!BBmqn&TD6ai&5Ya4Ez9t&(&>A5TJq4PoqEbuw(kJW9){1W$ zUW`nH(L!RTj6O%z3KDd;R@(ikxl$^5n;zgiWxS1)a}Y?=ab~7r@I*54H^zJ|w=v#Y z>*%5;Ds>&bF0bq$WthVbZV#NsB4eioFCxKNuBVpEuhzj}V4%gAxf3+F59R<7T5NW( zHOKPu41}A@fd`@Y?zx41*PTjn1SchJ;?hsr?T+DQm)Gx}RYt7c3j0@I5Q&cO z*8t&?$DDgJ3E%Fy-4h)Vs&ZR;p561e71}tV$Lr^ zgNnUFIjjjNP=K*49^fRYf)w-S7fcb~!beEPa7-|&>JyX(lMlKx6rGX-JABdx`lc9x zE@nU_?_^pGh?J)pujh)6$d?Cy7oTmpNbzm)OssZR;T}yPsLK@lxVGkgms4#qE{T@W zH+OnX#HaGv|QgG+n$8Q4k7{|K?0ZR*tqEe zvN?l|>vU1LRg+an=_OxX1aH`aGR_u zg-6Xga`}EEkv@^TWeA-aUukln6ih4DzO}{}7itOqJzMVJuV{gez%fgiS3qoxa0(eM zT3DEEtVjN8U4`g**X{GkC~YIwK~^>gMU<&$c0d?)s*PUF65d%D16O~+$fPsbg}ia< zTKn1LFE(o>69faO7++wH_haruc_H#Fi>bIo?C9jbBUw&G+9#ux9Sz(wJlB(`-zj;77I$BEm#01V zbjvqU>QQO)uXaK3sqi!M#Pr_^B{w`OlOOBl&yCeWu`8wk9n5~ia5M5@j?YxbeIp0w z2bNbfU@ajIspB#1SP?9G33{3_VO)9lrt2U?K?e=N{pR8mAOR7(Ioy?WAy7z;IM#Aw}V^ZJ7p8D$bkXd+j$uT5uLCH zt49hkN)_Gn)E8?v)0SJ5vRP+atSDMb7*t&ntaAIWC%<5H272i>1xyMTsknB8#7z6H zv{n`2s(fi_OCOc?Ig}9WG-4i6B^?|uzJ^4qWj!q|tsbgSiF86p&pAF>u=Sj#HWJ@# zEvC7jb6;1t8Q)B5J8|S!w%QA`#fDAne`l<`+Q~2$W=9AXN=h(G zFWjeQT&r82Bytu?29e<%bBWQqW?}w3P(3wYd`~Kv_?l@U=55ThHetH3>wfIR1yO&wGbH6}cYfXO8Oo87*>^QRsOJI$xkobUcgB&ebMt>Me$}4( zA)wEI2EX3JjDNsSbyL6h3-F{4%=$(91J%2Lb8l*X5TWWkgDs<%97IGV5~}@C)iu{M z&=pudjZbO};%BZ%5m{B2oj?mJhUcMvI~ul%RNyfHksfv5{TRoEwV1t7<@XJTSw>lTN*_8%+j5teK_(+5vs)Xv8$RFhW|i=3mWSV z$E7gh)HM1{fU{D{_9#)v{POqnMgtkwn_KUipwvHnX7ZL@M7O4-K`8AOH!G(-v+YFs zaN>*;e?ze`nRe*@|q7AH`J`c6kqT)6rspWLAl+E4 z1@j8exiya%1lQKY3MY+`4Fbyq5deK0yW1KyS4-bqwDQVh#r!CH%_?4UiiRfr;uZ>+ z@QS2lfcojm4vK)qZzq*=R@_Okx))wT)dp5wC4Jz#J+`2n+A2j-?0 zq`d%du+M!@2Gr!LWKB%pIZ_tKsJl#~eZnr3jU?do+Xhm2cq$*QYcDpa_LAs_v1}1i zTH2P2akLP~^EW+m-AVBigEmwr5%ea5ud}M%FQ@b91awHBxExV|Ns-!wcb#bF4p3s5 z%QXr3hsp0k5`Fk_m8s;Bwc(i2#A0MCCucxSo1oMoI=@iHWx7rs@#0<=onFHc>2s2sqb85TRyrRbJ#THLR&3Dl2NZeoQ;=p>o=WSSzu#qm4a2Z?2lORI z?#Lp%UGpr840LpFechv2$#e&=7?Jl>Dx$?COKG95+Mnr5h8IEQkW+bU89^3Yw&k?)p>p@8{H?k7iP>^|UrP-qb~V zpZTT)seR;XDsiH*o8Onf9(VVhRYZCcx7% zQa#tCWOm8f3lPF^ae;uU!>@XNz%r*d2o7~0p{QMMBW3J_!bFAZF0zL|nJ~+O5{m&p z`W=`mzGONena7&y*9Jp#`{x)KjBH+ACQ&^ESzKP&jLv<5;KfNK*CNInVn(Zed4&ry z1ynmzK^*h4M$EpL{r9Vrz8cYT53OcpZ7aG)D(`oA!l+~L?U%+63}+g1UWQcM(|^zP zBr1dTVrF1-;B^cjBreuLs#RvpH#sPYE^;WxXU$F2BW7fiyi>w)h!I`Ro5xgZ&q+QT z;+SxoupYM517deA_A4&xhPR7gICMm)`+Qq4C5O++(Z}f_IoB{n6w?E|L&$KHT1oMf z8i6?m^uQ`lAca*FYngglQxYnMtQm;-JC0a-`b>Py0QhZkE|7UdNEI(Z>UDJ(omzOW zdisjTzPJqNyT>vTN^={{PqL`VB$LYR7L%c!tCgCEHWbELUo{^`;S2rp;bW3Zs|vNq z1pfT2Z3i`iVBCfa=-LDPh#;%8t)5lF^m>|^+G!01kGt$Y@jQ0QKR#MJ5eln+ zcK>I|%CmSC^+hJ@ET(d>t~+VC_WFTIL!2`ygr%eDer?WQ$yzVIdQ$W6~CWPDfu))X{L!OO`X5`+&x>RHgiRy3@3JCl^1e@f zPB1Eh?NVi*^qAtESGwp|qLa+GSYCnTS}CVSqY3Pu4UMa_8~>p)@yyp)3^da}d>bw$ zyGe_K+;tHf_?DQ{F{;8n5%YF{T%U=LH-SgVi)blQK z**a0<1_Po%)A2T!C4eek&A2%0A=I)5u_K9wq9s#@)Z2_Y^$}oL_aI6JuU{bOEJ*b0KCw3S?4PW@M zvz&A*QmM59NkHY(MX0HizS~{_I$!B(Dou5#^h!PJ9*;R7<6zoQJ(UUi4bT^4hg;8< zrnxw>M5T6kNGCu(5Kb@iS2YhmTZARj_=XI|ekACylMW}`iRv}=a6fwcm&g(0RQU8C zxrsK@Th(ih?PH1My;}?7&-}{&N^IuZk>_!rpcQ{m(|A>Jh6J!!XeNPW|M=d30lfQ; ziDp^BiP|?sYqFuy4xfl>56n8h59@_nOIGFg$4jWy9umKM%I;OfDwzT$S-x$4tYVrq z$SFy7djm9c@@HuJ?9~KnNl+bq)(IORN_xl}7_&kP> zZM&5C{AOwdjsB>Ac%@vbFfmokGg;HvoFg$v3Hh4^ZFQ8EbjSAo&E^tQ&MwB9E!DR(`%xp>Xqs%Od2ZEQ-|)w1ux_X7=w5D)qnE>BFOnW zMQK+B5Gkr{70G$|$|{F!mp{kE86%;tXT>z|KVJDmcFiv-+jiqr1>x%9d%H~-HrLzkzJJA+3PfnJIb$-eK@@#(^Qk>Fb@@X5H09nqhS^mzv z`o(BfA%)5SxEVO@WA&!(OS_)T2~E$eVa^jtcp89>ZGM}~9-LI#J0l*Q&4={)Z_`nq zCi9?O2D<4XAyAjyPfi#Rh-l7E6vz28($596Q)SjFUt3ANPypda>B=F5ZQO}>enn`d zTMXB8`D`nZ`I0Agu--kU_K9FSdy?IQQcxmx(68Jw50P7G$6H8$_)Hm1V@`J1@DiyD zA?tWFc^pppWjA&DJ~Kk3sct-_Rh+1l3OG6s9>V{P*9&W6FTf)W+SFSUs$PYbf> ztQ<87ecZ`aSh`aFSC&5fp?Jf}ws?D~2+T;rt~ISbz#>}VL?+5VYozGLfmNFx2PPD(!=E~=~5v|tls7Xp2jFuX3t?U4g2a3?&NcfJ`#ei3>z>$-gjyfh8R!X;0 zN$E0woII@G5A!C4k!f9Iy95)xGLteFtd!K~SK8HkC+}mGvJC5FbbI$Jai?VNj{<$M!7!|#)}s1rNi-ZSn`m=TyPfx~pUx|F>|f4>bEfb$>K0apM=p|f)}?}+_-=1t(4`g+P3by()x}ppiQW4(Bxk3D$m0}^%XvxJ`$102J!1@Ko;c1 z+wd6KB+jNme{`wIm`bZP$aA>i0m&w}ut2NyNCeKlW#hDjj^J2XTbz(xAP6P;E7n=+ z%Mb7#b2kuCqH<3==4Y4jdp%4V=Ub@+*ZJNojrxvlIEAmBg|o6XU*squn%Gdl>8LEYhzwbou^s0S%Vu8Ck?@T?W+(*`RaLN;5{zW?IROj_>a`bP}A_&6do zi>;A!)n87{fVNXN-P)H^WcCi{aIYy%%d`}5?|2(SCOIjIDyeQ<3P}xQiJvws3DWVh zVk{BIr&Q{lQ+xL7y-~ut`tCntZwYv1Lwr$_;H>;4tlSG#0mSfuA=&}pRsI;o5>-N9 zQ~Z{xBOr;SZ!DuSZ3xxGtYOPG+Kcj$A=yU~c3N_GPZ8K2_L&9$bQ6qg?2*L%0Z)0= zkV5M?E&7~vQW4e#vm@%1@Chv&sEqlh?_z8um$-J@tu{xWN7PJz@Ik1!{#(=-6p$hi zndK1#j9sgg#5damMr|2)yrCVItzmLoy{TMCcG=AIV2_p~-Y2hCime!3DK4CaS&qIu z>0qb9VgtRZRn=5DDGd-G&ZD&XHuUg%cD3MxXCDm$l0mgO=d{$GT92Q%9#xD2XJNIc z8VGe=rI%smw)R<0M_thQBD2qCS@O$%Td^^UDJWO)0E6#814sIT@qe`+aupoWoKZOt z8fwc;pIV<*m_BkbgV97Ac!1qNigLzwswx<%ON-4{W37c7ifoLVKllIv`B%&5jffbt z_o>5WkMfhcZp@DU`(|5YU-#U%2y=MhjW2aMJ?4C8mmg;qi$-@TOh`-L(ag;5*}|BF z(W)20?_l|Ns&)U#$j-|_A5P-%^-R3*`-2jNOV^JXtyHg8Cq9JSw_XKdwmrZC9<($?dt~n8U`#=) zSq(*!m_T9V`Sj!mVs|l5VVD0uAvT7hYz0Ef%Y%e8*HV7VUh%4OTMps-QIb7H@#bI5 zHGNMQf<1ft#h9X|!d$ktot3PpXi_l{dwv^{r6|Ps_CON_kcj!lE2f|h=Y)-Ys^1-B z3=BT^W22^#CB`W6wVXGPB#9>LXeR-S_-CK_pJdE5eULRH61K;Ve!^0J_=pPtS9#Zb z`N{iF_(O+3P#LM3&IfySFiCqVW4j3f&ylmZHAa1GhWW_026|x!#IxQG6Y>;hb3)|h zDOi-lzV$}aB8xj?HV*=3gLaKk2lXB|P!Rj-`iJYhOG>jqi~*SyWjR|ZDX4tTiiEWj zw6OJCioK?N#P z>&_v9^HZg8GQ*&IZqKzr@ZllY8L=0S6XtIF(khhH^U4xlnNdU*HGc`TOs}vzSuh%n zW?y0CMg&O=*W1?UoJzDYsU*(;Z~4f6mfa~#)z*zdtdkHf23#J}0w zo60In-qwWx&m_EA6g_^C)LxYbtEf;?j!J0V}1NElS%1VA#`2lai^@2f~>)ZGwFddS-(Y0fIgNTPeo-oxcQIx}W@u|mO_9SWE`f{Mpvn$-9S$A^-Wg&iNVStV&b)uawlB`qQ1 zS&x6Gntj)fh96X7x(v?!e0C1C_EfS)cC9`$N3qZTx`R9*1v(^CALNv>RB@K@${wlZ z%Y8vsnEgEEnPLOA`n;4i_=IOy@@L#8^K@pOIUu1U2Q8uD26 zj2sSB4aQ?);4Ad*>7XW3ut&bz`X>0?YBvYx7x0$wF(v{=^sP&-ZcZPZ#56*|4y@P& z*I-9Y5$w=Sw~?=&pR2#^;BLF}G#PoIh=RVm(UxYH`cVRW>m}_h-RY(hnmne#u={Bo zuKEQ;2S4eBMX};OK}6ZihiOh~>GT)&{DqY*XRJh$lcmU6B0@!Er}I7v5Hhv8GGVx< zO@pPoK-o2Ft>tsBSJLTqm+qcIpQ-MOBdRJqigrkKQWuA*iJ6%KF6AfGEoar6CWS|s z+R2IdiH%@=O~j4DY%kz#yEaq3i}mHB-RpZWZ!m8oEA>F|vef2c?o!<~kd!@c;cB%8 z)$^5B#S}C%ZmXcfp7ANdhqno@a069L{_BS&zE;DhJ74i#1ryEr&fQ2aC{fQvcD-}} z^OxJkMEy2Fo$~Jm)qfbK30Z1$aV+~~vbT3wroITN8r zGOuhw!)T&Iz?LO-=8UItqE|~eQ#Xizs*!3jUQ;hcog&iJn!&$o@_eeL?hW|X#gztt z01`T!vb!T$Lor@=yrRgQ`467LP^Fx76-N>C_fEFWnBSQF{OP{l48Wn2BSUn!o=@{N zP7)FOd!l^!x97a|HVcQZjQ@4~ix|IhxR)IkW)pvE9r2?S9!w<@4ju(1C=7Tq-*kKo z9P{<<@>)DRQ$2 zlyw-YG<5>1WVrw|>{i5Fa-r_kg6|X67i(0{ga8xL`w|XkU!Ce7wKj!Dvx}J#{8&L3Ay&HaS*0OUI_*qg)>F~o9|rpIKwl!f z+976&=Id40(tT8trBBW7?Gc*G8EC8wX*hv}9qD6ycs+uwr3tS;3wmOis9S7U22aB~ z+!-D4?l5TAU3bi%K1&fIom&j^=eumQr(Bpp zUZ0)$B%ncRNi!V$v|A1b{STlWHoP(L6LQ|vCfH65?Mpv@Asfq)-Is>1|C#IxAzpyg zjO?EM$LdXagy~sOCe27;uVJva6kiJg$-rN2_G-`~GXC@bZj=_(`J1Xoc+c4!I2O3} zGSH_u7m;<02Zae1O<4A>;QQ;D7tb?X+!=>v#75TrS7my~@G%vWoMqVwWG^2_SF2Lh z&>e+Z%80w&eyRqvs6XG&O*>CUQ85|EW!Ei06ZT4FmTP%k%f8AM8jf4{E{=1ptGgmk zMnH9o78BhtdA*zN%jbNCw+^YXPnNg+GNNnU6xBaZ&O;Cabad3QvWVYPySaKcvvU&g z+6yZ6k%igEHLj;0ZBz;6pO8Cuc$juJoLB64!bS;*jI~`cvHT~-T>?^vrLF6ZiF{eo zG4c#0XD2#rZl8G=0?uPkX=wp~GAU`YTm@w?1VR7AS1!YUy6t3a%!{rhdz!cE>8CmB z@C20TH!aHRw-&k{3qH_k4yD*jd@Y7ES$a8CFD|Oa5r4?fsq?<*;5=;*qvIa^yV*?c z0nh#@9>ar@zD+bSq>W@vS8;<5;Td}$?i+CJR>d;7O3>I?me3D_0u7P_5P&pUb@*iDY zMvpOJW=dL!9A4=*mF1YVFp0^`_ipn_#;PM@nA+K?X4y2AL^Q?Gd?^fL5p7ZZ|L_dx z&qoHl$kwDxa^cw0R7bHz0yh`5tw79}Y3bGU*hnjA88{sWUM!BstR*uRYW6Bl*AVS@ z$tf{Mex^LSKW%U)LiB$2W+5AJGv?!>=Pz99de=WpR?a6v5dVwk!k9yrzu9qty0mS3 zZQY9#Dm;qv^+{!O2hA&9cs|9Aa`)t_q%qQBCZK1V`X|SJm9PEj`$DZt?N~x$$yQHk zuBQT?^SxN$gZk#jwqw0)X*4?3LHARDwR?6?qCEp;AVTKnm2N*?S!bP5bIQtkl0a$i zLG7FWrVn57qotb970ObeNj8(|?3@@YlHhZV_qF;1!S@MyqqJKc#Y}LqG=nZW{DzwF zHb1&s^K&~DslM{`llTsxaIhf4=Cyb>R|Gl^lQXX?)Uv8i)bCj03?3ytaj=zZYsz`5 z0m`w{wb4)p%K|b*e^Xzo_6XqR%KbA`cGef!>+dxV{IvFVmy=5 zrLo$PwOrMn)LUQvt{A`PLpO^Qi4jur7O*}Z{U9EL)q@SA;O93-Iv1U ziFP(_C%Gc&GM!YXHWCH8a_8U2B6q7y@~IW*ipy8vPwdcDS!cbS8<%2PFL?-k-p%sP z)$(3~wp zOba!x2$&ewxQ#_Y)mv9%=NeEW%R&nid?eia!k7f>-Lt79YB7^IQ=u>bC6C#E98zp2 ztzk2pV*s^%L2$Nat-vw*%@e-3rg6=skq+A>lIKcpdEh4Lc8 zuX(gA!4}d%K`p$#SHDxdtc`H31X-=ZCL1;J>Y*j?_YKbVJFV`**;9(-n%Fa}WQHGWmhG22n*#Z^=} zMo+WoWML~XSH%`K=VQ~byh>?&Kw{4wwj)v^Hf*>;OE*o)Le~+zRPFk-N_56 zdlg2@Vi;-wxKN8nl*w`;^&4QJ>67WgAAyZ$P!4FjWzJLY&wQyD3^~2D0on<^U8k=m zoX9Esi>Nb6W5G*t0$$Q6Ucamm{BA1wl;EBVAe%St`^=xqc{2deC!KhsnQ8W&)-|cw z1FD0a3aVA z@x`m@{6D!hE{bZY9l>y*4${x6xtSgjXYp|`2l2e;T45xi&d7LcC@rTYsQD!DZ|sy2 zPgdE;m#fC3@Km9Y7zL%hn)fm$Di#8ysmZBK1YgatX8w$EDP|?4aQ4!v`_uZLRn(fl zyy)Zv2w9(b6{>5cN1zCzzEb265J5YDBBKHe!t%_oi1Qmd$AjdXe6#Hub;(*v{b>YuF0U~DYBk_X{_n3=F7m}r2J-Tbokec< zgnI$R(uoE2P?oLd{{UGiFlZX#!v851sBHOk4o$1odPy!s@t&_sS8nF$ou0AcN8M9Q zj!ruiDFr=N@>J7x%dSg3%~`SwJn^d}PbhQ<6CgMG7v*BaU1}FTx?O(qZ&}eUX?u#O zo`qjA4?f`$^@2G^wzn&4rQiSD`)+#I?Sk4_>kT3yk5=S1GP6x|`1pHLS&{zH!GTG|KFX*La4yxy`T;*Th9$X#bp!Kef^uSO98jQG>S z8E=0+pne{QQt8x+*|ZSjedYHfzngPjYqC``-qcBl%Xexr858#pPX-sP_J^7~jQEvf zfh|pe&=v@0bBhd1F4szz8PFCOm24y>LqXPBcz1|3f{^5vus0M{=Hdsx+&aP7yM(ZR zfU3r?*WMxIYgu$T0&`J2vOHNin5IBQ-yOZMx-FGy@8EhL69yPESXOUSdE+;ra|n(*^KiYaBc zqW@|QC*p?{$)V(wOB3Dsxk==_7>e*GTdfNyCUQ1Jgl50s08^g-qpv^2(7uyhkI9dr z&i4Qve;A&n%N*6QsFakCv#QFkxqJsEf8%5YvRafD=6A@(P)atG#N0UN72uKpLY^}XF!6Z7wuvw z|295Y-~;VA#XcY4k!cWnT*NhCPXAGnl8&hs6Qy$;<%NW;={E1yLZP*s4WILD8H2e= zO=Et9?a7Olj=K7?l}A5b{5G;V=GS6y%2VK&k>bX>+m2d%J4F+upp-o~YxZQoNeemX z7JHF2E(2fM7y*;T+2ljB?R*ueY0Ladf{oU z(fx#F`PCyiBE0G4kwpQe*O~SVr{J1HnaZFZkdreG81j3vToQ6>im;3uk3aSiu5H!WQ0`#n6cXzG6zN zh}ofn+5Oh-i&+t}h^2j5NlGei6ko^6_)Vlsu!+4PU z#|Ez^M;~>jd8@qwdG^M+{rou{uwUV9o?ria#|T{wODbW%?g%cr{s8|GTC|l>0@0s_P^7B|D>q6dcG`q)}t6_oW~*^wm;*t}^eb zGVQKL6E+u#`H)LwA;ja*$SKL|{S=4N2LA^gSs4X>jbF?|u|KB~AzDVkFFaX7#kHk3 zs~Ha;i6KoaTMY**C0Vpm35@BK^A+K|C z9dd9aSIBGZYgkLPIhnlqx7)S-BOeL%qvk?p zP%!FX(@t}e_zGJGQ8C%31tEzeb;dxOhXIB zbxTR zQEz2?U4w4@D$OHcj8QyLO=~KMlcJ&QpyPMDRMA+5fWYd{B}5DKLJcSs`UW^Jui7z| zcXyeA2sNTQ)$RX0OnO!EVe5L2meVr;fM8)OPUG87 zE6I<7L-ox?u1?6Tzxi+Ild+=zT<$;OE1P|&WJw8?Y4}Tb^UI@Hd0Wd4*DJ)UaU#h@ z+tw#?8+w+_L~L>;^>tm^iJ%=BLv{x6F##-;y5*+l9cJ@%qVel5LmX*NC5h>^NVo!$ z8UP39eaz#WzMiT(y2gZ034*CUnYxQj*A=|EvVd2*u1A~rpH7mgJVLMYIwBL9+jR-V zahbwNrwQiOpgQt}t4Na;uDR!e#;0u-O~EvvR}Kr8GY6WhruPY>1i(Kk%6Kf0NvzO` zSib)CLd9s$uei<}G7BW9p|$AgHAJHWcAIv(q^k4c21b`ZQA zVPvxC^HGlzP-7L&s8*PcnuJBQ)p2rpp_O#trkf8IkyjGQFn7uV<#Od6sQ7zr4s$Q8 zSJh1RYn8D_bi({1V0Tw^*lnj{dQS|24mnSlx@W%Glic{c8jFQ=#~jWqypOA;)aNlf z3A6)2t%$yY(x!EVUR}=rRdnuwOz)5XA7)IhyOev28Jp_ko_llO)-c1|x-1rRJ#G=| zv<<~VHkViuV+@-pbEy;#g~(lQIUPw<>Qv~QF6aB(@4x-AzutemFVENW`FNBRG(K~} z47=mB^n2fr5(|<|(jenKnCHGb;)_ITgz%HACyjwsmrfaOW5Nv_C~evw@msvoh79OC zQ=~A9ZB(Bd!6Rr+)#LM+9-U&o^!UOgRhq3r$LZM{PO)UOey)2wBGZ2vp75>S=8ub6 zPpy9V#$|tFo2L$OX1DS{2z_b5@ZRdPtQz#>lS&=`ZTR$33tj7um&VZ1=Tf6g(a!42 zF|dnW$nIHH`Q#vFUlvP;MbPqLAC#kUhM78Gp~8^2^%mk^T1P( z+GZao`^>;A-r-CA;+e5P4OFN4lHrm_8LqL{j8vNIV3}%}9*a>kpyz@0%SxnoWXiPw z$)ZZ}{^Q7|H6V%B**MB^&T1B){#LTI_LQZMF*a+UVi_|bv_Km0jNmA2o^KOn^~ zLXQgq>@JK>N6;uIh_9LguP9_|S)P`Sn(i<`d?9|jgQzetRdiK%8lqq6uY#T?Rsz_E zI<`TvfMy-t0;QKne!WvykkHMaL{CG;PR6+O!+Pfd6aK{cWa-mz?H-m$en+4OnxG1&A#RM!PmX;0d6hS`IBr1j`Izi{~oO(7Qku`I#! zuiy*ZjHiUYqcY|N0p(BnF5EG4wd3)tfB+Tgf0*ERrjg2*XcQ&v8&6+BfJtMjE~)O| zkxO+zRr$k5gjG@2lP9e#m#!qtqAx;u|85zs7Bm6r^N->OQ$L;Bs~RP}fRAr=j^Kew z)-!Ewr81{%BXAox@9BAMl3EF$dq;U&pH9BKul(Y8qGdpZVU2VqRvnw#5IgMdr|4ocv(5*@9?Y4RDivMRrAfqA_@P0^m3HPplwtL2i#Ky z=|$9U{G~di;iE%PC)4Vli$=j?nj?)t0v}xc(e{l`C#Qd#dLds2?Cw%?acYJM@d(1z z)20g`dDoF#Y4hXH2r=Mcol`7~bxf6X8H%kvT@7mYK|69>?CXq*kaV;VmsohF}J zOX=2pHIM08o{tn(h=cz<5bdf|g_di`5RHHTs5+}B{8k_Tsr#fD`$H-HTpTuEcdu@A zj;L6xfkb_t%oc<+)_LXICucSUz3^#SZnrs=Y?v zF3S^D^pdS@fBwj(GDK!Ml6Np+kn$u%5| ztyK^k>OiahMGxna3B%kBy=YEr0pw5DaMNSy@iNBI@2B$}{4I>mNSVu86OYBQ))y*Z=iV^CKRZ zWy%|s< z9+K7w4Yk3Vp`2odZy06e>(9ziTcSphxiYh)cb!H^W#3SGg8-rG6W$!U!H9GH)idN4 zZ*uI(D%UqOxdm?y{p4mBCznXBsCX-efl#T%;fiSA%)Yl)=l)P4)H>?V1uVVZIv8{4 zp3Rt=&!gGBH`BMY=fL@)--tVcgGX_O?B6b~@E5xoh!J!dScR^KCp_OP=vVmiFuck^UR!FYFAz2fxiW<<-`+KQuJpxJ&P9;kiu;XC(O2q ziT>J+vdR(T?XbJOO~yUfdo4Z?;;57I(VWTKtA%GKYq4RsrdxMnuV@rhYPJ~jkI1AS zk9m03c}lN&83~DbYUcHOOmyzz2L7{*F)RjltZHzQ{=KX?B~R?!n5 zzQ0A8%r|&y2X+m!T@U?5FQ0~~{wbl?EY-#^UB1-w7Oy5M4#mAUQ(gQ)xluZQiE)E(Ct=N4GklGA8d4`Ez4{DUht@RcLpUj%suV-h8GT>Tl zn|CE1ob~Mu+4tjm{H+hlN#5dHgPG)g=Q--(ytYRh)dr+rWwOrAc6Ze<^X`H3VrPFFH-YE*;FkgoX2?;*#a zo;eA{M}K}+5Xa5j8V~d?ce{lte~eBoleJ?n$|KahF>0knyMLuVqWL%YUegp~ zx#>#qH!6)|rgEt`hk`66`0}kMvSm71e!qDV7S&858Dv)XtNAx)$&vIRMUW@PV|phf zMn_SXJ%p)_J$4FHtSBqIVFnOs+h8o2)^VJwKy^Z!JYLFf7s(mR$bupyig=$*jZa=0 zj=LgVx{GX^vTGi~{RN|%n8^$x0}$C1OZ`EqyY;*Tt+9On+h;^;bp`8ZN?Ug}g_6x91f-T@uvcs*C1zQHI# zMOLiXBZkoDlf50QbG?d@oH4(`F7R4g7qJpH(m6(9$pMdir>G+O)zR5E{f2K7P9=UP zE5bwZ`>4kh7vNShSW8sirSW$b8KUXczux;j(C*hYD+8#(cpnBQ=&iYXX;J~$L<|_M zgj{R*P;Zm+;siigf=@Uz!ZLZkD|~Ld#&tjxe1@AI<;Xv;%A9pEujDbpqgES-+OutB_+YO0)T7GG86MoA+&Y;E7y5ei=kJU2dHMu7c2^YC0!3Sx5?k9Q+vAJPchzn zSYs;`v)B>&EDI;&nCXu?r6O?C+PGo+H|~&z{?B=rh01=tLT8SRC^cbVEignpzTVHl zS_FKA01pz9;s!_@B=nc5Y50Atsbm$57*g84?{5*voWJ>~CdYK#s{GCNo3n_)^F!81 zO#HP!&mS3f(Tyx`8X#sf$2C&V8f!35%xuwt=rgfH%v2~bCtnI_o#5f54wQ%NeD=7f@>GOTNkQjVPE z1W!@y>Y1rFe&iycf3Pqo%6fy-1~GSf{{%jJJqeyR3;c*=E?SrAOr0;*(7R$1(8U!N zO~qbuO;JZC@@kLgkNh|FXq-9 z7bspM2V`Dq8dx+<&h#c68LseMpVuBTDli2q2a;aQauCFsGe8$E{)2pEe2sp+6otHd z*JqL+kyyzeV3l=7K`CxJd_8{QyL=s^9{g@|w=$sOxFk;1e2CWG<2Q+l zJ<3p3_H_%b4ZB)DvBI^=Iz!NKKUFh-``O$D&vY=S{Y4jWBkq41y3|F|-b8I+i+4|Ip7h2(RT7yq*`VWG_W_;Q(S` zHpEL3{-Gq_d6!=%NeS+5N12&Y3Mwg8#);(8@aEUQA99{sziyXp-IYkZ)a2=TSlG3* zYY|!KHU*-aam%n{>3w2#7QE{&jS^;gmaE8{lePB-E)}>~T46(gat0EB*&D-O%{g=G zpG+{wblz7sxVql&)V@F6e{W5hoKopEFy_IUsbgM*oNfqE7@21>wf9XN=4sd@7zOIp za~9;hnq0#12iYo98&6@aS5A1{taQ};5ZOtfejZn5LfZhoF=+mU|2kw7a{7Vk;s9<2 z^v`KQ{l97*lx*~#I22&_pB7&yF(U0!a|@32dbMpd5DoEy6;6_*Tbd^(@S1fkyZ0W4 z+`kyqcfy~<+TU;qqJ?$3z?K#P4Ct_i%W};?j(@ohQBlEZ#$T%288}i=%JBMh3RJzi z@Dvn%?w*^z-#?A>Cd=DFIcWsqZ**2^{-!~Nkzam^>vCfZ@$>sI;z;Xev)7S@FoJ>Q z_`!sAYIO;I@-H7;MO{T&vYITqM7qR#z^%Tp)42bml-d4#$(E3eA3D~VqOcH15?y#! zF;d4%X#)J1E{Ag!4T|CuO=~KW;f^T_e#9zp6}j&NW&DwF7?3qDuNoR`xSH=X^*q|) ze^1pfjl*s>4Mt733mq%qmBC|?xoQcvx5tA`ztvG2{3eM*z<08dSz}YG4uDrYc(Uj# z)aCX%OaJrq(Gs5xo2;fb?&YcT?pfK~k{oCUB$E zM8hEWJQi=YP2YFVx|<(s4JKw=?@PIj7P-~0sm2)jo76;F8Z*NVDF}zM1{|25LXn(4 z<4-Q~JM@{~cgcrb&EL;=5^6q&8nv~`Ybd*KXKRXP6|1ffY5k?nc^~+KU4BF;CcvS_3t8|80eQ@BiLuF1dMbv#!{Nq%i9 zx#%Iq^g)3fhndrH@ooMgm)$_<5(>~i2>)-BLTj;%#JWRU~p{{svC4X zwKwPOXP-N|rT;$_zZ+${V^z+XQa`_%pTn${s5}&408%{?QXgf~sf8eBk;5Y94n!N5 zY3T`{jOi+VazmeWV3_O0%Abla6`Y)jPNVMHs&fNNmc@z>1Q78QOuN-fGms`Iv0JJh z$X$Sd14r>W1@n6&Uv`rRuq=AUwqRjXB#X7ZuW?4W%9{)yisfI2KEoBQ2Sn9>J zJ^(a%35Ur4DaPN(_;R;~ZxEd|3MdA}==T&UdjsC|4ZF)qZH$!7^Ezg|r+-axTtM|K zEOU%!L~D{3)O*h%l|7^5Qs4jxAUBhuBkSH|F4ZU8d%7bGqanEwOBIw&J&7fhm z2p1iv9merSQ=h%rMKL>w* ziRMlg<_rT8SWdBK?iEO-k!OWX5RbE8W%|O^di4=RlU?JTIT%(>Jy!lWpkGenbuwQYe|Fzf}62Q-Sr}fz6=oQ5{=rC{ArUYe@RxM9G??f-3Ll1c$5)B>B8TKW7 zV{vP6)soAQm~>@Wz8r5P_-Gy(3o?rkQ?IONBj%nl>a#21>p~1fp;S?feP9CA+Q`($ zr1d$nvI6IZs{;D%3P@)1)3`p(YOm0np0mfi;18OVY!TU^HVg46g{4YUP{H%4#QeM_ zL~~l7g1ZZS2GnWJtia_#(xx@_IF(0f2L zWLwiZ?d{oYn8iw75nmF0+RdXb7)M$#7E@#lWSAkj8Cb`_hXcyOqMLa`v6r+i-YiUkzALTGrh1X z1`c(hMje~vnz7~8p~UGZHn`EAFNah?$$f3kNM%8U4 zKd2+%2<>u=Mq|Z0dBs76*pIl7_tR^DimSv(>WpMYlPj zCGCUyRV3o*1ESvtmB#^%^OYD_Y<#sE;ZisPmBMz5^K}i(YfOtNcF4AC{^PEt{@t^( zA^|h6WGlF%z0)C3{|qYj?v}b|?IeCkBVn^TV^~m)(A7p?HU5nsbMh(vmOTHf&vpRL z%#`sBvPHI#(>N;q=wq)Gz-0fC1dd)Cf}h_|CW2!$fz}otnUQa8niAYg~ma5u1lyg`wKp^?}yk{)?HLj8k|KNm3uEh$qKf1y2-~f z=r>fBpBNC{$8?+;55KY8tK?^eq=Y)A%7g}d zt}%U#uZ3;m?e0L}D7wu6*T#5E-0W9X>`&;Og z%5WLv(mUVw5+f5%C}<*P{$&rsJ=^07cB^Pdg$j1r<7>sftET64N;?nt0hN3?YpcK! zxQtGc?o^m8Q-c)WXUD;gu=d+mXX?+IZY^6)8VZ144U?SSYX;XhU+jJF)QIGan;miG zo62`4$;rLo^*f`=N8okzaqR{hv+&{-}E=j+8pgpvnQ2EpB|W^;WQ$ zgljp{uV95)`53*x#HG-Oa4D(d0KF2J8%=vMl0J8mwiwD|{<7nZrsgN z{q^P63=`k_Klj_pZ|6H7_kb1BcUu$QedBWGdvv+>hjd(gPepN?Pk(*TKva@{_6NvC zM3UIo-cRGr2YTe7?|#V1uU(z_y3?GY3$RaMy5XQH*h`gmd=rbbr@D&@y+AWS($3NXpWdW7w`_nxE2e}`728ljH( ztm^{TN|qv5nlpA>aGbFEyXH~0=J$5cbvpdERo|AC(S0&!9iPuTaumpd&fn*EB;PyN zUEU9nDcWiJAXo3WNq_Hx#YuCko>f)`|gjxV=iC8PTyyN zISm$^q7tcDf9gl;vR&&CJ+J&w1>P|#tCYn zFMU+i)x=TXmAuU)Sqfoc4D+W>@>DH{(r}!H7YA(_%r%y?N|mQJ$AD6t-pdYShzwte zc7#QYjY@rsp+L#{5z{RysgF60AssqRl2VZV0viLhbsd0DVpZ-Xr>h_DjU=U;0F37) z(RY)*N>jz;g#fOMlM5eR5?8^xY|v>;Wy0$P3IIietBHN5U+34k2Ahsz%95djvqPZ?h zph>mbhS#pe#^V)h57+WsvVnoP{vp3e|0Vl*5;oFh)9+xxeBlFhwHoz8jUEDia44e- z2G=;ano}1Us(ZtNcVz`~H*+%2k0c#_b>)mSeXhN8RtaJY-86Njg116G%Gl{O>yp$j z$ng;O(Fps93ihOahx4m|O!h*PiSfZ0+9Q{aj9`WNoDfDv_E5qagPchB@U75I6P=4Z zM=OS1sN;=N$?NqcV>9~_(+XPNZKC6q<2#;QXuAGCo!;+(14*dfPV>=dI+*Na7IB^M z#}Jy_aNIf_1^+*Q_mNBgBvzLOd2Z(_$TD@u{U#CTd0MKn@E0(@rFvxlT16j$=-^~P zpP$>^ATc1~^~U5Gi#J4=CfehC(o<5slVnQ-PvUc8)hypuS%b zxy=c++wespDLx1nl7^nCa+&6*O6m4vyS|=7a_>%;(@B)#j#ZC2Tt>AsZkUkz>*6WN z%2Lv}_EQkP*!_BnOYZht=d$|;*zuZkrifzC3%of~W{F|1D*xg# z)>zA!UED7B>>oj<#Tn4(S4b?9YuJ7Vo;H3W@g4-NqZbz)`IkjvL{Ec=1LYpKVl9sn z*;z5|UGI)3g&v)}>*fiG(p6>r;!kYhca6phvL#_+B)}GQ=jk-}-2$tIwakK&2yV zEFAm|CTkw-K-ATL&FiGiC`JmABeJcFATMhm#2gM*(hJXaGkgJ%jd(cy7KS&ngZ0lg zVWX@pj(pSQxqSGlWM<)Bp~csTUrS+^#}D|>XAhwVDOmHE$@uEnXEbA>?6qj53?<2- z0r;zF2(D2w-x)6jD&H+eN!_(UE^_Z)ksn>fF&6Pp47$98$9Z4!G!R4B(}7kEAEu4# z=g7)cn!5A(L}SoHM@a#F-piWbS$uJ^_F-_b^KPIM*nGadknp;_8GR@O3Egr#;9{7D z3E-2Jh!pcT4aA~Si^*aO`RIowP?S(STFq+nEq25Lp(jqJPsmY?YD#QPzYtm1>Oq+T3U#vN{Du!n^xRg2h;6-ot?umz8eBMrHdx1?rT^1(|W< z&jqhM)Te)J&lB0 zO=7fyp+-y}`fZf)iP*6*drwJoqa>^zuCfu^G(#@VskZl{jJ;I* zpq0bm%!tjC7vfl`cFB1alqZ$a`m%R-bNh!_hKF!{-e0qa}yrhSkF~Mq5X?!)hVI=ve!F961e#9zXTN>_r%;=}42WJ5z*U(M^Y@`w+dH_mpN5X8 zbnI_@_;@nCRxcfhYj*sbfRncVfZ`K=6Xk7A@;CCI%vMjToP3~Z;3PD7tm^73eSn$D zJE!!xm%JzPho+C+?*=^OU21X22(5{lzv@Of?|+YFG7lO^Vo1-c)m&8efi?QAQ@imb zAjxCG>X+XvrJ(xvJKg-;1-iN|$Ibb!EK`TM8MFT%f-maE@#H|W!nt%J%BfB!jT2>7u)Pce86QT9tLvhhDeP-c`mIj67G*)<1WRg^H7+VIY ziqf6QQwC(4v-NfXxf%5wIRAwruz=%c_m%_~A)1f-o(X+`%ku;B?oSfxG}kbr&YE@z z)hCRYYn?S}y_%diID@i7BLQPSrkLvjX&`qiR%!q5ZT7@@H9O#7!G?TB*D(y9H#EoT zW;|}!z|HKy-D^j*j!S|czp(>Ic{*s9uUg+k`=)jnckt|h_ijP29D5&*tp@l5IcPQAp=+EP4+Qea}vH#Usl}G9>;uDOmZ?f|? zv3bo_-w9N&wf2nu5_7$*nVo&;#8+U)&z4r=7e?;1+cDE``aAx;uHkZ3Y2NGRytl$+ zxX9?yB}F#+27th<+^aQ^*=A231%UzICq0j?*tFEVoNryw!4pR|=&8+#UUuR|XI!z< zRxlZC0G*J64k2|<1*d85)d!XPG0#r$M>Q`njIEiOMuUajW0T8@LG@D^i0`SNNMT0q@wOUH zn@o%O1_*HK9F`STl!R>{CmWbzKj-O5__!+}(sw_sc!kCCVprhAo2#ji`o2_d;&#Pl^~&Bg-I( zNXn}B`uRL}lcIr`^tc%Dm9S1$jWGVoic5`7oJR;OL1ue(TX_@E4`<22Z?+SKT9Vu}^*KhJ#9^Vi*K4f*HH zVqwY29(Na2qn#-?P z!M*2~yp#8omi*zOz?TqBYf%j2THJWsM+YtobMVrSgxM zvzmS+3q$r0*2P%%u73_I+^Nv>qx(c}j8zt*5lC@X$RFk{MRvp$*7XwGj*av^`76k~ zrwVduv+daVU++^56nJckDfKcYfXMtuR6b0=Bf={yAdx0f;fm}2`{IqEY2nm^aKZgkQsXs|=^^}O} zn3`>fd{|7Lbf|5U{bO2LeY$?bl$>OxNm2snXYTQ*N7BRj;3l_8 z9L0(LQubPR{+@Md{HW(8df}@^V0bem&`Bs&vaWksHw+ukKZMFvSl8Bs-+ZkR;Qz2+ z7PF4e<7weZ9QYNl(k%n~$>+9SbCVt?n0T%|>DbXGsG+(_gnVs`E-mn90^{=vxwExb z)jnPPlk8ewvcWd-H2V`H?ZRRp-atKO*guT=5&n_+>qQ@xq&C%a2s!;|sn-DyoXthM zdz!x0?Ytxp6zFuryLDVdyf0rOMFAdusoq?{Hi&-gMmMDy$<}&4E3WgZAwWtkoUT^ zI&{TM5qgSJKfk>y89&(;fWxiz+>AA&B!EJ$uVw=bd-`#=u{; z%`OXKy?y@^t;Jr3U9S(%iN-G*3h4Wh6NJ{q=P&xZL!frNzwwEq&7n{fCMa2BOfG;u zWxN>J0Yy$8P^i%e>q)MZ)^JIPk-eJn@Z|zy7tb%RJvaB-6F2 zK>!;@Zb^IQH9|wvpR;1G^uGtaPD^;9rjVv(Un=JHAWLEC-cc8ItgIwd(Z>)(KR_06 zfl{&@p#HHf(M0>mQJ8UDzn+R|GzPbE?I@sNkB1pbD1?cp@uM*s=IFlK6U)QIK|h(k z(rsTD>3U?kG1afw*Ll{IP9Ec$Nt+Kh_N@qquQz-6mEi$N-qc1n8y%nij!T7b>4pyQ zlrTDrvr5LYGB_G8ALix1yuua5j=D=L(;i1oL9im5?278XNzV_USH%H}4s=+q?OTJ& zmCmzXjXNT8-z$#!3#Nye6hhPJh~-~ABb1rGnw9Lo!2o59G~Z1>f-x3Y1p>lLbk_h) zp-l5%rl>gy#?X0}RqcGJu+Gq&XmAVx%@@gnwkmdEt_L1gZZS{0X3C=J5l4yv&9z#4 z8`pkW%315UY`L<49kf2_^4mtlki8z^(JE6-{p{xRMm=+iz1TXiJys3a7eiKcLxPi_R-Yf*5cO~W`4xx$(coL9M`pbRo z%6&&>5aE?N>guKB^xrG%GOa_h6V&T~*oi+SErql&b+J0ypvz(o#IUVIHXY2Y?7^I)tQH$lpdSfh0Swzup zHlgftC~6C7BFvf`6MjT6Pi&rB2nirYK|`)8YmF1i?%PuU+{bPn(b^SR83ZX_qms7;~bziGg^$I21nlthWMHvnFeQ{S_SooeC zip$*&t~Q~m@ArbbBV~QJvHfVB=2ciKCm@zp9Ld%qYQo5EHqd!sKk^fbTpXrwVlOx7Ed?QW- z6GsZ3pasI^{2{U_?WXqffJKUI9uW99-CeH+_w{_s4XhHd>U)rDecsGbLBAo#Em!gB zrx5W`iyB7+%rF#CU)*VvQe$IeY%!us3=Z{BqY+GDthLCZ_26rbrPtEBym>A%{7kpP*Jb~1*GQH&! z8>Wr?2cmY%>6j1rCH1FTLEv}O*-cap^-Kd z#$Tk|r>c90n4+?AOOYzt15wqVUHbzdlT7n*^=v8ex!P8&&V@6BQLnf5gt}4Ii3>|j z)*NFdk7iWoKBy`Eq>^FZCG)A_4?A4(j%>dYqWsr4KgbVCKdQSFqxl0w25s`Xb;*!S zZoa#87C2w)7Oeql4dX`0%~Cx5=krQW0e1Ni-=ttGWZzn+MFksFfAMVjY`vWOA)QG2 zwb*2mjP-d*hk9&3Zm&RQc{iwQ{6ft)4T@U|Q8pKu?I!Nw(aHtZyoHe=saPE}jWg)V z^C1SY)2dCaBij!5|5I`z!E>FnMJzROxU%6+?y#5|J7YjgYt8&CSC)D(>C~;E3X&g! zN7~IR85VZiq`Db+S=~%l{;c=aYZ+&&Ih63G&G$I_3qf0l?BN(FKRR$TB_@IuOIZaY zt9XFNGH#z;A6%s7{1;PRT;zB)_Ayw0w&AtE6gc{j8mFvnM>mORbns)}*T69$OC?dKD65ZX5jiv+Gw z6|!os;bjkz)&9JcBvasR3A)<_g4k%-<+m>ZmoV4jG|=mmWz43hH;nqf^7Vr8us7G^ z(i*_$EIIJ->*zdGbABi45=)llP8t~N57;ocRqlx+M6I^_hmnQMZTI~PwCF$zi&C}KJM6E=<=%t6{^vR{44KWN>;$j1S8L*CU?|Kp9)(~En zZfF+twlLqhtSgna8>w>Z9t;1gC35&{W+05kK|Ysvi!_s#+R^X&_B%6t^)MySAA$S& z5pqj0)fR-r4DnE<<+03s**5ITy~!-fr~@|{b7Wbb&<~_Ez}-w!Z&e)fuUflzUcpv+ zjPkNg79l-pg#q)hhOHTVm3?R2qwIEC!ZQ4-sq-(uFQt78*Us0XDeSgN@$bbB;7!a& zlWhaTZP(5o^fz&7HxSC4^$QrPi88w&`swPENjVTCV~a;-&kLVDcmc$}_ozece67;E z$8O29Gq;?UoRz_uqu+-ffv{*L^B$faP`DVAw?seJfp=$FdY4Y-qnhK%vWd~8c&*?r zg;Z#XyKVmxF;XI-17s8FZ#vy93?!DQ6gAZTrm^|XO$_e#+0(ta&(h`IHI(#n#*u~X zq89^85a-kXnCh5fQGUdF3Cc9>H19y{%Wekrh^k z7G_jZJVOn{mO7Ie9E?ZJrs9Y@$EoItTZz@&2uGtPa@rne? zx2p>&qsi!h4)`0V$=MTEV0Bm}w+X&y@8Q58bKMu*pPEls=JXShnpSNqfAk<_wtQie z)lZr)7g2W=*r7kF6Czu0bSdLSRg|w`^k;47TP7xd@Wz#lA|Otf_c#xIRH0N_`+vaigL`b;?TT3?a&Sw!g4GBGFJ{p|Q7r z)KG5qI*rWkU>c4|za(LG;*_8o_k#-G-kGjWO32r#7RhT`b-j)C-MHvagd{R|vV`t37 z?~it$FROckDfTn(ji}gH?Cqz4QHLMchsoz@8em(wBZ&Z%CG=!3=e@C0OWj?How1%r z+F;(V++qH-hg=r!=coq>d#^05TNXxmw`&6|<{%m3eRI6O$d?6bSTj}VbJOQ8Qg8|m8q$nbD#Pi_ z+1|kz*uhPF=K@2)#_u1H^BndDMs!rPt?uHUYHqf0)-)5XLJcsbN^e3>U|0|Dl6pclS%Fez1nvh$2%!ckn{2x@azM2K;>IO!jAcD+U;~bTOvmra7Dniw6{HNNi%;Tlo7i3ULL=S0gxwJSp4!8VhE>#Aq->?r*@0Fr_jnLao|i zE#BPQX?tyzbUDZWFxGA5CO^|f0WzB?VOM^|T8$J$hXxGoy)l+8In_w-gU$c7=?i0h ze>DO)I37*bm(A*u#gAi7J%5wo@4Rzr>6mOJ%VfdyW{Lguk#5dL-%jlrKPU`mc6{|p z;%_^36qdZaX8RjOV_))xBjgknpP0d9tb$?w$&2%5!=P7 z|B>C2uZ*?yy2{;=6-1)Xl>kLh;P)49F;ASEUxZr^f}hah^8*iGs%|tt1s?#sTsbdD zxVHvXmq!*y96IE#-73Rx)h4UrYFWtj$$awGrZMmFj4`mtBTC2jg7C91Y-<|1lR?<< zP`#_@WXM~@uKU8hD-gRP-lcumTJfUP1Mc7=XBEQ%Bdblk{JV8$bi!Y7kPK-R^ct3o z#7oP!{m;X%opXRoybSY1=CtKFVqf(W!TwiVz+2v&SJGFP7I~~JphUCTjdrS1dA=MMDURK+rl0h&!XB-abhIt|vU+K_sw)GRL1Cl7ag zeegXHVTwqa7WmsDr6QKEEh-8UApTOsw{*DD|z$|8<2cN-W7@@M6GS3{Jkni4bK~zDf-7M_;c>FB!@G#{H8Hrxn$>`c+|T%lXw3)=|6`^kb~f)mbIQgO7g+lNO*hssBnV4Cie1yy=Z#9OuQq zkGBJSE8X>X#BsC%Qd$AUXGIR#C#6$A0eo_`1wsh|mlnW-ZX?=QmD07Krv^Jnqa7$EHV{s&Z-!S{g8#Tq<<81jZ8)&|ML)-m_W@_jUm;Y^e9ll_? z^;wtHKQ<*Y{v%Euzyp``x>wWhK2~Wpwvx1qOUVyk#e*_uVjXL;{+oM_UOR<9I1c+i zfUof%&(B&Cri#NVhy|~$cz@AaU{BWn51>3)z$X25m0Y=z9V)Z7fa6$B(|^;zC#$`d z=!!ZOYcBt2_c&^j@T}=>y|6;?+=|S{{qpxIUXX$z_y5I62jB`OwdBGu;E$HXWmXAb zqeKu!E$VKVXRlPH!Y;|=8s|HvS<7T4BC-PJyediNOI!xwBw{MA(r!o*Y9QU_R;l~W zjC}Gi58ZddpUtWuXZnG^OE2bwA$!+g_|mrsCSKq99IR3Sz)ng$hcSLgZp1DRfp#=` zX59d2pBPXfw;kMJ(DCx%&&i+48B+3n=5WPHRW&OYhjk5xsUpVl^A>K>U$Kkys@k@W zXUK@og6`x2GX3bM=lWm}vsh}gA{2Ha`b&WaXlbv?Pi)uY%wmcb!NpE7n+`4ok+eYd z$As?pu-Y@l^KkX7s+s+StFkEst&0a%D$l4aeO5IBS&0TZ1dxLw3gP|{t zLBjkSiN<4=Qa{&h{aZ(m7>RC)8#=|z+-HET?_)Zmj#2s+j@dHzubr{_t;BO#1cgr1 zktVW0ze4I*8cgC6an0H_X<_CEJYrQog>nVI!`qAt?Ax=R%daZ2zXs38RXTn?WzAwJ z)brkmV0hMrR_^fjgduMS@jNY6*~EQ9wlEs;O|bXuqYTG8{%c8)`{KQL&9%r@^$1~$ zQJ6>M$>Cl8p}-p7nGl~p9N9N?b8@4`vG!Y18*8Vgt`XTg7|3E(TG;Z1a8@$>7fko`Hxw`^GN}vw4 zlEpbam?P&^&Anz~59I};^C5Ok8&~mcv~jUKRlSs@{2YUWV>Ew-|4n=#-WaVK{o7(>^{y(x9-iiPK diff --git a/tests/check_f16_weight_error.py b/tests/check_f16_weight_error.py deleted file mode 100644 index 0af1f04..0000000 --- a/tests/check_f16_weight_error.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python3 -"""Check f16 quantization error on ViT weights, focusing on dimension 679. - -Also: run the same ViT forward pass in Python with f16 weights to see if -PyTorch shows the same amplification. This tells us if the issue is inherent -to f16 or a C++ bug. -""" -import numpy as np -import torch -import os - -# Load checkpoint -print("Loading checkpoint...") -ckpt = torch.load("raw_weights/sam3.pt", map_location="cpu", weights_only=False) -if "model" in ckpt: - ckpt = ckpt["model"] - -vit_prefix = "detector.backbone.vision_backbone.trunk." - -# Check f16 quantization error on specific block weights -print("\n═══ F16 Quantization Error on ViT Weights ═══\n") -print(f"{'Weight':50s} {'MaxRelErr':>12} {'MaxAbsErr':>12} {'ErrAt679':>12} {'MaxVal':>12}") -print("-" * 100) - -for blk in [0, 7, 14, 15, 16, 31]: - for suffix in ["attn.qkv.weight", "attn.qkv.bias", "attn.proj.weight", - "mlp.fc1.weight", "mlp.fc2.weight"]: - key = f"{vit_prefix}blocks.{blk}.{suffix}" - if key not in ckpt: - continue - w = ckpt[key].float() - w_f16 = w.half().float() - err = (w - w_f16).abs() - max_abs = err.max().item() - max_rel = (err / (w.abs() + 1e-12)).max().item() - max_val = w.abs().max().item() - - # Check dimension 679 specifically - if w.ndim == 2 and w.shape[1] > 679: - err_679 = err[:, 679].max().item() - elif w.ndim == 2 and w.shape[0] > 679: - err_679 = err[679, :].max().item() - elif w.ndim == 1 and w.shape[0] > 679: - err_679 = err[679].item() - else: - err_679 = 0.0 - - name = f"blk{blk}.{suffix}" - print(f" {name:48s} {max_rel:>12.6e} {max_abs:>12.6e} {err_679:>12.6e} {max_val:>12.4f}") - -# ═══ Compare: Python f32 vs Python f16 ViT forward ═══ -print("\n═══ Python f32 vs f16 ViT Forward Pass ═══") -print("(This tells us if the error is inherent to f16 or a C++ bug)\n") - -# Load the preprocessed image -ref_dir = "tests/ref_phase3" -img_data = np.fromfile(f"{ref_dir}/preprocessed.bin", dtype=np.float32) -img = torch.from_numpy(img_data).reshape(1, 3, 1008, 1008) - -# Extract ViT weights -vit_sd = {k[len(vit_prefix):]: v for k, v in ckpt.items() if k.startswith(vit_prefix)} - -# Run ViT forward pass with f32 and f16 weights, compare outputs -# We'll just check the QKV output at block 15 and the block output - -from torchvision.transforms import v2 -import torch.nn.functional as F -import math - -def get_abs_pos(abs_pos, has_cls_token, hw, tiling=False): - h, w = hw - if has_cls_token: - abs_pos = abs_pos[:, 1:] - xy_num = abs_pos.shape[1] - size = int(math.sqrt(xy_num)) - if size != h or size != w: - new_abs_pos = abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2) - if tiling: - new_abs_pos = new_abs_pos.tile( - [1, 1] + [x // y + 1 for x, y in zip((h, w), new_abs_pos.shape[2:])] - )[:, :, :h, :w] - return new_abs_pos.permute(0, 2, 3, 1) - return abs_pos.reshape(1, h, w, -1) - -def apply_rotary_enc(xq, xk, freqs_cis): - xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) - freqs_cis = freqs_cis.view(*[d if i >= xq_.ndim - 2 else 1 for i, d in enumerate(xq_.shape)]) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) - return xq_out.type_as(xq), xk_out.type_as(xk) - -def window_partition(x, ws): - B, H, W, C = x.shape - x = x.view(B, H // ws, ws, W // ws, ws, C) - return x.permute(0, 1, 3, 2, 4, 5).reshape(-1, ws, ws, C), (H, W) - -def window_unpartition(windows, ws, pad_hw, hw): - Hp, Wp = pad_hw - H, W = hw - B = windows.shape[0] // (Hp * Wp // ws // ws) - x = windows.reshape(B, Hp // ws, Wp // ws, ws, ws, -1) - return x.permute(0, 1, 3, 2, 4, 5).reshape(B, Hp, Wp, -1)[:, :H, :W, :] - -def run_vit(vit_sd, img, use_f16_weights=False): - """Run ViT forward pass. If use_f16_weights, quantize all 2D weights to f16.""" - E, NH, HD, WS = 1024, 16, 64, 24 - global_blocks = {7, 15, 23, 31} - - # Optionally quantize weights to f16 - sd = {} - for k, v in vit_sd.items(): - if use_f16_weights and v.ndim >= 2: - sd[k] = v.half().float() # f32 → f16 → f32 roundtrip - else: - sd[k] = v.float() - - with torch.no_grad(): - patch_w = sd["patch_embed.proj.weight"] - x = F.conv2d(img, patch_w, stride=14).permute(0, 2, 3, 1) - pos = get_abs_pos(sd["pos_embed"], True, (72, 72), tiling=True) - x = x + pos - x = F.layer_norm(x, [E], sd["ln_pre.weight"], sd["ln_pre.bias"]) - - block_outs = {} - for blk_idx in range(32): - prefix = f"blocks.{blk_idx}." - is_global = blk_idx in global_blocks - ws = 0 if is_global else WS - shortcut = x - xn = F.layer_norm(x, [E], sd[prefix + "norm1.weight"], sd[prefix + "norm1.bias"]) - if ws > 0: - H, W = xn.shape[1], xn.shape[2] - xn, pad_hw = window_partition(xn, ws) - B_cur, Hc, Wc, _ = xn.shape - L = Hc * Wc - qkv = F.linear(xn, sd[prefix + "attn.qkv.weight"], sd[prefix + "attn.qkv.bias"]) - qkv = qkv.reshape(B_cur, L, 3, NH, HD) - q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0) - freqs_key = prefix + "attn.freqs_cis" - if freqs_key in sd: - q, k = apply_rotary_enc(q, k, sd[freqs_key]) - attn_out = F.scaled_dot_product_attention(q, k, v) - attn_out = attn_out.view(B_cur, NH, Hc, Wc, HD).permute(0, 2, 3, 1, 4).reshape(B_cur, Hc, Wc, E) - attn_out = F.linear(attn_out, sd[prefix + "attn.proj.weight"], sd[prefix + "attn.proj.bias"]) - if ws > 0: - attn_out = window_unpartition(attn_out, ws, pad_hw, (H, W)) - x = shortcut + attn_out - shortcut = x - xn = F.layer_norm(x, [E], sd[prefix + "norm2.weight"], sd[prefix + "norm2.bias"]) - h = F.gelu(F.linear(xn, sd[prefix + "mlp.fc1.weight"], sd[prefix + "mlp.fc1.bias"])) - h = F.linear(h, sd[prefix + "mlp.fc2.weight"], sd[prefix + "mlp.fc2.bias"]) - x = shortcut + h - block_outs[blk_idx] = x.clone() - - return block_outs - -print("Running f32 forward pass...") -outs_f32 = run_vit(vit_sd, img, use_f16_weights=False) -print("Running f16-weights forward pass...") -outs_f16 = run_vit(vit_sd, img, use_f16_weights=True) - -print(f"\n{'Block':>6} {'MaxErr':>12} {'MeanErr':>12} {'Cosine':>14}") -print("-" * 50) -for blk in sorted(outs_f32.keys()): - a = outs_f32[blk].flatten().numpy() - b = outs_f16[blk].flatten().numpy() - diff = np.abs(a - b) - cos = np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) - print(f"{blk:>6} {diff.max():>12.6e} {diff.mean():>12.6e} {cos:>14.10f}") - -# Check the worst element at dimension 679 -print(f"\n═══ Worst Element (dim 679) in Python f16 ═══") -for blk in [14, 15, 16, 23, 30]: - a = outs_f32[blk][0].numpy() # [72, 72, 1024] - b = outs_f16[blk][0].numpy() - diff = np.abs(a - b) - worst = np.unravel_index(np.argmax(diff), diff.shape) - print(f" Block {blk:2d}: worst at (h={worst[0]:2d}, w={worst[1]:2d}, e={worst[2]:4d}) " - f"f32={a[worst]:>10.4f} f16={b[worst]:>10.4f} diff={diff[worst]:>8.4f}") diff --git a/tests/check_preprocessing.py b/tests/check_preprocessing.py deleted file mode 100644 index 1875724..0000000 --- a/tests/check_preprocessing.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -"""Check if the preprocessing difference explains the ViT output difference.""" -import numpy as np -import sys - -def load_tensor(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",")] - with open(path + ".bin", "rb") as f: - data = np.frombuffer(f.read(), dtype=np.float32).copy() - return data.reshape(shape) - -ref_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/ref_phase3" - -# Check if C++ preprocessed image is available -try: - cpp_prep = load_tensor(f"{ref_dir}/cpp_out_phase3/preprocessed") - py_prep = load_tensor(f"{ref_dir}/preprocessed") - diff = np.abs(cpp_prep - py_prep) - print(f"Preprocessing comparison:") - print(f" C++ shape: {cpp_prep.shape}, range=[{cpp_prep.min():.6f}, {cpp_prep.max():.6f}]") - print(f" Py shape: {py_prep.shape}, range=[{py_prep.min():.6f}, {py_prep.max():.6f}]") - print(f" max_diff: {diff.max():.6f}") - print(f" mean_diff: {diff.mean():.6f}") -except: - print("C++ preprocessed tensor not available. Checking against known resize differences.") - print() - print("The C++ bilinear resize operates on uint8 values, then normalizes.") - print("Python v2.Resize uses PIL/pillow internally which may use different algorithms.") - print() - - # Check how different the two resize methods could be - from PIL import Image - import torch - from torchvision.transforms import v2 - - img = Image.open("tests/test_random.jpg").convert("RGB") - - # Python's method - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - img_tensor = v2.functional.to_image(img) - py_result = transform(img_tensor).numpy() - - # Manual bilinear (approximating C++) - import numpy as np - from PIL import Image - arr = np.array(img) - h, w = arr.shape[:2] - - # PIL resize with BILINEAR - img_pil = Image.fromarray(arr).resize((1008, 1008), Image.BILINEAR) - pil_arr = np.array(img_pil).astype(np.float32) / 255.0 - pil_normalized = (pil_arr - 0.5) / 0.5 # [H, W, C] - pil_chw = np.transpose(pil_normalized, (2, 0, 1)) # [C, H, W] - - diff = np.abs(pil_chw - py_result) - print(f"PIL BILINEAR vs torchvision Resize:") - print(f" max_diff: {diff.max():.6f}") - print(f" mean_diff: {diff.mean():.6f}") - print(f" This is approximately 1/127.5 = {1/127.5:.6f} per uint8 step") - - # Check v2.Resize method - print(f"\nPython transform output range: [{py_result.min():.4f}, {py_result.max():.4f}]") - print(f"PIL resize output range: [{pil_chw.min():.4f}, {pil_chw.max():.4f}]") - -print("\nKey insight: bilinear resize differences produce ~0.01 pixel-level errors") -print("After 32 transformer blocks, these can amplify significantly.") -print("This is EXPECTED behavior, not a bug.") diff --git a/tests/check_rounding.py b/tests/check_rounding.py deleted file mode 100644 index 8864b41..0000000 --- a/tests/check_rounding.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python3 -"""Check torchvision's uint8 rounding behavior for bilinear resize.""" -import torch -from torchvision.transforms.functional import resize -from torchvision.transforms import InterpolationMode - -# Create a simple test image where we know the exact bilinear result -# Use a 2x2 image resized to 4x4 - the bilinear result should be exactly deterministic -img = torch.tensor([[[100, 200], [50, 150]]], dtype=torch.uint8) # [1, 2, 2] -print(f"Input: {img}") - -# Resize to 4x4 -out = resize(img, [4, 4], interpolation=InterpolationMode.BILINEAR, antialias=False) -print(f"Output (antialias=False): {out}") - -out_aa = resize(img, [4, 4], interpolation=InterpolationMode.BILINEAR, antialias=True) -print(f"Output (antialias=True): {out_aa}") - -# Check rounding: does torch round or truncate? -# For a pixel value that should be 127.5 after bilinear: -# Truncate → 127, Round → 128 -img2 = torch.tensor([[[0, 255], [0, 255]]], dtype=torch.uint8) -out2 = resize(img2, [2, 3], interpolation=InterpolationMode.BILINEAR, antialias=False) -print(f"\nRounding test input: {img2}") -print(f"Rounding test output: {out2}") -# Middle column should be (0+255)/2 = 127.5 → check if 127 or 128 - -# Also check: does torchvision Resize on uint8 go through float internally? -import torchvision -print(f"\ntorchvision version: {torchvision.__version__}") -print(f"torch version: {torch.__version__}") diff --git a/tests/compare_actual_resize.py b/tests/compare_actual_resize.py deleted file mode 100644 index 06da279..0000000 --- a/tests/compare_actual_resize.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Compare ACTUAL C++ resized uint8 (from sam3_preprocess_image dump) -against Python torchvision v2.Resize uint8 output.""" -import numpy as np - -# Load C++ resized uint8 [H, W, 3] -cpp = np.fromfile("tests/ref_phase3/cpp_resized_uint8.bin", dtype=np.uint8).reshape(1008, 1008, 3) - -# Load Python torchvision resized uint8 [H, W, 3] -py = np.fromfile("tests/ref_phase3/resized_uint8.bin", dtype=np.uint8).reshape(1008, 1008, 3) - -print("═══ Actual C++ resize vs Python torchvision resize (uint8) ═══") -print(f" C++ shape: {cpp.shape}, range: [{cpp.min()}, {cpp.max()}]") -print(f" Python shape: {py.shape}, range: [{py.min()}, {py.max()}]") - -diff = np.abs(cpp.astype(np.int16) - py.astype(np.int16)) -print(f"\n max pixel diff: {diff.max()}") -print(f" mean pixel diff: {diff.mean():.6f}") -n_diff = (diff > 0).sum() -print(f" pixels differing: {n_diff} / {diff.size} ({100*n_diff/diff.size:.2f}%)") - -for t in [0, 1, 2, 3, 5, 10]: - n = (diff > t).sum() - if n > 0: - print(f" pixels with diff > {t}: {n}") - -# Find worst locations -if diff.max() > 0: - worst_flat = np.argmax(diff.flatten()) - worst_pos = np.unravel_index(worst_flat, diff.shape) - print(f"\n Worst pixel at (y={worst_pos[0]}, x={worst_pos[1]}, c={worst_pos[2]})") - print(f" C++={cpp[worst_pos]}, Python={py[worst_pos]}, diff={diff[worst_pos]}") - - # Show some worst locations - flat_diff = diff.flatten() - worst_indices = np.argsort(flat_diff)[-10:][::-1] - print(f"\n Top 10 worst positions:") - for idx in worst_indices: - pos = np.unravel_index(idx, diff.shape) - print(f" y={pos[0]:4d} x={pos[1]:4d} c={pos[2]} cpp={cpp[pos]:3d} py={py[pos]:3d} diff={diff[pos]:2d}") - -# After fix: compare normalized float outputs -print("\n═══ Normalized float comparison ═══") -cpp_float = (cpp.astype(np.float32) / 255.0 - 0.5) / 0.5 -py_float = (py.astype(np.float32) / 255.0 - 0.5) / 0.5 -float_diff = np.abs(cpp_float - py_float) -print(f" max float diff: {float_diff.max():.6e}") -print(f" mean float diff: {float_diff.mean():.6e}") -cos = np.dot(cpp_float.flatten(), py_float.flatten()) / ( - np.linalg.norm(cpp_float.flatten()) * np.linalg.norm(py_float.flatten())) -print(f" cosine: {cos:.10f}") diff --git a/tests/compare_cpp_dump_vs_finterp.py b/tests/compare_cpp_dump_vs_finterp.py deleted file mode 100644 index e7afdd1..0000000 --- a/tests/compare_cpp_dump_vs_finterp.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Compare the ACTUAL C++ uint8 dump against torch F.interpolate output.""" -import numpy as np -import torch -from PIL import Image -from torchvision.transforms import v2 - -image_path = "tests/test_random.jpg" -img_size = 1008 - -# Load image for torch -img = Image.open(image_path).convert("RGB") -img_tensor = v2.functional.to_image(img) # uint8 CHW [3, 480, 640] - -# ═══ Ground truth: v2.Resize (antialias=True) ═══ -tv_resized = v2.Resize(size=(img_size, img_size))(img_tensor) -tv_hwc = tv_resized.permute(1, 2, 0).numpy() # [H, W, 3] uint8 - -# ═══ F.interpolate (antialias=False) ═══ -float_input = img_tensor.float() -fi_resized = torch.nn.functional.interpolate( - float_input.unsqueeze(0), size=(img_size, img_size), - mode='bilinear', align_corners=False, antialias=False -).squeeze(0) -fi_uint8 = fi_resized.clamp(0, 255).round().to(torch.uint8) -fi_hwc = fi_uint8.permute(1, 2, 0).numpy() - -# ═══ F.interpolate (antialias=True) ═══ -fi_aa_resized = torch.nn.functional.interpolate( - float_input.unsqueeze(0), size=(img_size, img_size), - mode='bilinear', align_corners=False, antialias=True -).squeeze(0) -fi_aa_uint8 = fi_aa_resized.clamp(0, 255).round().to(torch.uint8) -fi_aa_hwc = fi_aa_uint8.permute(1, 2, 0).numpy() - -# ═══ Load ACTUAL C++ dump ═══ -cpp_hwc = np.fromfile("tests/ref_phase3/cpp_resized_uint8.bin", dtype=np.uint8).reshape(img_size, img_size, 3) - -print("═══ Comparisons ═══\n") - -for name, ref in [("v2.Resize", tv_hwc), - ("F.interp(aa=False)", fi_hwc), - ("F.interp(aa=True)", fi_aa_hwc)]: - diff = np.abs(cpp_hwc.astype(np.int16) - ref.astype(np.int16)) - n_diff = (diff > 0).sum() - print(f" C++ vs {name:22s}: max={diff.max():2d} n_diff={n_diff:7d} ({100*n_diff/diff.size:.2f}%)" - f" >1: {(diff>1).sum():6d} >2: {(diff>2).sum():4d}") - -# Also compare the torch methods against each other -for name, ref, comp in [("v2.Resize vs F(aa=F)", tv_hwc, fi_hwc), - ("v2.Resize vs F(aa=T)", tv_hwc, fi_aa_hwc), - ("F(aa=F) vs F(aa=T)", fi_hwc, fi_aa_hwc)]: - diff = np.abs(ref.astype(np.int16) - comp.astype(np.int16)) - n_diff = (diff > 0).sum() - print(f" {name:22s}: max={diff.max():2d} n_diff={n_diff:7d} ({100*n_diff/diff.size:.2f}%)") - -# ═══ Save the F.interpolate(aa=True) + round output for C++ to compare against ═══ -with open("tests/ref_phase3/resized_uint8_finterp_aa.bin", "wb") as f: - f.write(fi_aa_hwc.tobytes()) -with open("tests/ref_phase3/resized_uint8_finterp_aa.shape", "w") as f: - f.write(f"{img_size},{img_size},3") -print(f"\nSaved F.interp(aa=True) uint8 output") diff --git a/tests/compare_ddec_tensors.py b/tests/compare_ddec_tensors.py deleted file mode 100644 index cca04c0..0000000 --- a/tests/compare_ddec_tensors.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 -"""Comprehensive comparison of DETR decoder tensors: Python ref vs C++ output. - -Computes all 6 metrics: mae, max, rel, cos, p95, p99. -Handles layout differences between PyTorch (seq-first/batch-first) and ggml. -""" -import os -import sys -import numpy as np - -REF_DIR = os.path.join(os.path.dirname(__file__), "ref_phase5") -CPP_DIR = os.path.join(REF_DIR, "cpp_out_phase5") - - -def load_tensor(path): - """Load a .bin + .shape tensor pair.""" - shape_file = path + ".shape" - bin_file = path + ".bin" - if not os.path.exists(shape_file) or not os.path.exists(bin_file): - return None, None - with open(shape_file) as f: - shape = tuple(int(x) for x in f.read().strip().split(",") if x) - data = np.fromfile(bin_file, dtype=np.float32) - return data, shape - - -def compare(name, ref_path, cpp_path, atol=1e-4): - """Compare two tensors and return detailed metrics.""" - ref_data, ref_shape = load_tensor(ref_path) - cpp_data, cpp_shape = load_tensor(cpp_path) - - if ref_data is None: - return None, f"SKIP (missing ref: {ref_path})" - if cpp_data is None: - return None, f"SKIP (missing cpp: {cpp_path})" - - if ref_data.size != cpp_data.size: - return None, f"FAIL NUMEL MISMATCH ref={ref_data.size} ({ref_shape}) cpp={cpp_data.size} ({cpp_shape})" - - diff = np.abs(ref_data - cpp_data) - eps = 1e-8 - - mae = float(diff.mean()) - max_d = float(diff.max()) - rel_err = diff / (np.abs(ref_data) + eps) - mean_rel = float(rel_err.mean()) - cos_num = float(np.dot(ref_data, cpp_data)) - cos_den = float(np.linalg.norm(ref_data) * np.linalg.norm(cpp_data) + 1e-12) - cos = cos_num / cos_den - p95 = float(np.percentile(diff, 95)) - p99 = float(np.percentile(diff, 99)) - - worst_idx = int(np.argmax(diff)) - status = "PASS" if max_d < atol else "FAIL" - - metrics = { - "mae": mae, - "max": max_d, - "rel": mean_rel, - "cos": cos, - "p95": p95, - "p99": p99, - "worst_idx": worst_idx, - "worst_ref": float(ref_data[worst_idx]), - "worst_cpp": float(cpp_data[worst_idx]), - "n_over_atol": int(np.sum(diff > atol)), - "n_total": int(ref_data.size), - "ref_shape": ref_shape, - "cpp_shape": cpp_shape, - } - return metrics, status - - -def print_result(name, metrics, status, atol): - if metrics is None: - print(f" {status}") - return - - label = f"{status} {name:45s} ref_shape={metrics['ref_shape']} cpp_shape={metrics['cpp_shape']}" - print(f" {label}") - print(f" mae={metrics['mae']:.6e} max={metrics['max']:.6e} " - f"rel={metrics['rel']:.6e} cos={metrics['cos']:.10f} " - f"p95={metrics['p95']:.6e} p99={metrics['p99']:.6e}") - if status == "FAIL": - print(f" worst_idx={metrics['worst_idx']} " - f"ref={metrics['worst_ref']:.6e} cpp={metrics['worst_cpp']:.6e} " - f"n_over_atol={metrics['n_over_atol']}/{metrics['n_total']}") - - -# Define all decoder tensor cases with their tolerances -DECODER_TENSORS = [ - # Inputs (should be exact or near-exact) - ("ddec_query_embed", 1e-4), - ("ddec_ref_pts_raw", 1e-4), - ("ddec_presence_token", 1e-4), - ("ddec_ref_boxes_init", 1e-4), - - # Layer 0 intermediates - ("ddec_query_sine_0", 1e-4), - ("ddec_query_pos_0", 1e-4), - ("ddec_rpb_mask_0", 2e-4), - ("ddec_layer0_after_sa", 1e-4), - ("ddec_layer0_after_text_ca", 1e-4), - ("ddec_layer0_after_img_ca", 1e-4), - ("ddec_layer0_full_out", 1e-4), - ("ddec_layer0_presence", 1e-4), - - # Per-layer outputs (6 layers) - ("ddec_layer0_out", 1e-4), - ("ddec_layer0_refboxes", 1e-4), - ("ddec_layer1_out", 1e-4), - ("ddec_layer1_refboxes", 1e-4), - ("ddec_layer2_out", 2e-4), - ("ddec_layer2_refboxes", 1e-4), - ("ddec_layer3_out", 2e-4), - ("ddec_layer3_refboxes", 1e-4), - ("ddec_layer4_out", 2e-4), - ("ddec_layer4_refboxes", 1e-4), - ("ddec_layer5_out", 2e-4), - ("ddec_layer5_refboxes", 1e-4), - - # Final outputs - ("ddec_normed_output", 2e-4), - ("ddec_pred_boxes", 1e-4), - ("ddec_presence_logit", 1e-4), - - # Scoring - ("scoring_prompt_mlp_out", 1e-4), - ("scoring_pooled", 1e-4), - ("scoring_proj_pooled", 1e-4), - ("scoring_proj_hs", 1e-4), - ("scoring_class_scores", 1e-4), -] - - -def main(): - ref_dir = sys.argv[1] if len(sys.argv) > 1 else REF_DIR - cpp_dir = sys.argv[2] if len(sys.argv) > 2 else CPP_DIR - - print(f"Reference dir: {ref_dir}") - print(f"C++ output dir: {cpp_dir}") - print() - - n_pass = 0 - n_fail = 0 - n_skip = 0 - - # ═══ Section 1: Decoder Inputs ═══ - print("═══ Decoder Inputs (should be exact match) ═══") - for name, atol in DECODER_TENSORS[:4]: - metrics, status = compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol) - print_result(name, metrics, status, atol) - if "PASS" in status: - n_pass += 1 - elif "FAIL" in status: - n_fail += 1 - else: - n_skip += 1 - - # ═══ Section 2: Decoder Layer 0 Intermediates ═══ - print("\n═══ Decoder Layer 0 Intermediates ═══") - for name, atol in DECODER_TENSORS[4:12]: - metrics, status = compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol) - print_result(name, metrics, status, atol) - if "PASS" in status: - n_pass += 1 - elif "FAIL" in status: - n_fail += 1 - else: - n_skip += 1 - - # ═══ Section 3: Per-Layer Outputs ═══ - print("\n═══ Decoder Per-Layer Outputs (6 layers) ═══") - for name, atol in DECODER_TENSORS[12:24]: - metrics, status = compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol) - print_result(name, metrics, status, atol) - if "PASS" in status: - n_pass += 1 - elif "FAIL" in status: - n_fail += 1 - else: - n_skip += 1 - - # ═══ Section 4: Final Decoder Outputs ═══ - print("\n═══ Final Decoder Outputs ═══") - for name, atol in DECODER_TENSORS[24:27]: - metrics, status = compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol) - print_result(name, metrics, status, atol) - if "PASS" in status: - n_pass += 1 - elif "FAIL" in status: - n_fail += 1 - else: - n_skip += 1 - - # ═══ Section 5: Scoring ═══ - print("\n═══ DotProductScoring ═══") - for name, atol in DECODER_TENSORS[27:]: - metrics, status = compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol) - print_result(name, metrics, status, atol) - if "PASS" in status: - n_pass += 1 - elif "FAIL" in status: - n_fail += 1 - else: - n_skip += 1 - - # ═══ Summary ═══ - print(f"\n{'='*70}") - total = n_pass + n_fail - print(f"SUMMARY: {n_pass}/{total} PASS, {n_fail}/{total} FAIL, {n_skip} SKIP") - if n_fail == 0 and n_skip == 0: - print("ALL DECODER TENSORS PASS ✓") - return 0 if n_fail == 0 else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/compare_f32_vs_f16.py b/tests/compare_f32_vs_f16.py deleted file mode 100644 index 684e932..0000000 --- a/tests/compare_f32_vs_f16.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -"""Side-by-side comparison: F32 vs F16 mask decoder precision against Python reference.""" -import numpy as np -import os - -REF_BASE = "tests/ref_phase6" -F32_BASE = "tests/ref_phase6/cpp_out_phase6" -F16_BASE = "tests/ref_phase6/cpp_out_phase6_f16/cpp_out_phase6" - -CASES = ["point_single", "box_only", "box_and_points"] - -TENSORS = [ - "sam_pe_sparse", "sam_pe_dense", "sam_pe_image_pe", - "sam_dec_image_feats", "sam_dec_tokens_initial", - "sam_dec_block0_queries", "sam_dec_block0_keys", - "sam_dec_block1_queries", "sam_dec_block1_keys", - "sam_dec_final_queries", - "sam_dec_feat_s1_proj", "sam_dec_feat_s0_proj", - "sam_dec_upscaled", "sam_dec_mask_tokens", - "sam_dec_masks", "sam_dec_iou", "sam_dec_obj_score", "sam_dec_sam_token", -] - -def load(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - return np.fromfile(path + ".bin", dtype=np.float32), shape - -def metrics(ref, cpp): - diff = np.abs(ref - cpp) - eps = 1e-8 - return { - "mae": float(diff.mean()), - "max": float(diff.max()), - "rel": float((diff / (np.abs(ref) + eps)).mean()), - "cos": float(np.dot(ref, cpp) / (np.linalg.norm(ref) * np.linalg.norm(cpp) + 1e-12)), - "p95": float(np.percentile(diff, 95)), - "p99": float(np.percentile(diff, 99)), - } - -# Aggregate across all cases -all_f32 = {} -all_f16 = {} - -for case in CASES: - for t in TENSORS: - ref_path = os.path.join(REF_BASE, case, t) - f32_path = os.path.join(F32_BASE, case, t) - f16_path = os.path.join(F16_BASE, case, t) - if not all(os.path.exists(p + ".bin") for p in [ref_path, f32_path, f16_path]): - continue - ref_data, _ = load(ref_path) - f32_data, _ = load(f32_path) - f16_data, _ = load(f16_path) - if ref_data.size != f32_data.size or ref_data.size != f16_data.size: - continue - key = (case, t) - all_f32[key] = metrics(ref_data, f32_data) - all_f16[key] = metrics(ref_data, f16_data) - -# Print per-tensor summary (worst across cases) -print(f"{'Tensor':40s} | {'F32 max':>10s} {'F32 mae':>10s} {'F32 cos':>12s} | {'F16 max':>10s} {'F16 mae':>10s} {'F16 cos':>12s} | {'Ratio max':>10s}") -print("-" * 140) - -for t in TENSORS: - f32_worst_max = 0 - f32_worst_mae = 0 - f32_worst_cos = 1.0 - f16_worst_max = 0 - f16_worst_mae = 0 - f16_worst_cos = 1.0 - found = False - for case in CASES: - key = (case, t) - if key in all_f32: - found = True - f32_worst_max = max(f32_worst_max, all_f32[key]["max"]) - f32_worst_mae = max(f32_worst_mae, all_f32[key]["mae"]) - f32_worst_cos = min(f32_worst_cos, all_f32[key]["cos"]) - f16_worst_max = max(f16_worst_max, all_f16[key]["max"]) - f16_worst_mae = max(f16_worst_mae, all_f16[key]["mae"]) - f16_worst_cos = min(f16_worst_cos, all_f16[key]["cos"]) - if not found: - continue - ratio = f16_worst_max / f32_worst_max if f32_worst_max > 0 else float('inf') - print(f"{t:40s} | {f32_worst_max:10.2e} {f32_worst_mae:10.2e} {f32_worst_cos:12.8f} | {f16_worst_max:10.2e} {f16_worst_mae:10.2e} {f16_worst_cos:12.8f} | {ratio:10.1f}x") - -# Print per-case detail for key tensors -print(f"\n{'='*120}") -print("Detailed per-case metrics for key decoder tensors") -print(f"{'='*120}") -key_tensors = ["sam_dec_block0_queries", "sam_dec_block1_keys", "sam_dec_final_queries", - "sam_dec_upscaled", "sam_dec_masks", "sam_dec_iou", "sam_dec_obj_score"] - -for case in CASES: - print(f"\n Case: {case}") - print(f" {'Tensor':34s} | {'':5s} {'mae':>10s} {'max':>10s} {'rel':>10s} {'cos':>12s} {'p95':>10s} {'p99':>10s}") - print(f" {'-'*105}") - for t in key_tensors: - key = (case, t) - if key not in all_f32: - continue - f32 = all_f32[key] - f16 = all_f16[key] - print(f" {t:34s} | {'f32':5s} {f32['mae']:10.2e} {f32['max']:10.2e} {f32['rel']:10.2e} {f32['cos']:12.8f} {f32['p95']:10.2e} {f32['p99']:10.2e}") - print(f" {'':34s} | {'f16':5s} {f16['mae']:10.2e} {f16['max']:10.2e} {f16['rel']:10.2e} {f16['cos']:12.8f} {f16['p95']:10.2e} {f16['p99']:10.2e}") diff --git a/tests/compare_mask_decoder.py b/tests/compare_mask_decoder.py deleted file mode 100644 index d5f89ac..0000000 --- a/tests/compare_mask_decoder.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 -""" -Comprehensive comparison of SAM3 mask decoder tensors: Python reference vs C++ output. -Reports mae, max, rel, cos, p95, p99 for every tensor in every test case. -""" -import os -import sys -import numpy as np - -import argparse -_parser = argparse.ArgumentParser() -_parser.add_argument("--cpp-base", default="tests/ref_phase6/cpp_out_phase6") -_parser.add_argument("--ref-base", default="tests/ref_phase6") -_args = _parser.parse_args() - -REF_BASE = _args.ref_base -CPP_BASE = _args.cpp_base - -CASES = ["point_single", "box_only", "box_and_points"] - -TENSORS = [ - "sam_pe_sparse", - "sam_pe_dense", - "sam_pe_image_pe", - "sam_dec_image_feats", - "sam_dec_tokens_initial", - "sam_dec_block0_queries", - "sam_dec_block0_keys", - "sam_dec_block1_queries", - "sam_dec_block1_keys", - "sam_dec_final_queries", - "sam_dec_feat_s1_proj", - "sam_dec_feat_s0_proj", - "sam_dec_upscaled", - "sam_dec_mask_tokens", - "sam_dec_masks", - "sam_dec_iou", - "sam_dec_obj_score", - "sam_dec_sam_token", -] - - -def load_tensor(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32) - return data, shape - - -def compare(name, ref_path, cpp_path): - ref_data, ref_shape = load_tensor(ref_path) - cpp_data, cpp_shape = load_tensor(cpp_path) - - if ref_data.size != cpp_data.size: - return { - "status": "FAIL", - "note": f"SIZE MISMATCH ref={ref_data.size} cpp={cpp_data.size}", - "ref_shape": ref_shape, - "cpp_shape": cpp_shape, - } - - diff = np.abs(ref_data - cpp_data) - eps = 1e-8 - max_d = float(diff.max()) - mae = float(diff.mean()) - rel_err = diff / (np.abs(ref_data) + eps) - mean_rel = float(rel_err.mean()) - cos = float( - np.dot(ref_data, cpp_data) - / (np.linalg.norm(ref_data) * np.linalg.norm(cpp_data) + 1e-12) - ) - p95 = float(np.percentile(diff, 95)) - p99 = float(np.percentile(diff, 99)) - - worst_idx = int(np.argmax(diff)) - worst_ref = float(ref_data[worst_idx]) - worst_cpp = float(cpp_data[worst_idx]) - - return { - "status": "OK", - "ref_shape": ref_shape, - "cpp_shape": cpp_shape, - "mae": mae, - "max": max_d, - "rel": mean_rel, - "cos": cos, - "p95": p95, - "p99": p99, - "worst_idx": worst_idx, - "worst_ref": worst_ref, - "worst_cpp": worst_cpp, - "n_elements": ref_data.size, - } - - -def main(): - all_pass = True - - for case in CASES: - ref_dir = os.path.join(REF_BASE, case) - cpp_dir = os.path.join(CPP_BASE, case) - - print(f"\n{'='*80}") - print(f" Case: {case}") - print(f"{'='*80}") - - if not os.path.isdir(ref_dir): - print(f" [SKIP] Reference dir missing: {ref_dir}") - continue - if not os.path.isdir(cpp_dir): - print(f" [SKIP] C++ output dir missing: {cpp_dir}") - continue - - for tensor_name in TENSORS: - ref_path = os.path.join(ref_dir, tensor_name) - cpp_path = os.path.join(cpp_dir, tensor_name) - - if not os.path.exists(ref_path + ".bin"): - print(f" [SKIP] {tensor_name:40s} missing ref") - continue - if not os.path.exists(cpp_path + ".bin"): - print(f" [SKIP] {tensor_name:40s} missing cpp") - continue - - r = compare(tensor_name, ref_path, cpp_path) - - if r["status"] != "OK": - print(f" [FAIL] {tensor_name:40s} {r['note']}") - all_pass = False - continue - - # Determine pass/fail thresholds - is_fail = False - if r["cos"] < 0.999: - is_fail = True - if r["max"] > 0.1: - is_fail = True - - status = "FAIL" if is_fail else "PASS" - if is_fail: - all_pass = False - - print( - f" [{status}] {tensor_name:40s} shape={r['ref_shape']}" - ) - print( - f" mae={r['mae']:.6e} max={r['max']:.6e} " - f"rel={r['rel']:.6e} cos={r['cos']:.8f} " - f"p95={r['p95']:.6e} p99={r['p99']:.6e}" - ) - if is_fail: - print( - f" worst_idx={r['worst_idx']} " - f"ref={r['worst_ref']:.6e} cpp={r['worst_cpp']:.6e} " - f"n_elements={r['n_elements']}" - ) - - print(f"\n{'='*80}") - print(f" Overall: {'ALL PASS' if all_pass else 'FAILURES DETECTED'}") - print(f"{'='*80}") - return 0 if all_pass else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/compare_preprocessed_float.py b/tests/compare_preprocessed_float.py deleted file mode 100644 index 3268f51..0000000 --- a/tests/compare_preprocessed_float.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -"""Compare C++ preprocessed float CHW against Python preprocessed float CHW.""" -import numpy as np - -ref_dir = "tests/ref_phase3" - -# Load Python reference: [1, 3, 1008, 1008] NCHW -py = np.fromfile(f"{ref_dir}/preprocessed.bin", dtype=np.float32) -py_shape = [int(x) for x in open(f"{ref_dir}/preprocessed.shape").read().strip().split(",")] -py = py.reshape(py_shape) -print(f"Python preprocessed: shape={py.shape}") - -# Load C++ preprocessed: [1, 3, 1008, 1008] NCHW -cpp = np.fromfile(f"{ref_dir}/cpp_preprocessed.bin", dtype=np.float32) -cpp_shape = [int(x) for x in open(f"{ref_dir}/cpp_preprocessed.shape").read().strip().split(",")] -cpp = cpp.reshape(cpp_shape) -print(f"C++ preprocessed: shape={cpp.shape}") - -# Compare -diff = np.abs(py - cpp) -print(f"\n═══ Preprocessed Float Comparison ═══") -print(f" max_diff: {diff.max():.6e}") -print(f" mean_diff: {diff.mean():.6e}") -cos = np.dot(py.flatten(), cpp.flatten()) / (np.linalg.norm(py.flatten()) * np.linalg.norm(cpp.flatten())) -print(f" cosine: {cos:.10f}") -n_diff = (diff > 0).sum() -print(f" elements differing (>0): {n_diff} / {diff.size} ({100*n_diff/diff.size:.2f}%)") - -# Since normalization is (x/255 - 0.5)/0.5, a 1 uint8 diff = 1/255 * 2 = 0.00784 -expected_1px = 1.0 / 255.0 / 0.5 # 0.00784 -print(f"\n Expected diff for 1 uint8 pixel: {expected_1px:.6f}") -n_1px = (diff > expected_1px * 0.5).sum() -print(f" Elements with diff > 0.5 pixel: {n_1px} ({100*n_1px/diff.size:.2f}%)") - -# Look at where biggest differences are -worst = np.unravel_index(np.argmax(diff), diff.shape) -print(f"\n Worst at {worst}: py={py[worst]:.6f}, cpp={cpp[worst]:.6f}, diff={diff[worst]:.6f}") -print(f" That's {diff[worst] / expected_1px:.2f} uint8 pixel values") - -# Compare uint8 values (undo normalization) -py_uint8 = np.round((py * 0.5 + 0.5) * 255).astype(np.uint8) -cpp_uint8 = np.round((cpp * 0.5 + 0.5) * 255).astype(np.uint8) -uint8_diff = np.abs(py_uint8.astype(np.int16) - cpp_uint8.astype(np.int16)) -print(f"\n Recovered uint8 max pixel diff: {uint8_diff.max()}") -print(f" Recovered uint8 n_diff: {(uint8_diff > 0).sum()}") diff --git a/tests/compare_preprocessing.py b/tests/compare_preprocessing.py deleted file mode 100644 index b471280..0000000 --- a/tests/compare_preprocessing.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -"""Compare C++ vs Python image preprocessing. - -Replicates the C++ sam3_preprocess_image() in Python and compares -against the torchvision pipeline used by SAM3 Python. -""" -import numpy as np -import sys -from PIL import Image -import torch -from torchvision.transforms import v2 - - -def cpp_preprocess(image_path, img_size=1008): - """Replicate C++ sam3_preprocess_image exactly. - - C++ does: - 1. Bilinear resize to img_size x img_size (half-pixel center mapping) - 2. Quantize back to uint8 with rounding - 3. Normalize: (pixel/255.0 - 0.5) / 0.5 - 4. Layout: CHW - """ - img = Image.open(image_path).convert("RGB") - src = np.array(img, dtype=np.float32) # [H, W, 3] - src_h, src_w = src.shape[:2] - - # C++ bilinear resize (half-pixel center mapping) - dst = np.zeros((img_size, img_size, 3), dtype=np.uint8) - sx = src_w / img_size - sy = src_h / img_size - - for y in range(img_size): - fy = (y + 0.5) * sy - 0.5 - y0 = max(0, int(fy)) - y1 = min(src_h - 1, y0 + 1) - wy = fy - y0 - for x in range(img_size): - fx = (x + 0.5) * sx - 0.5 - x0 = max(0, int(fx)) - x1 = min(src_w - 1, x0 + 1) - wx = fx - x0 - for c in range(3): - v = ((1 - wy) * ((1 - wx) * src[y0, x0, c] + wx * src[y0, x1, c]) + - wy * ((1 - wx) * src[y1, x0, c] + wx * src[y1, x1, c])) - dst[y, x, c] = min(255, max(0, int(v + 0.5))) - - # Normalize and convert to CHW - result = np.zeros((3, img_size, img_size), dtype=np.float32) - for c in range(3): - for y in range(img_size): - for x in range(img_size): - v = dst[y, x, c] / 255.0 - result[c, y, x] = (v - 0.5) / 0.5 - - return result, dst - - -def python_preprocess(image_path, img_size=1008): - """SAM3 Python preprocessing pipeline.""" - img = Image.open(image_path).convert("RGB") - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(img_size, img_size)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - img_tensor = v2.functional.to_image(img) - return transform(img_tensor).numpy() # [3, H, W] - - -if __name__ == "__main__": - image_path = sys.argv[1] if len(sys.argv) > 1 else "tests/test_random.jpg" - - print(f"Comparing preprocessing for: {image_path}") - print() - - # Python preprocessing (torchvision) - py = python_preprocess(image_path) - print(f"Python shape: {py.shape}") - print(f" min={py.min():.6f} max={py.max():.6f} mean={py.mean():.6f}") - - # C++ preprocessing (replicated) - print("\nReplicating C++ preprocessing (this is slow in Python)...") - cpp, cpp_uint8 = cpp_preprocess(image_path) - print(f"C++ shape: {cpp.shape}") - print(f" min={cpp.min():.6f} max={cpp.max():.6f} mean={cpp.mean():.6f}") - - # Compare - diff = np.abs(py - cpp) - print(f"\n═══ Comparison ═══") - print(f" max_diff: {diff.max():.6e}") - print(f" mean_diff: {diff.mean():.6e}") - cos = np.dot(py.flatten(), cpp.flatten()) / (np.linalg.norm(py.flatten()) * np.linalg.norm(cpp.flatten())) - print(f" cosine: {cos:.10f}") - - # Pixel-level differences - n_diff = (diff > 1e-6).sum() - print(f" pixels differing: {n_diff} / {diff.size} ({100*n_diff/diff.size:.2f}%)") - - n_diff_1pct = (diff > 0.01).sum() - print(f" pixels with |err| > 0.01: {n_diff_1pct}") - - # Sample some differences - if diff.max() > 1e-6: - worst = np.unravel_index(np.argmax(diff), diff.shape) - print(f"\n Worst pixel at {worst}: py={py[worst]:.6f}, cpp={cpp[worst]:.6f}") - - # Check if the difference is due to resize or normalization - # PIL resize - img = Image.open(image_path).convert("RGB") - pil_resized = img.resize((1008, 1008), Image.BILINEAR) - pil_arr = np.array(pil_resized, dtype=np.uint8) - - # Compare uint8 resize outputs - uint8_diff = np.abs(pil_arr.astype(np.int16) - cpp_uint8.astype(np.int16)) - print(f"\n Resize comparison (uint8):") - print(f" max pixel diff: {uint8_diff.max()}") - print(f" pixels differing: {(uint8_diff > 0).sum()} / {uint8_diff.size}") diff --git a/tests/compare_resize.py b/tests/compare_resize.py deleted file mode 100644 index 24afd52..0000000 --- a/tests/compare_resize.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -"""Compare C++ bilinear resize against Python torchvision resize at uint8 level. - -Replicates the exact C++ sam3_resize_bilinear() in Python, then compares -against torchvision v2.Resize output. -""" -import numpy as np -from PIL import Image -import torch -from torchvision.transforms import v2 - -image_path = "tests/test_random.jpg" -img_size = 1008 - -# Load original image -img = Image.open(image_path).convert("RGB") -src = np.array(img, dtype=np.float32) # [H, W, 3] -src_h, src_w = src.shape[:2] -print(f"Original: {src_w}x{src_h}") - -# ═══ Method 1: C++ bilinear resize (replicated in Python) ═══ -print("\nReplicating C++ resize...") -dst_cpp = np.zeros((img_size, img_size, 3), dtype=np.uint8) -sx = src_w / img_size -sy = src_h / img_size - -for y in range(img_size): - fy = (y + 0.5) * sy - 0.5 - y0 = max(0, int(fy)) - y1 = min(src_h - 1, y0 + 1) - wy = np.float32(fy - y0) # Use f32 to match C++ - for x in range(img_size): - fx = (x + 0.5) * sx - 0.5 - x0 = max(0, int(fx)) - x1 = min(src_w - 1, x0 + 1) - wx = np.float32(fx - x0) - for c in range(3): - v = np.float32((1 - wy) * ((1 - wx) * src[y0, x0, c] + wx * src[y0, x1, c]) + - wy * ((1 - wx) * src[y1, x0, c] + wx * src[y1, x1, c])) - dst_cpp[y, x, c] = min(255, max(0, int(v + 0.5))) - -# ═══ Method 2: Python torchvision resize ═══ -img_tensor = v2.functional.to_image(img) # uint8 CHW tensor -step1 = v2.ToDtype(torch.uint8, scale=True)(img_tensor) -step2 = v2.Resize(size=(img_size, img_size))(step1) -dst_py = step2.permute(1, 2, 0).numpy() # [H, W, 3] uint8 - -# ═══ Compare ═══ -print("\n═══ C++ resize vs Python torchvision resize (uint8) ═══") -diff = np.abs(dst_cpp.astype(np.int16) - dst_py.astype(np.int16)) -print(f" max pixel diff: {diff.max()}") -print(f" mean pixel diff: {diff.mean():.6f}") -n_diff = (diff > 0).sum() -print(f" pixels differing: {n_diff} / {diff.size} ({100*n_diff/diff.size:.2f}%)") - -# Histogram of differences -for threshold in [0, 1, 2, 3]: - n = (diff > threshold).sum() - print(f" pixels with diff > {threshold}: {n}") - -# ═══ Check: does the float normalization match? ═══ -print("\n═══ Float normalization check ═══") - -# C++ normalization: (pixel / 255.0 - 0.5) / 0.5 -cpp_float = (dst_cpp.astype(np.float32) / 255.0 - 0.5) / 0.5 # [H, W, 3] -cpp_chw = cpp_float.transpose(2, 0, 1) # [3, H, W] - -# Python: v2.ToDtype(float32, scale=True) then Normalize(0.5, 0.5) -step3 = v2.ToDtype(torch.float32, scale=True)(step2) # scale=True divides by 255 -step4 = v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])(step3) -py_chw = step4.numpy() - -# If the uint8 images match, the float should also match -float_diff = np.abs(cpp_chw - py_chw) -print(f" max float diff: {float_diff.max():.6e}") -print(f" mean float diff: {float_diff.mean():.6e}") - -# ═══ Check: what if we use the SAME uint8 but different normalization? ═══ -# Python's ToDtype(float32, scale=True) does pixel / 255.0 -# C++ does pixel / 255.0 -# Then Python's Normalize does (x - 0.5) / 0.5 -# C++ does (v - 0.5) / 0.5 -# These should be identical -print("\n═══ Normalization formula check ═══") -# Take a single pixel value and check -test_val = np.uint8(128) -cpp_result = np.float32((np.float32(test_val) / np.float32(255.0) - np.float32(0.5)) / np.float32(0.5)) -py_result = np.float32((np.float32(test_val) / np.float32(255.0) - np.float32(0.5)) / np.float32(0.5)) -print(f" test pixel=128: cpp={cpp_result:.10f}, py={py_result:.10f}, diff={abs(cpp_result-py_result):.10e}") - -# Conclusion -print("\n═══ Conclusion ═══") -if diff.max() <= 1: - print(" C++ and Python resizes differ by at most 1 uint8 value.") - print(" This is a bilinear rounding difference, not a bug.") - print(" To make them identical, the C++ resize must replicate") - print(" the exact torchvision/PIL rounding behavior.") -else: - print(f" WARNING: C++ and Python resizes differ by up to {diff.max()} uint8 values!") - print(" This suggests an algorithmic difference, not just rounding.") diff --git a/tests/compare_sam2_video.py b/tests/compare_sam2_video.py deleted file mode 100644 index 89612a8..0000000 --- a/tests/compare_sam2_video.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -""" -Compare C++ vs Python SAM2 video tracking outputs. - -Compares: -1. Backbone features (neck_trk_2) — are images encoded the same? -2. Output masks — do they segment the same region? -3. Memory-conditioned features — is memory attention working? -""" -import numpy as np -import os, sys -from PIL import Image - -py_dir = "/tmp/debug_sam2_pipeline" -cpp_dir = "/tmp/debug_sam2_cpp" - -def load_tensor(path): - data = np.fromfile(path + ".bin", dtype=np.float32) - if os.path.exists(path + ".shape"): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - return data.reshape(shape) if shape else data - return data - -def compare(name, ref, cpp, atol=1e-3): - ref_f = ref.flatten() - cpp_f = cpp.flatten() - if ref_f.size != cpp_f.size: - print(f" FAIL {name:50s} SIZE MISMATCH ref={ref_f.size} cpp={cpp_f.size}") - return False - diff = np.abs(ref_f - cpp_f) - max_d = diff.max() - mean_d = diff.mean() - cos = np.dot(ref_f, cpp_f) / (np.linalg.norm(ref_f) * np.linalg.norm(cpp_f) + 1e-12) - p95 = np.percentile(diff, 95) - p99 = np.percentile(diff, 99) - status = "PASS" if cos > 0.99 else "FAIL" - print(f" {status} {name:50s}") - print(f" mae={mean_d:.6e} max={max_d:.6e} cos={cos:.8f} p95={p95:.6e} p99={p99:.6e}") - return status == "PASS" - -def mask_iou(m1, m2): - """Compute IoU between two binary masks.""" - intersection = np.logical_and(m1, m2).sum() - union = np.logical_or(m1, m2).sum() - return intersection / max(union, 1) - -print("=" * 80) -print("SAM2 Video Tracking: Python vs C++ Comparison") -print("=" * 80) - -# Compare vision features (backbone output) -# Python vision_feat_0 is [4096, 1, 256] (HW, B, C) = flattened (64, 64, 256) -# C++ neck_trk_2 is [256, 64, 64] in ggml -print("\n--- Backbone Features (neck_trk_2 / vision_feat_0) ---") -for fi in range(6): - py_path = os.path.join(py_dir, f"f{fi}_vision_feat_0") - cpp_path = os.path.join(cpp_dir, f"f{fi}_neck_trk_2") - if not os.path.exists(py_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - print(f" Frame {fi}: SKIP (missing tensors)") - continue - - py_data = load_tensor(py_path) # [4096, 1, 256] - cpp_data = load_tensor(cpp_path) # [256, 64, 64] in ggml - - print(f" Frame {fi}: py={py_data.shape} ({py_data.size} floats), cpp={cpp_data.shape} ({cpp_data.size} floats)") - - # The Python tensor is [HW, B, C] = [4096, 1, 256] - # The C++ tensor is ggml [D, H, W] = [256, 64, 64] - # Both have 256*64*64 = 1048576 floats - - # Flatten and compare (cosine similarity should be near 1 if they match) - compare(f"frame{fi}_backbone", py_data, cpp_data) - -# Compare output masks -print("\n--- Output Masks ---") -for fi in range(1, 6): - py_mask_path = os.path.join(py_dir, f"f{fi}_output_mask") - cpp_mask_path = os.path.join(cpp_dir, f"f{fi}_output_mask_binary") - - if not os.path.exists(py_mask_path + ".bin") or not os.path.exists(cpp_mask_path + ".bin"): - print(f" Frame {fi}: SKIP") - continue - - py_mask = load_tensor(py_mask_path) # [1, 540, 960] logits - cpp_mask = load_tensor(cpp_mask_path) # [540, 960] binary - - py_binary = (py_mask.flatten() > 0.0).astype(float) - cpp_binary = cpp_mask.flatten() - - iou = mask_iou(py_binary > 0.5, cpp_binary > 0.5) - py_fg = (py_binary > 0.5).sum() - cpp_fg = (cpp_binary > 0.5).sum() - - status = "PASS" if iou > 0.5 else "FAIL" - print(f" {status} frame{fi}_mask IoU={iou:.4f} py_fg={int(py_fg)} cpp_fg={int(cpp_fg)}") - -# Compare memory-conditioned features -print("\n--- Memory-Conditioned Features ---") -for fi in range(6): - py_path = os.path.join(py_dir, f"f{fi}_mem_conditioned") - if not os.path.exists(py_path + ".bin"): - continue - py_data = load_tensor(py_path) # [1, 256, 64, 64] - print(f" Frame {fi}: py_mem_conditioned shape={py_data.shape} " - f"range=[{py_data.min():.4f}, {py_data.max():.4f}]") - -# Compare SAM decoder outputs -print("\n--- SAM Decoder Outputs ---") -for fi in range(6): - for name in ["sam_pred_masks", "sam_ious", "sam_obj_ptr", "sam_obj_score"]: - path = os.path.join(py_dir, f"f{fi}_{name}") - if os.path.exists(path + ".bin"): - data = load_tensor(path) - print(f" Frame {fi} {name:20s}: shape={data.shape} " - f"range=[{data.min():.4f}, {data.max():.4f}]") - -# Print summary -print("\n--- Summary ---") -print("Python ref: 2000-2200 fg pixels/frame, IoU scores 0.88-0.90") -print("If C++ masks have similar fg counts but different locations, memory attention is wrong.") -print("If C++ masks are reasonable, the issue may be in longer sequences (memory drift).") diff --git a/tests/compare_tensors.py b/tests/compare_tensors.py deleted file mode 100644 index c3fc608..0000000 --- a/tests/compare_tensors.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""Compare tensors from Python reference vs C++ output.""" -import numpy as np -import sys, os - -def load_tensor(path): - data = np.fromfile(path + ".bin", dtype=np.float32) - shape_file = path + ".shape" - if os.path.exists(shape_file): - with open(shape_file) as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - return data.reshape(shape) if shape else data - return data - -def compare(name, ref_path, cpp_path, atol=1e-4): - ref = load_tensor(ref_path) if isinstance(ref_path, str) else ref_path - cpp = load_tensor(cpp_path) if isinstance(cpp_path, str) else cpp_path - ref_f = ref.flatten() - cpp_f = cpp.flatten() - if ref_f.size != cpp_f.size: - print(f" FAIL {name:40s} SIZE MISMATCH ref={ref_f.size} cpp={cpp_f.size}") - return False - diff = np.abs(ref_f - cpp_f) - eps = 1e-8 - max_d = diff.max() - mean_d = diff.mean() - rel_err = (diff / (np.abs(ref_f) + eps)).mean() - cos = np.dot(ref_f, cpp_f) / (np.linalg.norm(ref_f) * np.linalg.norm(cpp_f) + 1e-12) - p95 = np.percentile(diff, 95) - p99 = np.percentile(diff, 99) - status = "PASS" if max_d < atol else "FAIL" - print(f" {status} {name:40s} shape={ref.shape}") - print(f" mae={mean_d:.6e} max={max_d:.6e} rel={rel_err:.6e} cos={cos:.8f} p95={p95:.6e} p99={p99:.6e}") - if status == "FAIL": - worst = np.argmax(diff) - print(f" worst idx={worst} ref={ref_f[worst]:.6e} cpp={cpp_f[worst]:.6e}") - return status == "PASS" - -if __name__ == "__main__": - ref_dir = sys.argv[1] # Python dump dir - cpp_dir = sys.argv[2] # C++ dump dir - atol = float(sys.argv[3]) if len(sys.argv) > 3 else 1e-4 - ref_tensors = {f[:-4] for f in os.listdir(ref_dir) if f.endswith(".bin")} - cpp_tensors = {f[:-4] for f in os.listdir(cpp_dir) if f.endswith(".bin")} - common = sorted(ref_tensors & cpp_tensors) - if not common: - print("No common tensors found!") - ref_only = sorted(ref_tensors - cpp_tensors) - cpp_only = sorted(cpp_tensors - ref_tensors) - if ref_only: print(f" Ref only: {ref_only[:10]}") - if cpp_only: print(f" Cpp only: {cpp_only[:10]}") - sys.exit(1) - n_pass = 0 - for name in common: - if compare(name, os.path.join(ref_dir, name), os.path.join(cpp_dir, name), atol): - n_pass += 1 - print(f"\n{n_pass}/{len(common)} passed (atol={atol})") diff --git a/tests/compare_vit_output.py b/tests/compare_vit_output.py deleted file mode 100644 index d08bef7..0000000 --- a/tests/compare_vit_output.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -"""Compare C++ and Python ViT outputs in detail to find outliers. - -The C++ dump stores data in ggml column-major order with shape ne[0], ne[1], ne[2]. -For a tensor with ne=[E, W, H], element (e, w, h) is at flat index e + w*E + h*E*W. -This is the REVERSE of numpy's row-major convention. -""" -import numpy as np -import sys -import os - -def load_ggml_tensor(path): - """Load a tensor dumped by sam3_dump_state_tensor. - Returns data in ggml flat order and the ggml shape (ne[0], ne[1], ne[2], ...). - """ - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",")] - with open(path + ".bin", "rb") as f: - data = np.frombuffer(f.read(), dtype=np.float32).copy() - return data, shape - -def ggml_to_pytorch_bchw(data, ggml_shape): - """Convert ggml [E, W, H] flat data to PyTorch [1, C, H, W] array. - ggml element (e, w, h) at flat index e + w*E + h*E*W - maps to PyTorch [0, e, h, w]. - """ - E, W, H = ggml_shape[:3] - result = np.zeros((1, E, H, W), dtype=np.float32) - for h in range(H): - for w in range(W): - for e in range(E): - result[0, e, h, w] = data[e + w*E + h*E*W] - return result - -def load_pytorch_tensor(path): - """Load a tensor saved by the Python dumper (standard row-major).""" - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",")] - with open(path + ".bin", "rb") as f: - data = np.frombuffer(f.read(), dtype=np.float32).copy() - return data.reshape(shape) - -ref_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/ref_phase3" -cpp_dir = sys.argv[2] if len(sys.argv) > 2 else "tests/ref_phase3/cpp_out_phase3" - -# Load Python reference (BCHW) -py_vit = load_pytorch_tensor(f"{ref_dir}/vit_output_bchw") # [1, 1024, 72, 72] -print(f"Python ViT output: shape={py_vit.shape}, range=[{py_vit.min():.4f}, {py_vit.max():.4f}]") - -# Load C++ output (ggml: ne=[E=1024, W=72, H=72]) -cpp_data, cpp_shape = load_ggml_tensor(f"{cpp_dir}/vit_output") -print(f"C++ ViT output: ggml_shape={cpp_shape}, range=[{cpp_data.min():.4f}, {cpp_data.max():.4f}]") - -# Transpose properly -E, W, H = cpp_shape[:3] -print(f" Transposing ggml [E={E}, W={W}, H={H}] to PyTorch [1, {E}, {H}, {W}]...") -cpp_transposed = ggml_to_pytorch_bchw(cpp_data, cpp_shape) - -diff = np.abs(cpp_transposed - py_vit) -print(f"\nDifference stats:") -print(f" max_diff: {diff.max():.6f}") -print(f" mean_diff: {diff.mean():.6f}") -print(f" median: {np.median(diff):.6f}") -print(f" 99.9th %: {np.percentile(diff, 99.9):.6f}") -print(f" 99.99th %: {np.percentile(diff, 99.99):.6f}") - -# Find the worst elements -flat_diff = diff.flatten() -worst_indices = np.argsort(flat_diff)[-20:][::-1] -print(f"\nTop 20 worst mismatches:") -for idx in worst_indices: - b, c, h, w = np.unravel_index(idx, diff.shape) - py_val = py_vit[b, c, h, w] - cpp_val = cpp_transposed[b, c, h, w] - d = flat_diff[idx] - rel = d / max(abs(py_val), 1e-8) - print(f" [{b},{c:4d},{h:2d},{w:2d}] py={py_val:10.4f} cpp={cpp_val:10.4f} diff={d:.4f} rel={rel:.4f}") - -# Per-channel analysis -chan_max = diff[0].max(axis=(1, 2)) # [1024] -print(f"\nPer-channel max diff distribution:") -for threshold in [0.1, 0.5, 1.0, 2.0, 5.0, 10.0]: - n = (chan_max > threshold).sum() - print(f" channels with max_diff > {threshold:5.1f}: {n}/{E}") - -# Overall element analysis -for threshold in [0.01, 0.1, 0.5, 1.0, 5.0]: - n = (diff > threshold).sum() - print(f"Elements with diff > {threshold}: {n}/{diff.size} ({100*n/diff.size:.2f}%)") - -# Cosine similarity -dot = np.sum(cpp_transposed * py_vit) -norm_a = np.sqrt(np.sum(cpp_transposed**2)) -norm_b = np.sqrt(np.sum(py_vit**2)) -cosine = dot / (norm_a * norm_b) -print(f"\nCosine similarity: {cosine:.8f}") diff --git a/tests/dump_cpp_preprocessed.py b/tests/dump_cpp_preprocessed.py deleted file mode 100644 index 7f20663..0000000 --- a/tests/dump_cpp_preprocessed.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Compare C++ vs Python image preprocessing. - -The C++ preprocessing: -1. Resize to 1008x1008 using bilinear interpolation (custom implementation) -2. Normalize: (pixel / 255.0 - 0.5) / 0.5 = pixel/127.5 - 1.0 - -The Python preprocessing (torchvision): -1. v2.Resize(size=(1008, 1008)) — uses PIL's bilinear -2. v2.Normalize(mean=[0.5,0.5,0.5], std=[0.5,0.5,0.5]) - -Differences come from the resize algorithm (C++ custom vs PIL bilinear). -This script replicates the C++ preprocessing in Python for verification. -""" -import numpy as np -import sys -from PIL import Image - -def cpp_style_preprocess(image_path, img_size=1008): - """Replicate C++ sam3_preprocess_image exactly.""" - img = Image.open(image_path).convert("RGB") - # PIL bilinear resize (this may differ from C++ implementation) - img_resized = img.resize((img_size, img_size), Image.BILINEAR) - pixels = np.array(img_resized, dtype=np.float32) # [H, W, 3] - - # Normalize: (pixel / 255.0 - 0.5) / 0.5 - pixels = (pixels / 255.0 - 0.5) / 0.5 - - # Convert HWC → CHW - chw = pixels.transpose(2, 0, 1) # [3, H, W] - return chw - -def load_tensor(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32) - return data.reshape(shape) - -if __name__ == "__main__": - ref_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/ref_phase3" - - # Load Python reference - ref = load_tensor(f"{ref_dir}/preprocessed") - print(f"Python ref shape: {ref.shape}") - print(f" min={ref.min():.6f} max={ref.max():.6f} mean={ref.mean():.6f}") - - # Also load raw float (before normalization) for debugging - try: - raw = load_tensor(f"{ref_dir}/preprocessed_raw_float") - print(f"Python raw float shape: {raw.shape}") - print(f" min={raw.min():.6f} max={raw.max():.6f} mean={raw.mean():.6f}") - except: - pass - - # Quick sanity: check normalization - # PyTorch Normalize: (x - 0.5) / 0.5 where x is in [0,1] - # So output range should be [-1, 1] - print(f"\n Expected range: [-1, 1]") - print(f" Actual range: [{ref.min():.6f}, {ref.max():.6f}]") - - # Check first few pixels for debugging - print(f"\n First 10 pixels (channel 0): {ref.flatten()[:10]}") diff --git a/tests/dump_ddec_from_package.py b/tests/dump_ddec_from_package.py deleted file mode 100644 index f230e1a..0000000 --- a/tests/dump_ddec_from_package.py +++ /dev/null @@ -1,411 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump DETR decoder reference tensors directly from the SAM3 Python package. -Uses the actual TransformerDecoder.forward() from the package (not reimplemented). - -This script: -1. Builds the model using the SAM3 package -2. Loads the checkpoint -3. Runs just the decoder forward pass with known inputs -4. Dumps per-layer outputs for comparison with C++ - -Usage: - cd ~/Documents/sam3 - python3 /path/to/dump_ddec_from_package.py --checkpoint raw_weights/sam3.pt --outdir /tmp/ddec_ref -""" -import argparse -import math -import os -import sys -import importlib - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F - - -def save_tensor(path, t): - """Save tensor as .bin + .shape files.""" - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)}") - - -def inverse_sigmoid(x, eps=1e-3): - x = x.clamp(min=0, max=1) - x1 = x.clamp(min=eps) - x2 = (1 - x).clamp(min=eps) - return torch.log(x1 / x2) - - -def gen_sineembed_for_position(pos_tensor, num_feats=256): - assert num_feats % 2 == 0 - num_feats = num_feats // 2 - scale = 2 * math.pi - dim_t = torch.arange(num_feats, dtype=torch.float32, device=pos_tensor.device) - dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode="floor")) / num_feats) - x_embed = pos_tensor[:, :, 0] * scale - y_embed = pos_tensor[:, :, 1] * scale - pos_x = x_embed[:, :, None] / dim_t - pos_y = y_embed[:, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - if pos_tensor.size(-1) == 2: - pos = torch.cat((pos_y, pos_x), dim=2) - elif pos_tensor.size(-1) == 4: - w_embed = pos_tensor[:, :, 2] * scale - pos_w = w_embed[:, :, None] / dim_t - pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) - h_embed = pos_tensor[:, :, 3] * scale - pos_h = h_embed[:, :, None] / dim_t - pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) - pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) - else: - raise ValueError(f"Unknown pos_tensor shape(-1):{pos_tensor.size(-1)}") - return pos - - -def box_cxcywh_to_xyxy(x): - x_c, y_c, w, h = x.unbind(-1) - b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] - return torch.stack(b, dim=-1) - - -def mlp_forward(x, sd, prefix, num_layers, activation="relu"): - for i in range(num_layers): - w = sd[f"{prefix}.layers.{i}.weight"] - b = sd[f"{prefix}.layers.{i}.bias"] - x = F.linear(x, w, b) - if i < num_layers - 1: - x = F.relu(x) - return x - - -def multihead_attention_forward(query, key, value, w_q, b_q, w_k, b_k, w_v, b_v, - w_out, b_out, num_heads, attn_mask=None, - key_padding_mask=None): - D = query.shape[-1] - HD = D // num_heads - Q = F.linear(query, w_q, b_q) - K = F.linear(key, w_k, b_k) - V = F.linear(value, w_v, b_v) - BS = Q.shape[0] - N_q = Q.shape[1] - N_kv = K.shape[1] - Q = Q.reshape(BS, N_q, num_heads, HD).permute(0, 2, 1, 3) - K = K.reshape(BS, N_kv, num_heads, HD).permute(0, 2, 1, 3) - V = V.reshape(BS, N_kv, num_heads, HD).permute(0, 2, 1, 3) - - combined_mask = attn_mask - if key_padding_mask is not None: - pad_bias = torch.zeros(BS, 1, 1, N_kv, dtype=Q.dtype, device=Q.device) - pad_bias = pad_bias.masked_fill(key_padding_mask[:, None, None, :], float("-inf")) - combined_mask = pad_bias if combined_mask is None else combined_mask + pad_bias - attn_output = F.scaled_dot_product_attention(Q, K, V, attn_mask=combined_mask) - attn_output = attn_output.permute(0, 2, 1, 3).reshape(BS, N_q, D) - out = F.linear(attn_output, w_out, b_out) - return out - - -def fused_mha_forward(query, key, value, in_proj_w, in_proj_b, out_w, out_b, - num_heads, attn_mask=None, key_padding_mask=None): - D = query.shape[-1] - w_q, w_k, w_v = in_proj_w[:D], in_proj_w[D:2*D], in_proj_w[2*D:] - b_q, b_k, b_v = in_proj_b[:D], in_proj_b[D:2*D], in_proj_b[2*D:] - return multihead_attention_forward(query, key, value, w_q, b_q, w_k, b_k, w_v, b_v, - out_w, out_b, num_heads, attn_mask, key_padding_mask) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--outdir", default="/tmp/ddec_ref") - parser.add_argument("--fenc-input", default=None, - help="Directory containing fenc output to use as decoder input") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" - D = 256 - NH = 8 - NQ = 200 - H = 72 - - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - ddec_prefix = "detector.transformer.decoder." - ddec_sd = {k[len("detector.transformer."):]: v for k, v in ckpt.items() - if k.startswith("detector.transformer.decoder.")} - - # Load fenc output as decoder input - fenc_dir = args.fenc_input - if fenc_dir is None: - # Use existing ref_phase5 data - script_dir = os.path.dirname(os.path.abspath(__file__)) - fenc_dir = os.path.join(script_dir, "ref_phase5") - - def load_tensor(path, dtype=np.float32): - shape = list(map(int, open(path + ".shape").read().strip().split(","))) - data = np.fromfile(path + ".bin", dtype=dtype).reshape(shape) - return torch.tensor(data) - - # Load fenc output and image PE - fenc_output_bf = load_tensor(os.path.join(fenc_dir, "fenc_output_bf")) - if fenc_output_bf.ndim == 3: - pass # [1, HW, D] batch-first - else: - # Try loading seq-first [HW, 1, D] and converting - fenc_output_sf = load_tensor(os.path.join(fenc_dir, "fenc_output")) - fenc_output_bf = fenc_output_sf.permute(1, 0, 2) # [1, HW, D] - - # Image PE - load and flatten - img_pe = load_tensor(os.path.join(fenc_dir, "img_pe_72")) # [1, 256, 72, 72] - img_pe_flat = img_pe.flatten(2).permute(0, 2, 1) # [1, 5184, 256] - - # Text features - text_features = load_tensor(os.path.join(fenc_dir, "text_features")) - if text_features.ndim == 3 and text_features.shape[0] == 32: - # [T, 1, D] seq-first - prompt_sf = text_features - prompt_bf = text_features.permute(1, 0, 2) - elif text_features.ndim == 2: - # [D, T] ggml layout -> convert - prompt_sf = text_features.T.unsqueeze(1) # [T, 1, D] - prompt_bf = prompt_sf.permute(1, 0, 2) - else: - prompt_sf = text_features - prompt_bf = text_features.permute(1, 0, 2) - - T = prompt_sf.shape[0] - - # Token IDs for text mask (saved as int32) - token_ids_path = os.path.join(fenc_dir, "token_ids") - if os.path.exists(token_ids_path + ".bin"): - token_ids = load_tensor(token_ids_path, dtype=np.int32).long().squeeze().tolist() - else: - # Assume all valid - token_ids = [1] * T - - txt_mask = torch.tensor([[tid == 0 for tid in token_ids]], dtype=torch.bool) - - print(f"fenc output: {list(fenc_output_bf.shape)}") - print(f"prompt: {list(prompt_sf.shape)}") - print(f"img PE: {list(img_pe_flat.shape)}") - - # ═══════════════════════════════════════════════════════════════════ - # DETR Decoder forward (matching Python package exactly) - # ═══════════════════════════════════════════════════════════════════ - print("\n=== DETR Decoder (from checkpoint weights) ===") - - with torch.no_grad(): - query_embed = ddec_sd["decoder.query_embed.weight"] # [NQ, D] - ref_pts_raw = ddec_sd["decoder.reference_points.weight"] # [NQ, 4] - reference_boxes = ref_pts_raw.sigmoid().unsqueeze(1) # [NQ, 1, 4] - - save_tensor(os.path.join(args.outdir, "ddec_query_embed"), query_embed) - save_tensor(os.path.join(args.outdir, "ddec_ref_pts_raw"), ref_pts_raw) - save_tensor(os.path.join(args.outdir, "ddec_ref_boxes_init"), reference_boxes) - - presence_token_w = ddec_sd["decoder.presence_token.weight"] # [1, D] - presence_out = presence_token_w[None].expand(1, 1, -1) # [1, 1, D] - save_tensor(os.path.join(args.outdir, "ddec_presence_token"), presence_out) - - enc_memory = fenc_output_bf.permute(1, 0, 2) # [5184, 1, D] seq-first - enc_pos = img_pe_flat.permute(1, 0, 2) # [5184, 1, D] seq-first - - valid_ratios = torch.ones(1, 1, 2, device=device) - spatial_shapes = torch.tensor([[H, H]], dtype=torch.long) - output = query_embed.unsqueeze(1).repeat(1, 1, 1) # [NQ, 1, D] - - coords_h = torch.arange(0, H, dtype=torch.float32) / H - coords_w = torch.arange(0, H, dtype=torch.float32) / H - - for layer_idx in range(6): - lp = f"decoder.layers.{layer_idx}." - - # Reference points input - reference_points_input = reference_boxes[:, :, None] * torch.cat([valid_ratios, valid_ratios], -1)[None, :] - query_sine_embed = gen_sineembed_for_position(reference_points_input[:, :, 0, :], D) - query_pos = mlp_forward(query_sine_embed, ddec_sd, "decoder.ref_point_head", 2) - - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_query_sine_0"), query_sine_embed) - save_tensor(os.path.join(args.outdir, "ddec_query_pos_0"), query_pos) - - # Box RPB - boxes_xyxy = box_cxcywh_to_xyxy(reference_boxes).permute(1, 0, 2) - bs_d, nq_d, _ = boxes_xyxy.shape - deltas_y = coords_h.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 1:4:2] - deltas_y = deltas_y.view(bs_d, nq_d, -1, 2) - deltas_x = coords_w.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 0:3:2] - deltas_x = deltas_x.view(bs_d, nq_d, -1, 2) - - deltas_x_log = deltas_x * 8 - deltas_x_log = torch.sign(deltas_x_log) * torch.log2(torch.abs(deltas_x_log) + 1.0) / np.log2(8) - deltas_y_log = deltas_y * 8 - deltas_y_log = torch.sign(deltas_y_log) * torch.log2(torch.abs(deltas_y_log) + 1.0) / np.log2(8) - deltas_x = deltas_x_log - deltas_y = deltas_y_log - - rpb_x = mlp_forward(deltas_x, ddec_sd, "decoder.boxRPB_embed_x", 2) - rpb_y = mlp_forward(deltas_y, ddec_sd, "decoder.boxRPB_embed_y", 2) - B_rpb = rpb_y.unsqueeze(3) + rpb_x.unsqueeze(2) - B_rpb = B_rpb.flatten(2, 3) - B_rpb = B_rpb.permute(0, 3, 1, 2).contiguous() - pres_rpb = torch.zeros(1, NH, 1, H*H, device=device) - memory_mask = torch.cat([pres_rpb, B_rpb], dim=2) - memory_mask = memory_mask.flatten(0, 1) - - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_rpb_mask_0"), memory_mask) - - # Self-attention - tgt_with_pres = torch.cat([presence_out, output], dim=0) - qpos_with_pres = torch.cat([torch.zeros_like(presence_out), query_pos], dim=0) - q_sa = k_sa = tgt_with_pres + qpos_with_pres - sa_w = ddec_sd[lp + "self_attn.in_proj_weight"] - sa_b = ddec_sd[lp + "self_attn.in_proj_bias"] - sa_ow = ddec_sd[lp + "self_attn.out_proj.weight"] - sa_ob = ddec_sd[lp + "self_attn.out_proj.bias"] - q_sa_bf = q_sa.permute(1, 0, 2) - k_sa_bf = k_sa.permute(1, 0, 2) - v_sa_bf = tgt_with_pres.permute(1, 0, 2) - sa_out = fused_mha_forward(q_sa_bf, k_sa_bf, v_sa_bf, sa_w, sa_b, sa_ow, sa_ob, NH) - tgt_with_pres = tgt_with_pres + sa_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm2.weight"], ddec_sd[lp + "norm2.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_sa"), tgt_with_pres) - - # Text cross-attention - tgt_q_ca_text = tgt_with_pres + qpos_with_pres - ca_text_w = ddec_sd[lp + "ca_text.in_proj_weight"] - ca_text_b = ddec_sd[lp + "ca_text.in_proj_bias"] - ca_text_ow = ddec_sd[lp + "ca_text.out_proj.weight"] - ca_text_ob = ddec_sd[lp + "ca_text.out_proj.bias"] - w_q_ct, w_k_ct, w_v_ct = ca_text_w[:D], ca_text_w[D:2*D], ca_text_w[2*D:] - b_q_ct, b_k_ct, b_v_ct = ca_text_b[:D], ca_text_b[D:2*D], ca_text_b[2*D:] - tq_bf = tgt_q_ca_text.permute(1, 0, 2) - p_bf = prompt_sf.permute(1, 0, 2) - ct_out = multihead_attention_forward(tq_bf, p_bf, p_bf, - w_q_ct, b_q_ct, w_k_ct, b_k_ct, w_v_ct, b_v_ct, - ca_text_ow, ca_text_ob, NH, - key_padding_mask=txt_mask) - tgt_with_pres = tgt_with_pres + ct_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "catext_norm.weight"], - ddec_sd[lp + "catext_norm.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_text_ca"), tgt_with_pres) - - # Image cross-attention with RPB - tgt_q_ca_img = tgt_with_pres + qpos_with_pres - mem_k = enc_memory + enc_pos - ca_w = ddec_sd[lp + "cross_attn.in_proj_weight"] - ca_b = ddec_sd[lp + "cross_attn.in_proj_bias"] - ca_ow = ddec_sd[lp + "cross_attn.out_proj.weight"] - ca_ob = ddec_sd[lp + "cross_attn.out_proj.bias"] - w_q_ci, w_k_ci, w_v_ci = ca_w[:D], ca_w[D:2*D], ca_w[2*D:] - b_q_ci, b_k_ci, b_v_ci = ca_b[:D], ca_b[D:2*D], ca_b[2*D:] - tqi_bf = tgt_q_ca_img.permute(1, 0, 2) - mk_bf = mem_k.permute(1, 0, 2) - mv_bf = enc_memory.permute(1, 0, 2) - ci_out = multihead_attention_forward(tqi_bf, mk_bf, mv_bf, - w_q_ci, b_q_ci, w_k_ci, b_k_ci, w_v_ci, b_v_ci, - ca_ow, ca_ob, NH, - attn_mask=memory_mask) - tgt_with_pres = tgt_with_pres + ci_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm1.weight"], ddec_sd[lp + "norm1.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_img_ca"), tgt_with_pres) - - # FFN - ffn = F.linear(tgt_with_pres, ddec_sd[lp + "linear1.weight"], ddec_sd[lp + "linear1.bias"]) - ffn = F.relu(ffn) - ffn = F.linear(ffn, ddec_sd[lp + "linear2.weight"], ddec_sd[lp + "linear2.bias"]) - tgt_with_pres = tgt_with_pres + ffn - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm3.weight"], ddec_sd[lp + "norm3.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_full_out"), tgt_with_pres) - - # Split presence and queries - presence_out = tgt_with_pres[:1] - output = tgt_with_pres[1:] - - # Box refinement - reference_before_sigmoid = inverse_sigmoid(reference_boxes) - normed_output = F.layer_norm(output, [D], - ddec_sd["decoder.norm.weight"], ddec_sd["decoder.norm.bias"]) - delta_unsig = mlp_forward(normed_output, ddec_sd, "decoder.bbox_embed", 3) - outputs_unsig = delta_unsig + reference_before_sigmoid - new_reference_points = outputs_unsig.sigmoid() - reference_boxes = new_reference_points.detach() - - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_out"), output) - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_refboxes"), reference_boxes) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_presence"), presence_out) - - # Final - normed_output = F.layer_norm(output, [D], - ddec_sd["decoder.norm.weight"], ddec_sd["decoder.norm.bias"]) - save_tensor(os.path.join(args.outdir, "ddec_normed_output"), normed_output) - save_tensor(os.path.join(args.outdir, "ddec_pred_boxes"), reference_boxes) - - pres_normed = F.layer_norm(presence_out, [D], - ddec_sd["decoder.presence_token_out_norm.weight"], - ddec_sd["decoder.presence_token_out_norm.bias"]) - pres_logit = mlp_forward(pres_normed, ddec_sd, "decoder.presence_token_head", 3) - save_tensor(os.path.join(args.outdir, "ddec_presence_logit"), pres_logit) - - # Scoring - scoring_prefix = "detector.dot_prod_scoring." - scoring_sd = {k[len(scoring_prefix):]: v for k, v in ckpt.items() - if k.startswith(scoring_prefix)} - - prompt_for_scoring = prompt_sf.clone() - orig = prompt_for_scoring.clone() - prompt_mlp_out = F.linear(prompt_for_scoring, scoring_sd["prompt_mlp.layers.0.weight"], - scoring_sd["prompt_mlp.layers.0.bias"]) - prompt_mlp_out = F.relu(prompt_mlp_out) - prompt_mlp_out = F.linear(prompt_mlp_out, scoring_sd["prompt_mlp.layers.1.weight"], - scoring_sd["prompt_mlp.layers.1.bias"]) - prompt_mlp_out = prompt_mlp_out + orig - prompt_mlp_out = F.layer_norm(prompt_mlp_out, [D], - scoring_sd["prompt_mlp.out_norm.weight"], - scoring_sd["prompt_mlp.out_norm.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_prompt_mlp_out"), prompt_mlp_out) - - is_valid = (~txt_mask).float().permute(1, 0)[..., None] - num_valid = torch.clamp(is_valid.sum(dim=0), min=1.0) - pooled = (prompt_mlp_out * is_valid).sum(dim=0) / num_valid - save_tensor(os.path.join(args.outdir, "scoring_pooled"), pooled) - - proj_pooled = F.linear(pooled, scoring_sd["prompt_proj.weight"], - scoring_sd["prompt_proj.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_proj_pooled"), proj_pooled) - - hs = normed_output.permute(1, 0, 2).unsqueeze(0) - proj_hs = F.linear(hs, scoring_sd["hs_proj.weight"], scoring_sd["hs_proj.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_proj_hs"), proj_hs) - - scores = torch.matmul(proj_hs, proj_pooled.unsqueeze(-1)) - scores = scores * (1.0 / math.sqrt(D)) - scores = scores.clamp(-12.0, 12.0) - save_tensor(os.path.join(args.outdir, "scoring_class_scores"), scores) - - print("\nDone! All decoder tensors saved.") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_fenc_from_package.py b/tests/dump_fenc_from_package.py deleted file mode 100644 index 4c62f6e..0000000 --- a/tests/dump_fenc_from_package.py +++ /dev/null @@ -1,469 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump fusion encoder inputs and per-layer outputs from the ACTUAL SAM3 Python package. - -This script monkey-patches TransformerEncoder.forward to intercept the exact tensors -flowing through the fusion encoder during real inference. This ensures the reference -tensors are bit-identical to what the real model produces — no reimplementation. - -Usage: - cd ~/Documents/sam3 - uv run python /Users/pierre-antoine/Documents/sam3.cpp/tests/dump_fenc_from_package.py \ - [--image /path/to/image.jpg] [--text "yellow school bus"] \ - [--outdir /Users/pierre-antoine/Documents/sam3.cpp/tests/ref_fenc_pkg] -""" -import argparse -import os -import sys -import types - -# ── Mock triton for macOS (must happen before any torch/sam3 import) ── -def _install_triton_mock(): - """Install a mock triton package so SAM3 and torch._dynamo can import.""" - if "triton" in sys.modules: - return # Already real or already mocked - - triton = types.ModuleType("triton") - triton.__path__ = [] # Make it a package - triton.jit = lambda fn=None, **kw: fn if fn else (lambda f: f) - triton.autotune = lambda **kw: lambda f: f - triton.heuristics = lambda **kw: lambda f: f - triton.Config = type("Config", (), {"__init__": lambda self, **kw: None}) - - lang = types.ModuleType("triton.language") - lang.__path__ = [] - lang.constexpr = int - lang.dtype = type # torch._dynamo.utils needs this - triton.language = lang - - core = types.ModuleType("triton.language.core") - core.dtype = type - lang.core = core - - backends = types.ModuleType("triton.backends") - backends.__path__ = [] - compiler = types.ModuleType("triton.backends.compiler") - backends.compiler = compiler - - runtime = types.ModuleType("triton.runtime") - runtime.__path__ = [] - - triton_compiler = types.ModuleType("triton.compiler") - triton_compiler.__path__ = [] - triton_compiler_compiler = types.ModuleType("triton.compiler.compiler") - triton_compiler.compiler = triton_compiler_compiler - - # Wire submodule attributes (so triton.backends.compiler works) - triton.backends = backends - triton.runtime = runtime - triton.compiler = triton_compiler - lang.core = core - backends.compiler = compiler - triton_compiler.compiler = triton_compiler_compiler - - for name, mod in [ - ("triton", triton), - ("triton.language", lang), - ("triton.language.core", core), - ("triton.backends", backends), - ("triton.backends.compiler", compiler), - ("triton.runtime", runtime), - ("triton.compiler", triton_compiler), - ("triton.compiler.compiler", triton_compiler_compiler), - ]: - sys.modules[name] = mod - -_install_triton_mock() - -import numpy as np -import torch - -# Force CPU — SAM3 model builder uses device="cuda" in some places. -# Monkey-patch torch.zeros/torch.ones/torch.empty to replace device="cuda" with "cpu". -_orig_zeros = torch.zeros -_orig_ones = torch.ones -_orig_empty = torch.empty - -def _cpu_override(orig_fn): - def wrapper(*args, **kwargs): - dev = kwargs.get("device") - if isinstance(dev, str) and "cuda" in dev: - kwargs["device"] = "cpu" - elif isinstance(dev, torch.device) and dev.type == "cuda": - kwargs["device"] = "cpu" - return orig_fn(*args, **kwargs) - return wrapper - -_orig_arange = torch.arange -_orig_linspace = torch.linspace -_orig_full = torch.full -_orig_randn = torch.randn -_orig_rand = torch.rand -_orig_tensor = torch.tensor - -# Also neuter pin_memory — it tries to use MPS on macOS and causes device mismatch -_orig_pin_memory = torch.Tensor.pin_memory -torch.Tensor.pin_memory = lambda self, *a, **kw: self - -torch.zeros = _cpu_override(_orig_zeros) -torch.ones = _cpu_override(_orig_ones) -torch.empty = _cpu_override(_orig_empty) -torch.arange = _cpu_override(_orig_arange) -torch.linspace = _cpu_override(_orig_linspace) -torch.full = _cpu_override(_orig_full) -torch.randn = _cpu_override(_orig_randn) -torch.rand = _cpu_override(_orig_rand) -torch.tensor = _cpu_override(_orig_tensor) - -# --------------------------------------------------------------------------- -# Tensor save helpers (matching existing test infrastructure format) -# --------------------------------------------------------------------------- - -def save_tensor(path, t): - """Save tensor as .bin + .shape (PyTorch native layout).""" - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)} " - f"min={t.min().item():.4f} max={t.max().item():.4f}") - - -def save_tensor_i32(path, t): - t = t.detach().cpu().to(torch.int32).contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)} dtype=int32") - - -# --------------------------------------------------------------------------- -# Global capture dict — filled by the monkey-patched encoder forward -# --------------------------------------------------------------------------- - -captured = {} - - -def install_encoder_patch(): - """ - Monkey-patch TransformerEncoder.forward (the base class used by - TransformerEncoderFusion) to dump inputs and per-layer outputs. - - TransformerEncoderFusion.forward() does: - 1. Reshape src from seq-first to NCHW - 2. (Optional pooled text — disabled with add_pooled_text_to_img_feat=False) - 3. Call super().forward(src_NCHW, ..., prompt=prompt.transpose(0,1), ...) - - super().forward() (TransformerEncoder.forward) does: - 1. _prepare_multilevel_features → src_flatten [B, HW, D], lvl_pos_embed_flatten [B, HW, D] - 2. Loop through 6 encoder layers - - We patch TransformerEncoder.forward to capture the flattened tensors. - """ - from sam3.model.encoder import TransformerEncoder - from sam3.model.act_ckpt_utils import activation_ckpt_wrapper - - original_forward = TransformerEncoder.forward - - def patched_forward(self, src, src_key_padding_masks=None, pos=None, - prompt=None, prompt_key_padding_mask=None, - encoder_extra_kwargs=None): - # --- _prepare_multilevel_features (same as original) --- - ( - src_flatten, - key_padding_masks_flatten, - lvl_pos_embed_flatten, - level_start_index, - valid_ratios, - spatial_shapes, - ) = self._prepare_multilevel_features(src, src_key_padding_masks, pos) - - reference_points = self.get_reference_points( - spatial_shapes, valid_ratios, device=src_flatten.device - ) - - # ═══ DUMP INPUTS ═══ - # src_flatten: [B, HW, D] = [1, 5184, 256] batch-first image features - captured["fenc_input_tgt"] = src_flatten.detach().clone() - # lvl_pos_embed_flatten: [B, HW, D] = [1, 5184, 256] positional encoding - captured["fenc_input_pos"] = lvl_pos_embed_flatten.detach().clone() - # prompt: [B, T, D] = [1, 32, 256] batch-first text tokens - if prompt is not None: - captured["fenc_input_prompt"] = prompt.detach().clone() - # prompt_key_padding_mask: [B, T] = [1, 32] boolean (True=padding) - if prompt_key_padding_mask is not None: - captured["fenc_input_prompt_mask"] = prompt_key_padding_mask.detach().clone() - - # --- Layer loop (same as original) --- - output = src_flatten - for layer_idx, layer in enumerate(self.layers): - layer_kwargs = {} - - assert hasattr(layer, 'forward_pre') or hasattr(layer, 'forward_post') - layer_kwargs["memory"] = prompt - layer_kwargs["memory_key_padding_mask"] = prompt_key_padding_mask - layer_kwargs["query_pos"] = lvl_pos_embed_flatten - layer_kwargs["tgt"] = output - layer_kwargs["tgt_key_padding_mask"] = key_padding_masks_flatten - - if encoder_extra_kwargs is not None: - layer_kwargs.update(encoder_extra_kwargs) - output = activation_ckpt_wrapper(layer)( - **layer_kwargs, - act_ckpt_enable=self.training and self.use_act_checkpoint, - ) - - # ═══ DUMP PER-LAYER OUTPUT ═══ - captured[f"fenc_layer{layer_idx}_out"] = output.detach().clone() - - # --- Return (same as original) --- - return ( - output.transpose(0, 1), - ( - key_padding_masks_flatten.transpose(0, 1) - if key_padding_masks_flatten is not None - else None - ), - lvl_pos_embed_flatten.transpose(0, 1), - level_start_index, - spatial_shapes, - valid_ratios, - ) - - TransformerEncoder.forward = patched_forward - print("[PATCH] TransformerEncoder.forward patched for tensor dumping") - - -def install_text_encoder_hook(model): - """ - Hook into the text encoder output to capture text features and token info. - We intercept backbone.forward_text by wrapping it. - """ - original_forward_text = model.backbone.forward_text - - def patched_forward_text(text_list, **kwargs): - result = original_forward_text(text_list, **kwargs) - # language_features: [T, num_prompts, D] = [32, 1, 256] - if "language_features" in result: - captured["text_features_raw"] = result["language_features"].detach().clone() - if "language_mask" in result: - captured["text_mask_raw"] = result["language_mask"].detach().clone() - return result - - model.backbone.forward_text = patched_forward_text - print("[PATCH] backbone.forward_text wrapped for text feature capture") - - -def install_prompt_hook(model): - """ - Hook into _encode_prompt to capture the assembled prompt (text + geo + visual). - """ - original_encode_prompt = model._encode_prompt - - def patched_encode_prompt(*args, **kwargs): - prompt, prompt_mask, backbone_out = original_encode_prompt(*args, **kwargs) - # prompt: [seq, batch, dim] = [T+geo, 1, 256] seq-first - captured["prompt_assembled"] = prompt.detach().clone() - captured["prompt_mask_assembled"] = prompt_mask.detach().clone() - return prompt, prompt_mask, backbone_out - - model._encode_prompt = patched_encode_prompt - print("[PATCH] _encode_prompt wrapped for prompt capture") - - -def install_backbone_hook(model): - """ - Capture backbone FPN features after set_image. - """ - original_forward_image = model.backbone.forward_image - - def patched_forward_image(image): - result = original_forward_image(image) - if "backbone_fpn" in result: - for i, feat in enumerate(result["backbone_fpn"]): - captured[f"backbone_fpn_{i}"] = feat.detach().clone() - if "vision_pos_enc" in result: - for i, pe in enumerate(result["vision_pos_enc"]): - captured[f"vision_pos_enc_{i}"] = pe.detach().clone() - return result - - model.backbone.forward_image = patched_forward_image - print("[PATCH] backbone.forward_image wrapped for FPN capture") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - -def main(): - parser = argparse.ArgumentParser(description="Dump fenc tensors from SAM3 package") - parser.add_argument("--image", default=None, - help="Test image path (default: tests/test_random.jpg)") - parser.add_argument("--text", default="yellow school bus", - help="Text prompt") - parser.add_argument("--outdir", - default="/Users/pierre-antoine/Documents/sam3.cpp/tests/ref_fenc_pkg", - help="Output directory for dumped tensors") - args = parser.parse_args() - - if args.image is None: - args.image = "/Users/pierre-antoine/Documents/sam3.cpp/tests/test_random.jpg" - - os.makedirs(args.outdir, exist_ok=True) - - # ── Load model ── - print("Loading SAM3 model...") - sys.path.insert(0, os.path.expanduser("~/Documents/sam3")) - - import sam3 - from sam3 import build_sam3_image_model - from sam3.model.sam3_image_processor import Sam3Processor - - sam3_pkg_dir = os.path.dirname(sam3.__file__) # ~/Documents/sam3/sam3/ - torch.set_default_dtype(torch.float32) - - bpe_path = os.path.join(sam3_pkg_dir, "assets", "bpe_simple_vocab_16e6.txt.gz") - if not os.path.exists(bpe_path): - # Try parent dir - bpe_path = os.path.join(sam3_pkg_dir, "..", "assets", "bpe_simple_vocab_16e6.txt.gz") - print(f" BPE path: {bpe_path} (exists={os.path.exists(bpe_path)})") - - ckpt_path = "/Users/pierre-antoine/Documents/sam3.cpp/raw_weights/sam3.pt" - print(f" Checkpoint: {ckpt_path} (exists={os.path.exists(ckpt_path)})") - - model = build_sam3_image_model( - bpe_path=bpe_path, - checkpoint_path=ckpt_path, - device="cpu", - load_from_HF=False, - ) - model = model.float() - model.eval() - - # ── Install all patches ── - install_encoder_patch() - install_text_encoder_hook(model) - install_prompt_hook(model) - install_backbone_hook(model) - - # ── Load and preprocess image ── - from PIL import Image - print(f"Loading image: {args.image}") - image = Image.open(args.image).convert("RGB") - - processor = Sam3Processor(model, confidence_threshold=0.5, device="cpu") - - # ── Step 1: Encode image (runs backbone) ── - print("\n=== Step 1: Encoding image ===") - state = processor.set_image(image) - - # Save backbone features - for key in sorted(captured.keys()): - if key.startswith("backbone_fpn_") or key.startswith("vision_pos_enc_"): - save_tensor(os.path.join(args.outdir, key), captured[key]) - - # ── Step 2: Run text prompt (runs text encoder + fusion encoder + decoder) ── - print(f"\n=== Step 2: Running text prompt '{args.text}' ===") - state = processor.set_text_prompt(prompt=args.text, state=state) - - # ── Save all captured tensors ── - print(f"\n=== Saving tensors to {args.outdir} ===") - - # Save prompt text - with open(os.path.join(args.outdir, "prompt.txt"), "w") as f: - f.write(args.text + "\n") - - # -- Fusion encoder inputs -- - print("\n-- Fusion encoder inputs --") - # fenc_input_tgt: [1, 5184, 256] batch-first image features - # Memory layout: 5184 blocks of 256 floats = same as ggml [256, 5184, 1] - if "fenc_input_tgt" in captured: - save_tensor(os.path.join(args.outdir, "fenc_input_tgt"), captured["fenc_input_tgt"]) - - # fenc_input_pos: [1, 5184, 256] batch-first positional encoding - if "fenc_input_pos" in captured: - save_tensor(os.path.join(args.outdir, "fenc_input_pos"), captured["fenc_input_pos"]) - - # fenc_input_prompt: [1, T, 256] batch-first prompt tokens - if "fenc_input_prompt" in captured: - save_tensor(os.path.join(args.outdir, "fenc_input_prompt"), captured["fenc_input_prompt"]) - - # fenc_input_prompt_mask: [1, T] boolean (True=padding) - if "fenc_input_prompt_mask" in captured: - mask = captured["fenc_input_prompt_mask"] - save_tensor(os.path.join(args.outdir, "fenc_input_prompt_mask"), - mask.float()) - # Also save as attention bias (0.0 for valid, -1e9 for padding) - # This is what the C++ uses directly - attn_bias = torch.where(mask, torch.tensor(-1e9), torch.tensor(0.0)) - save_tensor(os.path.join(args.outdir, "fenc_attn_bias"), attn_bias) - - # -- Also save in format compatible with existing phase 5 test -- - print("\n-- Phase 5 compatible format --") - if "fenc_input_tgt" in captured: - # [1, 5184, 256] → same binary layout as [32, 1, 256] seq-first - save_tensor(os.path.join(args.outdir, "fenc_img_input"), captured["fenc_input_tgt"]) - if "fenc_input_pos" in captured: - save_tensor(os.path.join(args.outdir, "fenc_pos_embed"), captured["fenc_input_pos"]) - - # Text features in seq-first format [T, B, D] for compatibility - if "prompt_assembled" in captured: - # prompt_assembled is [T, B, D] seq-first - save_tensor(os.path.join(args.outdir, "text_features"), captured["prompt_assembled"]) - - # -- Fusion encoder per-layer outputs -- - print("\n-- Fusion encoder per-layer outputs --") - for i in range(6): - key = f"fenc_layer{i}_out" - if key in captured: - save_tensor(os.path.join(args.outdir, key), captured[key]) - - # -- Text encoder raw output -- - print("\n-- Text encoder output --") - if "text_features_raw" in captured: - save_tensor(os.path.join(args.outdir, "text_features_raw"), captured["text_features_raw"]) - if "text_mask_raw" in captured: - save_tensor(os.path.join(args.outdir, "text_mask_raw"), captured["text_mask_raw"].float()) - - # -- Also save neck features for potential full pipeline test -- - print("\n-- Neck detector features --") - # The detector uses backbone_fpn levels. Save the last 3 for the seg head. - for i in range(4): - key = f"backbone_fpn_{i}" - if key in captured: - t = captured[key] - # Save as neck_det_{i} in NCHW format - save_tensor(os.path.join(args.outdir, f"neck_det_{i}"), t) - - # -- Summary -- - print(f"\n=== Summary ===") - print(f"Total tensors captured: {len(captured)}") - if "fenc_input_tgt" in captured: - print(f" fenc image features: {list(captured['fenc_input_tgt'].shape)}") - if "fenc_input_prompt" in captured: - t = captured["fenc_input_prompt"] - print(f" fenc prompt tokens: {list(t.shape)}") - if "fenc_input_prompt_mask" in captured: - mask = captured["fenc_input_prompt_mask"] - n_valid = (~mask).sum().item() - n_total = mask.numel() - print(f" prompt mask: {n_valid}/{n_total} valid tokens") - if "fenc_layer5_out" in captured: - print(f" fenc output: {list(captured['fenc_layer5_out'].shape)}") - - # Check for detection results - if "boxes" in state and len(state.get("boxes", [])) > 0: - print(f"\n Detections: {len(state['boxes'])} objects found") - else: - print(f"\n Detections: none (may need different prompt/image)") - - print(f"\nAll tensors saved to {args.outdir}/") - print("Next: run the C++ test_debug_fenc against these references.") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_geom_encoder_reference.py b/tests/dump_geom_encoder_reference.py deleted file mode 100644 index 4b2d37f..0000000 --- a/tests/dump_geom_encoder_reference.py +++ /dev/null @@ -1,482 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump geometry encoder reference tensors for numerical comparison against sam3.cpp. - -Tests the SequenceGeometryEncoder with exemplar box prompts. -Dumps intermediate tensors at each stage: - 1. Input box coordinates (CxCyWH normalized) - 2. Direct box projection output - 3. ROI align pooled features - 4. Box positional encoding - 5. Label embedding - 6. Combined box embedding - 7. CLS token - 8. Final projection output - 9. Per-layer transformer outputs - 10. Final output - -Also tests the dummy prompt case (no exemplars, just CLS token). - -Usage: - cd ~/Documents/sam3 - uv run python /Users/pierre-antoine/Documents/sam3.cpp/tests/dump_geom_encoder_reference.py \ - --checkpoint raw_weights/sam3.pt \ - --prephase-ref /Users/pierre-antoine/Documents/sam3.cpp/tests/ref_phase3 \ - --outdir /Users/pierre-antoine/Documents/sam3.cpp/tests/ref_geom -""" - -import argparse -import math -import os -import sys -from typing import Dict, List, Tuple - -import numpy as np -import torch -import torch.nn.functional as F -import torchvision - -# ── Tensor save utilities ─────────────────────────────────────────────────── - -def save_raw(path: str, arr: np.ndarray, shape) -> None: - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(arr.astype(np.float32, copy=False).tobytes()) - with open(path + ".shape", "w", encoding="utf-8") as f: - f.write(",".join(str(int(d)) for d in shape)) - - -def save_ggml_bnd(path: str, x_bnd: torch.Tensor) -> None: - """Save [B, N, D] as ggml [D, N, B].""" - assert x_bnd.ndim == 3 - x = x_bnd.detach().cpu().float().contiguous() - b, n, d = x.shape - save_raw(path, x.numpy(), (d, n, b)) - - -def save_ggml_sbd(path: str, x_sbd: torch.Tensor) -> None: - """Save [S, B, D] (seq-first) as ggml [D, S, B].""" - assert x_sbd.ndim == 3 - x = x_sbd.detach().cpu().float().contiguous() - s, b, d = x.shape - # Permute to [B, S, D] then save as ggml [D, S, B] - x_bsd = x.permute(1, 0, 2).contiguous() - save_raw(path, x_bsd.numpy(), (d, s, b)) - - -def save_ggml_nchw(path: str, x_nchw: torch.Tensor) -> None: - """Save [N, C, H, W] as ggml [C, W, H, N].""" - assert x_nchw.ndim == 4 - x = x_nchw.detach().cpu().float().contiguous() - n, c, h, w = x.shape - x_nhwc = x.permute(0, 2, 3, 1).contiguous() - save_raw(path, x_nhwc.numpy(), (c, w, h, n)) - - -def save_ggml_1d(path: str, x: torch.Tensor) -> None: - """Save a 1D tensor.""" - x = x.detach().cpu().float().contiguous() - save_raw(path, x.numpy(), tuple(x.shape)) - - -def load_tensor(path: str) -> torch.Tensor: - with open(path + ".shape", "r") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32).reshape(shape) - return torch.from_numpy(data) - - -# ── Sinusoidal position encoding (matches PositionEmbeddingSine) ────────── - -def sine_encode_xy(x: torch.Tensor, y: torch.Tensor, num_pos_feats: int = 128, - temperature: int = 10000, scale: float = 2 * math.pi): - """Encode (x, y) with sinusoidal PE. x, y: [N] normalized to [0,1].""" - x_embed = x * scale - y_embed = y * scale - - dim_t = torch.arange(num_pos_feats, dtype=torch.float32, device=x.device) - dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) - - pos_x = x_embed[:, None] / dim_t # [N, 128] - pos_y = y_embed[:, None] / dim_t # [N, 128] - pos_x = torch.stack((pos_x[:, 0::2].sin(), pos_x[:, 1::2].cos()), dim=2).flatten(1) # [N, 128] - pos_y = torch.stack((pos_y[:, 0::2].sin(), pos_y[:, 1::2].cos()), dim=2).flatten(1) # [N, 128] - return pos_x, pos_y - - -def sine_encode_boxes(cx, cy, w, h, num_pos_feats=128, temperature=10000): - """Encode box (cx, cy, w, h) with sinusoidal PE. Returns [N, 258].""" - pos_x, pos_y = sine_encode_xy(cx, cy, num_pos_feats, temperature) - pos = torch.cat((pos_y, pos_x, h[:, None], w[:, None]), dim=1) # [N, 258] - return pos - - -# ── Box coordinate conversion ────────────────────────────────────────────── - -def box_cxcywh_to_xyxy(x): - """Convert CxCyWH to XYXY format.""" - cx, cy, w, h = x.unbind(-1) - x0 = cx - 0.5 * w - y0 = cy - 0.5 * h - x1 = cx + 0.5 * w - y1 = cy + 0.5 * h - return torch.stack([x0, y0, x1, y1], dim=-1) - - -# ── ROI Align implementation ──────────────────────────────────────────────── - -def do_roi_align(img_feats_nchw, boxes_cxcywh, roi_size=7): - """ - ROI Align from image features using box coordinates in CxCyWH [0,1] format. - - Args: - img_feats_nchw: [B, C, H, W] backbone feature map - boxes_cxcywh: [N_boxes, B, 4] in normalized CxCyWH format - roi_size: output spatial size - - Returns: - roi_pooled: [N_boxes*B, C, roi_size, roi_size] - """ - n_boxes, bs, _ = boxes_cxcywh.shape - H, W = img_feats_nchw.shape[-2:] - - boxes_xyxy = box_cxcywh_to_xyxy(boxes_cxcywh) - scale = torch.tensor([W, H, W, H], dtype=boxes_xyxy.dtype, device=boxes_xyxy.device) - scale = scale.view(1, 1, 4) - boxes_xyxy = boxes_xyxy * scale - - # roi_align expects list of [N_boxes, 4] per batch element - rois_list = boxes_xyxy.float().transpose(0, 1).unbind(0) - sampled = torchvision.ops.roi_align(img_feats_nchw, rois_list, roi_size) - return sampled # [B*N_boxes, C, roi_size, roi_size] - - -# ── Main dump logic ─────────────────────────────────────────────────────── - -def dump_geometry_encoder( - ckpt: Dict[str, torch.Tensor], - img_feats_hwc: torch.Tensor, # [H*W, B, C] seq-first image features - img_feats_nchw: torch.Tensor, # [B, C, H, W] image features for ROI align - img_pe_hwc: torch.Tensor, # [H*W, B, C] image positional encoding - boxes_cxcywh: torch.Tensor, # [N_boxes, B, 4] normalized CxCyWH [0,1] - box_labels: torch.Tensor, # [N_boxes, B] long (0=pos, 1=neg) - box_mask: torch.Tensor, # [B, N_boxes] bool (True=padded) - outdir: str, - case_name: str, -): - D = 256 - roi_size = 7 - case_dir = os.path.join(outdir, case_name) - os.makedirs(case_dir, exist_ok=True) - - n_boxes, bs = boxes_cxcywh.shape[:2] - print(f"\n=== Geometry Encoder: {case_name} ===") - print(f" boxes: {n_boxes}, batch: {bs}, img_feats: {img_feats_hwc.shape}") - - # Save input coordinates - save_raw(os.path.join(case_dir, "input_boxes_cxcywh"), - boxes_cxcywh.detach().cpu().float().numpy(), - boxes_cxcywh.shape) - save_raw(os.path.join(case_dir, "input_box_labels"), - box_labels.detach().cpu().float().numpy(), - box_labels.shape) - - # ── 1. Direct box projection: Linear(4, 256) ─────────────────────── - box_proj_w = ckpt["detector.geometry_encoder.boxes_direct_project.weight"].float() - box_proj_b = ckpt["detector.geometry_encoder.boxes_direct_project.bias"].float() - boxes_direct = F.linear(boxes_cxcywh, box_proj_w, box_proj_b) # [N, B, D] - save_ggml_sbd(os.path.join(case_dir, "boxes_direct_proj"), boxes_direct) - print(f" boxes_direct_proj: {boxes_direct.shape} " - f"mean={boxes_direct.mean():.6f} std={boxes_direct.std():.6f}") - - # ── 2. ROI Align pooled features ──────────────────────────────────── - # img_pre_norm on image features before pooling - img_pre_norm_w = ckpt["detector.geometry_encoder.img_pre_norm.weight"].float() - img_pre_norm_b = ckpt["detector.geometry_encoder.img_pre_norm.bias"].float() - # Apply LayerNorm to seq-first features for pooling (operate on the NCHW version) - img_normed_hwc = F.layer_norm(img_feats_hwc, [D], img_pre_norm_w, img_pre_norm_b) - H, W = img_feats_nchw.shape[-2:] - img_normed_nchw = img_normed_hwc.permute(1, 2, 0).reshape(bs, D, H, W) - - if n_boxes > 0: - roi_pooled = do_roi_align(img_normed_nchw, boxes_cxcywh, roi_size) - # roi_pooled: [B*N_boxes, D, roi_size, roi_size] - - # boxes_pool_project is Conv2d(D, D, roi_size) - pool_proj_w = ckpt["detector.geometry_encoder.boxes_pool_project.weight"].float() - pool_proj_b = ckpt["detector.geometry_encoder.boxes_pool_project.bias"].float() - pool_proj = F.conv2d(roi_pooled, pool_proj_w, pool_proj_b) # [B*N, D, 1, 1] - pool_proj = pool_proj.view(bs, n_boxes, D).transpose(0, 1) # [N, B, D] - save_ggml_sbd(os.path.join(case_dir, "boxes_pool_proj"), pool_proj) - print(f" boxes_pool_proj: {pool_proj.shape} " - f"mean={pool_proj.mean():.6f} std={pool_proj.std():.6f}") - else: - pool_proj = torch.zeros(0, bs, D) - - # ── 3. Box positional encoding: sinusoidal PE + Linear(258, 256) ─── - if n_boxes > 0: - cx = boxes_cxcywh[:, :, 0].flatten() - cy = boxes_cxcywh[:, :, 1].flatten() - w = boxes_cxcywh[:, :, 2].flatten() - h = boxes_cxcywh[:, :, 3].flatten() - pos_enc = sine_encode_boxes(cx, cy, w, h) # [N*B, 258] - pos_enc = pos_enc.view(n_boxes, bs, 258) # [N, B, 258] - save_ggml_sbd(os.path.join(case_dir, "boxes_pos_enc_raw"), pos_enc) - - pos_proj_w = ckpt["detector.geometry_encoder.boxes_pos_enc_project.weight"].float() - pos_proj_b = ckpt["detector.geometry_encoder.boxes_pos_enc_project.bias"].float() - pos_proj = F.linear(pos_enc, pos_proj_w, pos_proj_b) # [N, B, D] - save_ggml_sbd(os.path.join(case_dir, "boxes_pos_proj"), pos_proj) - print(f" boxes_pos_proj: {pos_proj.shape} " - f"mean={pos_proj.mean():.6f} std={pos_proj.std():.6f}") - else: - pos_proj = torch.zeros(0, bs, D) - - # ── 4. Label embedding ─────────────────────────────────────────────── - label_embed_w = ckpt["detector.geometry_encoder.label_embed.weight"].float() # [2, D] - if n_boxes > 0: - type_embed = F.embedding(box_labels.long(), label_embed_w) # [N, B, D] - save_ggml_sbd(os.path.join(case_dir, "label_embed"), type_embed) - print(f" label_embed: {type_embed.shape}") - else: - type_embed = torch.zeros(0, bs, D) - - # ── 5. Combined box embedding ──────────────────────────────────────── - if n_boxes > 0: - boxes_embed = boxes_direct + pool_proj + pos_proj + type_embed # [N, B, D] - else: - boxes_embed = torch.zeros(0, bs, D) - save_ggml_sbd(os.path.join(case_dir, "boxes_combined_embed"), boxes_embed) - print(f" boxes_combined: {boxes_embed.shape}") - - # ── 6. CLS token ──────────────────────────────────────────────────── - cls_embed_w = ckpt["detector.geometry_encoder.cls_embed.weight"].float() # [1, D] - cls = cls_embed_w.view(1, 1, D).repeat(1, bs, 1) # [1, B, D] - cls_mask = torch.zeros(bs, 1, dtype=torch.bool) - - # Concatenate: [boxes, CLS] - # Inline concat_padded_sequences for simplicity (no mask padding needed here) - if n_boxes > 0: - final_embeds = torch.cat([boxes_embed, cls], dim=0) # [N+1, B, D] - final_mask = torch.cat([box_mask, cls_mask], dim=1) # [B, N+1] - else: - final_embeds = cls # [1, B, D] - final_mask = cls_mask # [B, 1] - save_ggml_sbd(os.path.join(case_dir, "pre_final_proj"), final_embeds) - print(f" pre_final_proj: {final_embeds.shape}") - - # ── 7. Final projection + norm ────────────────────────────────────── - final_proj_w = ckpt["detector.geometry_encoder.final_proj.weight"].float() - final_proj_b = ckpt["detector.geometry_encoder.final_proj.bias"].float() - norm_w = ckpt["detector.geometry_encoder.norm.weight"].float() - norm_b = ckpt["detector.geometry_encoder.norm.bias"].float() - - final_embeds = F.layer_norm( - F.linear(final_embeds, final_proj_w, final_proj_b), - [D], norm_w, norm_b - ) - save_ggml_sbd(os.path.join(case_dir, "post_final_proj"), final_embeds) - print(f" post_final_proj: {final_embeds.shape}") - - # ── 8. Transformer layers (3 layers) ───────────────────────────────── - for layer_idx in range(3): - lp = f"detector.geometry_encoder.encode.{layer_idx}." - - # Self-attention (pre-norm, pos_enc_at_attn=False) - shortcut = final_embeds - xn = F.layer_norm(final_embeds, [D], - ckpt[lp + "norm1.weight"].float(), - ckpt[lp + "norm1.bias"].float()) - - # Self-attention with fused in_proj - sa_w = ckpt[lp + "self_attn.in_proj_weight"].float() # [3D, D] - sa_b = ckpt[lp + "self_attn.in_proj_bias"].float() # [3D] - sa_ow = ckpt[lp + "self_attn.out_proj.weight"].float() - sa_ob = ckpt[lp + "self_attn.out_proj.bias"].float() - - # pos_enc_at_attn=False, so Q=K=V=xn (no pos added) - # key_padding_mask = final_mask - S, B, _ = xn.shape - qkv = F.linear(xn, sa_w, sa_b) # [S, B, 3D] - q, k, v = qkv.chunk(3, dim=-1) - n_heads = 8 - hd = D // n_heads - - q = q.reshape(S, B, n_heads, hd).permute(1, 2, 0, 3) - k = k.reshape(S, B, n_heads, hd).permute(1, 2, 0, 3) - v = v.reshape(S, B, n_heads, hd).permute(1, 2, 0, 3) - - # Apply key_padding_mask - attn_mask = None - if final_mask.any(): - # final_mask: [B, S] True=padded -> attn_mask: [B, 1, 1, S] with -inf - attn_mask = final_mask.unsqueeze(1).unsqueeze(2).float() * (-1e9) - - sa_out = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask) - sa_out = sa_out.permute(2, 0, 1, 3).reshape(S, B, D) - sa_out = F.linear(sa_out, sa_ow, sa_ob) - final_embeds = shortcut + sa_out - - save_ggml_sbd(os.path.join(case_dir, f"layer{layer_idx}_after_sa"), final_embeds) - - # Cross-attention (pre-norm, pos_enc_at_cross_attn_keys=True) - shortcut = final_embeds - xn = F.layer_norm(final_embeds, [D], - ckpt[lp + "norm2.weight"].float(), - ckpt[lp + "norm2.bias"].float()) - - ca_w = ckpt[lp + "cross_attn_image.in_proj_weight"].float() # [3D, D] - ca_b = ckpt[lp + "cross_attn_image.in_proj_bias"].float() - ca_ow = ckpt[lp + "cross_attn_image.out_proj.weight"].float() - ca_ob = ckpt[lp + "cross_attn_image.out_proj.bias"].float() - - # Q from xn (no pos), K from img_feats + img_pe, V from img_feats - q_w, k_w, v_w = ca_w.chunk(3, dim=0) - q_b, k_b, v_b = ca_b.chunk(3, dim=0) - - # pos_enc_at_cross_attn_queries=False, so Q = proj(xn) - # pos_enc_at_cross_attn_keys=True, so K = proj(img_feats + img_pe) - q_proj = F.linear(xn, q_w, q_b) # [S_q, B, D] - k_input = img_feats_hwc + img_pe_hwc # [S_kv, B, D] - k_proj = F.linear(k_input, k_w, k_b) # [S_kv, B, D] - v_proj = F.linear(img_feats_hwc, v_w, v_b) # [S_kv, B, D] - - S_q = q_proj.shape[0] - S_kv = k_proj.shape[0] - q_proj = q_proj.reshape(S_q, B, n_heads, hd).permute(1, 2, 0, 3) - k_proj = k_proj.reshape(S_kv, B, n_heads, hd).permute(1, 2, 0, 3) - v_proj = v_proj.reshape(S_kv, B, n_heads, hd).permute(1, 2, 0, 3) - - ca_out = F.scaled_dot_product_attention(q_proj, k_proj, v_proj) - ca_out = ca_out.permute(2, 0, 1, 3).reshape(S_q, B, D) - ca_out = F.linear(ca_out, ca_ow, ca_ob) - final_embeds = shortcut + ca_out - - save_ggml_sbd(os.path.join(case_dir, f"layer{layer_idx}_after_ca"), final_embeds) - - # FFN (pre-norm, ReLU) - shortcut = final_embeds - xn = F.layer_norm(final_embeds, [D], - ckpt[lp + "norm3.weight"].float(), - ckpt[lp + "norm3.bias"].float()) - ffn = F.linear(xn, ckpt[lp + "linear1.weight"].float(), - ckpt[lp + "linear1.bias"].float()) - ffn = F.relu(ffn) - ffn = F.linear(ffn, ckpt[lp + "linear2.weight"].float(), - ckpt[lp + "linear2.bias"].float()) - final_embeds = shortcut + ffn - - save_ggml_sbd(os.path.join(case_dir, f"layer{layer_idx}_after_ffn"), final_embeds) - print(f" layer{layer_idx}: shape={final_embeds.shape} " - f"mean={final_embeds.mean():.6f} std={final_embeds.std():.6f}") - - # ── 9. Final encode norm ──────────────────────────────────────────── - enc_norm_w = ckpt["detector.geometry_encoder.encode_norm.weight"].float() - enc_norm_b = ckpt["detector.geometry_encoder.encode_norm.bias"].float() - final_embeds = F.layer_norm(final_embeds, [D], enc_norm_w, enc_norm_b) - save_ggml_sbd(os.path.join(case_dir, "geom_output"), final_embeds) - print(f" geom_output: {final_embeds.shape} " - f"mean={final_embeds.mean():.6f} std={final_embeds.std():.6f}") - - # Also save the mask - save_raw(os.path.join(case_dir, "geom_mask"), - final_mask.cpu().float().numpy(), final_mask.shape) - - return final_embeds, final_mask - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--prephase-ref", required=True, - help="Phase 3 reference dir with neck features") - parser.add_argument("--outdir", required=True) - args = parser.parse_args() - - D = 256 - H = 72 - - # Load checkpoint - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=True) - - # Load backbone features from Phase 3 reference (saved in NCHW format) - print("Loading Phase 3 reference features...") - neck_det_2 = load_tensor(os.path.join(args.prephase_ref, "neck_det_2")) # [1, 256, 72, 72] - print(f" neck_det_2: {neck_det_2.shape}") - - # Image features in sequence-first format [H*W, B, C] - img_feats_hwc = neck_det_2.flatten(2).permute(2, 0, 1) # [H*W, B, C] - - # Image positional encoding [H*W, B, C] — sinusoidal PE computed on the fly - def compute_sine_pe(h, w, num_pos_feats=128, temperature=10000, scale=2*math.pi): - """Sinusoidal PE matching SAM3's PositionEmbeddingSine.""" - not_mask = torch.ones(1, h, w, dtype=torch.float32) - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - eps = 1e-6 - y_embed = (y_embed - 0.5) / (y_embed[:, -1:, :] + eps) * scale - x_embed = (x_embed - 0.5) / (x_embed[:, :, -1:] + eps) * scale - - dim_t = torch.arange(num_pos_feats, dtype=torch.float32) - dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) - - pos_x = x_embed[:, :, :, None] / dim_t - pos_y = y_embed[:, :, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) - pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) # [1, D, H, W] - return pos - - img_pe_nchw = compute_sine_pe(H, H, num_pos_feats=D // 2) # [1, 256, 72, 72] - img_pe_hwc = img_pe_nchw.flatten(2).permute(2, 0, 1) # [H*W, 1, C] - print(f" img_pe: {img_pe_hwc.shape}") - - # ══════════════════════════════════════════════════════════════════════ - # Test Case 1: Dummy prompt (no exemplars, just CLS) - # ══════════════════════════════════════════════════════════════════════ - dummy_boxes = torch.zeros(0, 1, 4) - dummy_labels = torch.zeros(0, 1, dtype=torch.long) - dummy_mask = torch.zeros(1, 0, dtype=torch.bool) - - dump_geometry_encoder( - ckpt, img_feats_hwc, neck_det_2, img_pe_hwc, - dummy_boxes, dummy_labels, dummy_mask, - args.outdir, "dummy_prompt" - ) - - # ══════════════════════════════════════════════════════════════════════ - # Test Case 2: Single positive exemplar box - # Box at center of image: cx=0.5, cy=0.5, w=0.3, h=0.3 - # ══════════════════════════════════════════════════════════════════════ - single_box = torch.tensor([[[0.5, 0.5, 0.3, 0.3]]]) # [1, 1, 4] - single_label = torch.tensor([[0]], dtype=torch.long) # [1, 1] positive=0 - single_mask = torch.tensor([[False]]) # [1, 1] - - dump_geometry_encoder( - ckpt, img_feats_hwc, neck_det_2, img_pe_hwc, - single_box, single_label, single_mask, - args.outdir, "single_box" - ) - - # ══════════════════════════════════════════════════════════════════════ - # Test Case 3: Two exemplar boxes (one positive, one negative) - # ══════════════════════════════════════════════════════════════════════ - two_boxes = torch.tensor([ - [[0.3, 0.4, 0.2, 0.25]], # box 1: positive - [[0.7, 0.6, 0.15, 0.2]], # box 2: negative - ]) # [2, 1, 4] - two_labels = torch.tensor([[0], [1]], dtype=torch.long) # [2, 1] - two_mask = torch.tensor([[False, False]]) # [1, 2] - - dump_geometry_encoder( - ckpt, img_feats_hwc, neck_det_2, img_pe_hwc, - two_boxes, two_labels, two_mask, - args.outdir, "two_boxes" - ) - - print(f"\nAll reference tensors saved to {args.outdir}/") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_phase3_reference.py b/tests/dump_phase3_reference.py deleted file mode 100644 index c5197c9..0000000 --- a/tests/dump_phase3_reference.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python3 -""" -Phase 3 Numerical Audit — dump ALL intermediate tensors from the ViT backbone, -neck (SimpleFPN), sinusoidal PE, and RoPE for numerical comparison against C++. - -Usage: - uv run python tests/dump_phase3_reference.py \ - --checkpoint raw_weights/sam3.pt \ - --image tests/test_random.jpg \ - --outdir tests/ref_phase3 -""" -import argparse -import math -import os -import sys -from functools import partial -from typing import Tuple - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from PIL import Image -from torchvision.transforms import v2 - - -# ── Helpers ──────────────────────────────────────────────────────────────── - -def save_tensor(path, t): - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)} " - f"min={t.min().item():.6f} max={t.max().item():.6f} mean={t.mean().item():.6f}") - - -# ── Inlined from vitdet.py ───────────────────────────────────────────────── - -def init_t_xy(end_x, end_y, scale=1.0, offset=0): - t = torch.arange(end_x * end_y, dtype=torch.float32) - t_x = (t % end_x).float() - t_y = torch.div(t, end_x, rounding_mode="floor").float() - return t_x * scale + offset, t_y * scale + offset - - -def compute_axial_cis(dim, end_x, end_y, theta=10000.0, scale_pos=1.0, offset=0): - freqs_x = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - freqs_y = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - t_x, t_y = init_t_xy(end_x, end_y, scale_pos, offset) - freqs_x = torch.outer(t_x, freqs_x) - freqs_y = torch.outer(t_y, freqs_y) - freqs_cis_x = torch.polar(torch.ones_like(freqs_x), freqs_x) - freqs_cis_y = torch.polar(torch.ones_like(freqs_y), freqs_y) - return torch.cat([freqs_cis_x, freqs_cis_y], dim=-1) - - -def reshape_for_broadcast(freqs_cis, x): - ndim = x.ndim - shape = [d if i >= ndim - 2 else 1 for i, d in enumerate(x.shape)] - return freqs_cis.view(*shape) - - -def apply_rotary_enc(xq, xk, freqs_cis, repeat_freqs_k=False): - xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) if xk.shape[-2] != 0 else None - freqs_cis = reshape_for_broadcast(freqs_cis, xq_) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) - if xk_ is None: - return xq_out.type_as(xq), xk - if repeat_freqs_k: - r = xk_.shape[-2] // xq_.shape[-2] - freqs_cis = freqs_cis.repeat(*([1] * (freqs_cis.ndim - 2)), r, 1) - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) - return xq_out.type_as(xq), xk_out.type_as(xk) - - -def window_partition(x, window_size): - B, H, W, C = x.shape - pad_h = (window_size - H % window_size) % window_size - pad_w = (window_size - W % window_size) % window_size - if pad_h > 0 or pad_w > 0: - x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) - Hp, Wp = H + pad_h, W + pad_w - x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) - windows = x.permute(0, 1, 3, 2, 4, 5).reshape(-1, window_size, window_size, C) - return windows, (Hp, Wp) - - -def window_unpartition(windows, window_size, pad_hw, hw): - Hp, Wp = pad_hw - H, W = hw - B = windows.shape[0] // (Hp * Wp // window_size // window_size) - x = windows.reshape(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) - x = x.permute(0, 1, 3, 2, 4, 5).reshape(B, Hp, Wp, -1) - if Hp > H or Wp > W: - x = x[:, :H, :W, :] - return x - - -def get_abs_pos(abs_pos, has_cls_token, hw, tiling=False): - h, w = hw - if has_cls_token: - abs_pos = abs_pos[:, 1:] - xy_num = abs_pos.shape[1] - size = int(math.sqrt(xy_num)) - if size != h or size != w: - new_abs_pos = abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2) - if tiling: - new_abs_pos = new_abs_pos.tile( - [1, 1] + [x // y + 1 for x, y in zip((h, w), new_abs_pos.shape[2:])] - )[:, :, :h, :w] - else: - new_abs_pos = F.interpolate(new_abs_pos, size=(h, w), mode="bicubic", align_corners=False) - return new_abs_pos.permute(0, 2, 3, 1) - return abs_pos.reshape(1, h, w, -1) - - -# ── Sinusoidal PE (from position_encoding.py) ──────────────────────────── - -def sinusoidal_pe_2d(H, W, num_pos_feats=256, temperature=10000, scale=2*math.pi): - """Match PositionEmbeddingSine.forward() exactly.""" - half = num_pos_feats // 2 # 128 - - y_embed = torch.arange(1, H + 1, dtype=torch.float32).view(-1, 1).repeat(1, W) - x_embed = torch.arange(1, W + 1, dtype=torch.float32).view(1, -1).repeat(H, 1) - - # Normalize - eps = 1e-6 - y_embed = y_embed / (y_embed[-1:, :] + eps) * scale - x_embed = x_embed / (x_embed[:, -1:] + eps) * scale - - dim_t = torch.arange(half, dtype=torch.float32) - dim_t = temperature ** (2 * (dim_t // 2) / half) - - pos_x = x_embed[:, :, None] / dim_t # [H, W, half] - pos_y = y_embed[:, :, None] / dim_t - - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - - # [H, W, 256] → [1, 256, H, W] - pos = torch.cat((pos_y, pos_x), dim=2).permute(2, 0, 1).unsqueeze(0) - return pos - - -# ═══════════════════════════════════════════════════════════════════════════ -# Main -# ═══════════════════════════════════════════════════════════════════════════ - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--image", required=True) - parser.add_argument("--outdir", default="tests/ref_phase3") - parser.add_argument("--blocks", default="0,1,7,15,23,31", - help="Comma-separated block indices to dump detailed intermediates") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" - dump_blocks = set(int(b) for b in args.blocks.split(",")) - - # ── Load checkpoint ───────────────────────────────────────────────── - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - vit_prefix = "detector.backbone.vision_backbone.trunk." - vit_sd = {k[len(vit_prefix):]: v for k, v in ckpt.items() if k.startswith(vit_prefix)} - print(f" Found {len(vit_sd)} ViT keys") - - # ── Step 3.4: RoPE ───────────────────────────────────────────────── - print("\n=== RoPE Frequencies ===") - - # Window RoPE (24x24, scale_pos=1.0) - window_freqs = compute_axial_cis(dim=64, end_x=24, end_y=24, theta=10000.0, scale_pos=1.0) - # Store as real: [576, 32, 2] (cos, sin) - window_real = torch.view_as_real(window_freqs) - save_tensor(os.path.join(args.outdir, "rope_window_real"), window_real) - - # Global RoPE (72x72, scale_pos=24/72=1/3) - global_freqs = compute_axial_cis(dim=64, end_x=72, end_y=72, theta=10000.0, scale_pos=24.0/72.0) - global_real = torch.view_as_real(global_freqs) - save_tensor(os.path.join(args.outdir, "rope_global_real"), global_real) - - # Also dump the checkpoint's freqs_cis for block 0 (window) and block 7 (global) - for bi in [0, 7]: - key = f"blocks.{bi}.attn.freqs_cis" - if key in vit_sd: - fc = vit_sd[key] - fc_real = torch.view_as_real(fc) if fc.is_complex() else fc - save_tensor(os.path.join(args.outdir, f"ckpt_freqs_cis_block{bi}"), fc_real) - - # ── Step 3.9: Sinusoidal PE ───────────────────────────────────────── - print("\n=== Sinusoidal PE ===") - for S, name in [(288, "pe_288"), (144, "pe_144"), (72, "pe_72"), (36, "pe_36")]: - pe = sinusoidal_pe_2d(S, S, num_pos_feats=256) - save_tensor(os.path.join(args.outdir, name), pe) - - # ── Load and preprocess image ───────────────────────────────────── - print(f"\n=== Preprocessing: {args.image} ===") - img = Image.open(args.image).convert("RGB") - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - img_tensor = v2.functional.to_image(img) - img_preprocessed = transform(img_tensor).unsqueeze(0).to(device) - save_tensor(os.path.join(args.outdir, "preprocessed"), img_preprocessed) - - # Also save the raw CHW float image before normalization (for debugging preprocessing) - raw_transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - ]) - img_raw_float = raw_transform(v2.functional.to_image(img)).unsqueeze(0) - save_tensor(os.path.join(args.outdir, "preprocessed_raw_float"), img_raw_float) - - # ── Manual ViT forward pass ─────────────────────────────────────── - print("\n=== ViT Forward Pass ===") - with torch.no_grad(): - E = 1024 - NH = 16 - HD = 64 - depth = 32 - MLP_DIM = 4736 - WS = 24 - global_blocks = {7, 15, 23, 31} - - # Step 3.2: Patch embedding - patch_w = vit_sd["patch_embed.proj.weight"] # [1024, 3, 14, 14] - x = F.conv2d(img_preprocessed, patch_w, stride=14) # [1, 1024, 72, 72] - x = x.permute(0, 2, 3, 1) # [1, 72, 72, 1024] - save_tensor(os.path.join(args.outdir, "patch_embed"), x) - - # Step 3.3: Positional embedding (tiled) - pos_embed = vit_sd["pos_embed"] # [1, 577, 1024] - pos = get_abs_pos(pos_embed, has_cls_token=True, hw=(72, 72), tiling=True) - save_tensor(os.path.join(args.outdir, "pos_embed_tiled"), pos) - x = x + pos - save_tensor(os.path.join(args.outdir, "after_pos_embed"), x) - - # ln_pre - ln_pre_w = vit_sd["ln_pre.weight"] - ln_pre_b = vit_sd["ln_pre.bias"] - x = F.layer_norm(x, [E], ln_pre_w, ln_pre_b, eps=1e-5) - save_tensor(os.path.join(args.outdir, "after_ln_pre"), x) - - # Step 3.6 & 3.7: Transformer blocks - for blk_idx in range(depth): - prefix = f"blocks.{blk_idx}." - is_global = blk_idx in global_blocks - ws = 0 if is_global else WS - dump_this = blk_idx in dump_blocks - blk_prefix = f"block_{blk_idx}" - - # Pre-norm - n1_w = vit_sd[prefix + "norm1.weight"] - n1_b = vit_sd[prefix + "norm1.bias"] - shortcut = x - xn = F.layer_norm(x, [E], n1_w, n1_b, eps=1e-5) - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_after_norm1"), xn) - - # Window partition - if ws > 0: - H, W = xn.shape[1], xn.shape[2] - xn, pad_hw = window_partition(xn, ws) - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_after_winpart"), xn) - - # Attention - B_cur, Hc, Wc, _ = xn.shape - L = Hc * Wc - qkv_w = vit_sd[prefix + "attn.qkv.weight"] - qkv_b = vit_sd[prefix + "attn.qkv.bias"] - qkv = F.linear(xn, qkv_w, qkv_b).reshape(B_cur, L, 3, NH, HD) - q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0) # [B, NH, L, HD] - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_q_pre_rope"), q) - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_k_pre_rope"), k) - - # RoPE - freqs_key = prefix + "attn.freqs_cis" - if freqs_key in vit_sd: - freqs = vit_sd[freqs_key] - q, k = apply_rotary_enc(q, k, freqs) - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_q_post_rope"), q) - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_k_post_rope"), k) - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_v"), v) - - # Scaled dot-product attention - attn_out = F.scaled_dot_product_attention(q, k, v) - attn_out = attn_out.view(B_cur, NH, Hc, Wc, HD).permute(0, 2, 3, 1, 4).reshape(B_cur, Hc, Wc, E) - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_attn_out"), attn_out) - - # Output projection - proj_w = vit_sd[prefix + "attn.proj.weight"] - proj_b = vit_sd[prefix + "attn.proj.bias"] - attn_out = F.linear(attn_out, proj_w, proj_b) - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_proj_out"), attn_out) - - # Window unpartition - if ws > 0: - attn_out = window_unpartition(attn_out, ws, pad_hw, (H, W)) - - # Residual - x = shortcut + attn_out - - if dump_this: - save_tensor(os.path.join(args.outdir, f"{blk_prefix}_after_attn_residual"), x) - - # FFN - n2_w = vit_sd[prefix + "norm2.weight"] - n2_b = vit_sd[prefix + "norm2.bias"] - shortcut = x - xn = F.layer_norm(x, [E], n2_w, n2_b, eps=1e-5) - - fc1_w = vit_sd[prefix + "mlp.fc1.weight"] - fc1_b = vit_sd[prefix + "mlp.fc1.bias"] - fc2_w = vit_sd[prefix + "mlp.fc2.weight"] - fc2_b = vit_sd[prefix + "mlp.fc2.bias"] - h = F.linear(xn, fc1_w, fc1_b) - h = F.gelu(h) - h = F.linear(h, fc2_w, fc2_b) - - x = shortcut + h - - # Always save per-block output for selected blocks - save_tensor(os.path.join(args.outdir, f"block_{blk_idx}_out"), x) - - # Final output - save_tensor(os.path.join(args.outdir, "vit_final"), x) - feats = x.permute(0, 3, 1, 2) # [1, 1024, 72, 72] - save_tensor(os.path.join(args.outdir, "vit_output_bchw"), feats) - - # ── Neck (SimpleFPN) — detector path ────────────────────────────── - print("\n=== Neck (Detector) ===") - neck_prefix = "detector.backbone.vision_backbone.convs." - neck_sd = {k[len(neck_prefix):]: v for k, v in ckpt.items() if k.startswith(neck_prefix)} - - with torch.no_grad(): - # Scale 0 (4x) - s0 = F.conv_transpose2d(feats, neck_sd["0.dconv_2x2_0.weight"], neck_sd["0.dconv_2x2_0.bias"], stride=2) - save_tensor(os.path.join(args.outdir, "neck_det_s0_deconv1"), s0) - s0 = F.gelu(s0) - save_tensor(os.path.join(args.outdir, "neck_det_s0_gelu"), s0) - s0 = F.conv_transpose2d(s0, neck_sd["0.dconv_2x2_1.weight"], neck_sd["0.dconv_2x2_1.bias"], stride=2) - save_tensor(os.path.join(args.outdir, "neck_det_s0_deconv2"), s0) - s0 = F.conv2d(s0, neck_sd["0.conv_1x1.weight"], neck_sd["0.conv_1x1.bias"]) - save_tensor(os.path.join(args.outdir, "neck_det_s0_conv1x1"), s0) - s0 = F.conv2d(s0, neck_sd["0.conv_3x3.weight"], neck_sd["0.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_0"), s0) - - # Scale 1 (2x) - s1 = F.conv_transpose2d(feats, neck_sd["1.dconv_2x2.weight"], neck_sd["1.dconv_2x2.bias"], stride=2) - save_tensor(os.path.join(args.outdir, "neck_det_s1_deconv"), s1) - s1 = F.conv2d(s1, neck_sd["1.conv_1x1.weight"], neck_sd["1.conv_1x1.bias"]) - save_tensor(os.path.join(args.outdir, "neck_det_s1_conv1x1"), s1) - s1 = F.conv2d(s1, neck_sd["1.conv_3x3.weight"], neck_sd["1.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_1"), s1) - - # Scale 2 (1x) - s2 = F.conv2d(feats, neck_sd["2.conv_1x1.weight"], neck_sd["2.conv_1x1.bias"]) - save_tensor(os.path.join(args.outdir, "neck_det_s2_conv1x1"), s2) - s2 = F.conv2d(s2, neck_sd["2.conv_3x3.weight"], neck_sd["2.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_2"), s2) - - # Scale 3 (0.5x) - s3 = F.max_pool2d(feats, kernel_size=2, stride=2) - save_tensor(os.path.join(args.outdir, "neck_det_s3_pool"), s3) - s3 = F.conv2d(s3, neck_sd["3.conv_1x1.weight"], neck_sd["3.conv_1x1.bias"]) - save_tensor(os.path.join(args.outdir, "neck_det_s3_conv1x1"), s3) - s3 = F.conv2d(s3, neck_sd["3.conv_3x3.weight"], neck_sd["3.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_3"), s3) - - # ── Neck (SimpleFPN) — tracker path ────────────────────────────── - print("\n=== Neck (Tracker) ===") - trk_prefix = "detector.backbone.vision_backbone.sam2_convs." - trk_sd = {k[len(trk_prefix):]: v for k, v in ckpt.items() if k.startswith(trk_prefix)} - - if trk_sd: - with torch.no_grad(): - s0t = F.conv_transpose2d(feats, trk_sd["0.dconv_2x2_0.weight"], trk_sd["0.dconv_2x2_0.bias"], stride=2) - s0t = F.gelu(s0t) - s0t = F.conv_transpose2d(s0t, trk_sd["0.dconv_2x2_1.weight"], trk_sd["0.dconv_2x2_1.bias"], stride=2) - s0t = F.conv2d(s0t, trk_sd["0.conv_1x1.weight"], trk_sd["0.conv_1x1.bias"]) - s0t = F.conv2d(s0t, trk_sd["0.conv_3x3.weight"], trk_sd["0.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_trk_0"), s0t) - - s1t = F.conv_transpose2d(feats, trk_sd["1.dconv_2x2.weight"], trk_sd["1.dconv_2x2.bias"], stride=2) - s1t = F.conv2d(s1t, trk_sd["1.conv_1x1.weight"], trk_sd["1.conv_1x1.bias"]) - s1t = F.conv2d(s1t, trk_sd["1.conv_3x3.weight"], trk_sd["1.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_trk_1"), s1t) - - s2t = F.conv2d(feats, trk_sd["2.conv_1x1.weight"], trk_sd["2.conv_1x1.bias"]) - s2t = F.conv2d(s2t, trk_sd["2.conv_3x3.weight"], trk_sd["2.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_trk_2"), s2t) - - s3t = F.max_pool2d(feats, kernel_size=2, stride=2) - s3t = F.conv2d(s3t, trk_sd["3.conv_1x1.weight"], trk_sd["3.conv_1x1.bias"]) - s3t = F.conv2d(s3t, trk_sd["3.conv_3x3.weight"], trk_sd["3.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_trk_3"), s3t) - else: - print(" No tracker neck weights found (sam2_convs); skipping.") - - print("\n=== Done! ===") - print(f"All tensors saved to: {args.outdir}") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_phase4_reference.py b/tests/dump_phase4_reference.py deleted file mode 100644 index f3efb39..0000000 --- a/tests/dump_phase4_reference.py +++ /dev/null @@ -1,257 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump Phase 4 text encoder reference tensors for numerical comparison. - -Usage: - uv run python tests/dump_phase4_reference.py \ - --checkpoint raw_weights/sam3.pt \ - --tokenizer raw_weights/tokenizer.json \ - --prompts tests/phase4_prompts.tsv \ - --outdir tests/ref_phase4 -""" - -import argparse -import os -from typing import List, Tuple - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from tokenizers import Tokenizer - - -def save_tensor(path: str, t: torch.Tensor) -> None: - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - - -def save_i32_tensor(path: str, values: List[int]) -> None: - arr = np.asarray(values, dtype=np.int32) - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(arr.tobytes()) - with open(path + ".shape", "w") as f: - f.write(str(arr.shape[0])) - - -def load_prompt_cases(path: str) -> List[Tuple[str, str]]: - cases: List[Tuple[str, str]] = [] - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.rstrip("\n") - if not line: - continue - if "\t" in line: - prompt_id, text = line.split("\t", 1) - else: - prompt_id, text = f"prompt_{len(cases):02d}", line - cases.append((prompt_id, text)) - return cases - - -def save_ggml_2d(path: str, t_td: torch.Tensor) -> None: - # ggml stores a [D, T] tensor with D contiguous. A contiguous PyTorch [T, D] - # tensor has the same byte order, so we write those bytes but record ggml dims. - assert t_td.ndim == 2 - t_td = t_td.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t_td.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(f"{t_td.shape[1]},{t_td.shape[0]}") - -def save_ggml_2d_batch_first(path: str, x: torch.Tensor) -> None: - assert x.ndim == 3 and x.shape[0] == 1 - save_ggml_2d(path, x[0]) - - -def save_ggml_2d_seq_first(path: str, x: torch.Tensor) -> None: - assert x.ndim == 3 and x.shape[1] == 1 - save_ggml_2d(path, x[:, 0, :]) - - -def tokenize_prompt(tokenizer: Tokenizer, text: str, context_length: int) -> List[int]: - sot_token = 49406 - eot_token = 49407 - - token_ids = list(tokenizer.encode(text).ids) - if not token_ids or token_ids[0] != sot_token: - token_ids = [sot_token] + token_ids - if token_ids[-1] != eot_token: - token_ids.append(eot_token) - - if len(token_ids) > context_length: - token_ids = token_ids[:context_length] - token_ids[-1] = eot_token - while len(token_ids) < context_length: - token_ids.append(0) - return token_ids - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--tokenizer", default="raw_weights/tokenizer.json") - parser.add_argument("--prompts", default="tests/phase4_prompts.tsv") - parser.add_argument("--outdir", default="tests/ref_phase4") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" - - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - text_prefix = "detector.backbone.language_backbone.encoder." - text_sd = {k[len(text_prefix):]: v for k, v in ckpt.items() if k.startswith(text_prefix)} - resizer_prefix = "detector.backbone.language_backbone.resizer." - resizer_sd = { - k[len(resizer_prefix):]: v for k, v in ckpt.items() if k.startswith(resizer_prefix) - } - - tokenizer = Tokenizer.from_file(args.tokenizer) - prompts = load_prompt_cases(args.prompts) - - text_width = 1024 - text_heads = 16 - text_layers = 24 - context_length = 32 - - causal_mask = torch.full((context_length, context_length), float("-inf"), device=device) - causal_mask = torch.triu(causal_mask, diagonal=1) - - with torch.no_grad(): - for prompt_id, prompt_text in prompts: - prompt_dir = os.path.join(args.outdir, prompt_id) - os.makedirs(prompt_dir, exist_ok=True) - - token_ids = tokenize_prompt(tokenizer, prompt_text, context_length) - with open(os.path.join(prompt_dir, "prompt.txt"), "w", encoding="utf-8") as f: - f.write(prompt_text) - - save_i32_tensor(os.path.join(prompt_dir, "token_ids"), token_ids) - save_tensor(os.path.join(prompt_dir, "causal_mask"), causal_mask) - - tokens = torch.tensor([token_ids], dtype=torch.long, device=device) - - x_bf = F.embedding(tokens, text_sd["token_embedding.weight"]) - save_ggml_2d_batch_first(os.path.join(prompt_dir, "text_token_embed"), x_bf) - - x_bf = x_bf + text_sd["positional_embedding"][:context_length] - save_ggml_2d_batch_first(os.path.join(prompt_dir, "text_after_pos_embed"), x_bf) - - x = x_bf.transpose(0, 1).contiguous() # [T, 1, D] - - mha = nn.MultiheadAttention(text_width, text_heads, batch_first=False).to(device) - mha.eval() - - for i in range(text_layers): - prefix = f"transformer.resblocks.{i}." - - xn1 = F.layer_norm( - x, - (text_width,), - text_sd[prefix + "ln_1.weight"], - text_sd[prefix + "ln_1.bias"], - ) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_after_ln1"), - xn1[:, 0, :], - ) - - qkv = F.linear( - xn1, - text_sd[prefix + "attn.in_proj_weight"], - text_sd[prefix + "attn.in_proj_bias"], - ) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_qkv"), - qkv[:, 0, :], - ) - - mha.in_proj_weight.copy_(text_sd[prefix + "attn.in_proj_weight"]) - mha.in_proj_bias.copy_(text_sd[prefix + "attn.in_proj_bias"]) - mha.out_proj.weight.copy_(text_sd[prefix + "attn.out_proj.weight"]) - mha.out_proj.bias.copy_(text_sd[prefix + "attn.out_proj.bias"]) - - attn_out = mha(xn1, xn1, xn1, need_weights=False, attn_mask=causal_mask)[0] - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_attn_out"), - attn_out[:, 0, :], - ) - - x = x + attn_out - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_after_attn_residual"), - x[:, 0, :], - ) - - xn2 = F.layer_norm( - x, - (text_width,), - text_sd[prefix + "ln_2.weight"], - text_sd[prefix + "ln_2.bias"], - ) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_after_ln2"), - xn2[:, 0, :], - ) - - mlp_fc1 = F.linear( - xn2, - text_sd[prefix + "mlp.c_fc.weight"], - text_sd[prefix + "mlp.c_fc.bias"], - ) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_mlp_fc1"), - mlp_fc1[:, 0, :], - ) - - mlp_gelu = F.gelu(mlp_fc1) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_mlp_gelu"), - mlp_gelu[:, 0, :], - ) - - mlp_out = F.linear( - mlp_gelu, - text_sd[prefix + "mlp.c_proj.weight"], - text_sd[prefix + "mlp.c_proj.bias"], - ) - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_mlp_out"), - mlp_out[:, 0, :], - ) - - x = x + mlp_out - save_ggml_2d( - os.path.join(prompt_dir, f"text_block_{i:02d}_out"), - x[:, 0, :], - ) - - x = F.layer_norm( - x, - (text_width,), - text_sd["ln_final.weight"], - text_sd["ln_final.bias"], - ) - save_ggml_2d_seq_first(os.path.join(prompt_dir, "text_final_ln"), x) - - text_features = F.linear(x, resizer_sd["weight"], resizer_sd["bias"]) - save_ggml_2d_seq_first( - os.path.join(prompt_dir, "text_features_2d"), - text_features, - ) - - print(f"[done] {prompt_id}: \"{prompt_text}\"") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_phase5_reference.py b/tests/dump_phase5_reference.py deleted file mode 100644 index 5703671..0000000 --- a/tests/dump_phase5_reference.py +++ /dev/null @@ -1,778 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump Phase 5 (Detector/PCS) reference tensors from SAM3 checkpoint for C++ verification. -Covers: Fusion Encoder, DETR Decoder, DotProductScoring, Segmentation Head. -Self-contained — builds forward pass from raw weights, no triton dependency. - -Usage: - uv run python tests/dump_phase5_reference.py --checkpoint raw_weights/sam3.pt --image tests/test_random.jpg --outdir tests/ref_phase5/ -""" -import argparse -import math -import os -import sys - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from PIL import Image -from torchvision.transforms import v2 - - -def save_tensor(path, t): - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)}") - - -def save_tensor_i32(path, t): - t = t.detach().cpu().to(torch.int32).contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)} dtype=int32") - - -# ── Inlined helpers ───────────────────────────────────────────────────── - -def inverse_sigmoid(x, eps=1e-3): - x = x.clamp(min=0, max=1) - x1 = x.clamp(min=eps) - x2 = (1 - x).clamp(min=eps) - return torch.log(x1 / x2) - - -def gen_sineembed_for_position(pos_tensor, num_feats=256): - """Generate sinusoidal positional embedding for 2D/4D reference points.""" - assert num_feats % 2 == 0 - num_feats = num_feats // 2 - scale = 2 * math.pi - dim_t = torch.arange(num_feats, dtype=torch.float32, device=pos_tensor.device) - dim_t = 10000 ** (2 * (torch.div(dim_t, 2, rounding_mode="floor")) / num_feats) - x_embed = pos_tensor[:, :, 0] * scale - y_embed = pos_tensor[:, :, 1] * scale - pos_x = x_embed[:, :, None] / dim_t - pos_y = y_embed[:, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - if pos_tensor.size(-1) == 2: - pos = torch.cat((pos_y, pos_x), dim=2) - elif pos_tensor.size(-1) == 4: - w_embed = pos_tensor[:, :, 2] * scale - pos_w = w_embed[:, :, None] / dim_t - pos_w = torch.stack((pos_w[:, :, 0::2].sin(), pos_w[:, :, 1::2].cos()), dim=3).flatten(2) - h_embed = pos_tensor[:, :, 3] * scale - pos_h = h_embed[:, :, None] / dim_t - pos_h = torch.stack((pos_h[:, :, 0::2].sin(), pos_h[:, :, 1::2].cos()), dim=3).flatten(2) - pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=2) - else: - raise ValueError(f"Unknown pos_tensor shape(-1):{pos_tensor.size(-1)}") - return pos - - -def box_cxcywh_to_xyxy(x): - x_c, y_c, w, h = x.unbind(-1) - b = [(x_c - 0.5 * w), (y_c - 0.5 * h), (x_c + 0.5 * w), (y_c + 0.5 * h)] - return torch.stack(b, dim=-1) - - -def mlp_forward(x, sd, prefix, num_layers, activation="relu"): - """Forward through MLP layers from state dict.""" - for i in range(num_layers): - w = sd[f"{prefix}.layers.{i}.weight"] - b = sd[f"{prefix}.layers.{i}.bias"] - x = F.linear(x, w, b) - if i < num_layers - 1: - x = F.relu(x) - return x - - -def multihead_attention_forward(query, key, value, w_q, b_q, w_k, b_k, w_v, b_v, - w_out, b_out, num_heads, attn_mask=None, - key_padding_mask=None): - """Manual multi-head attention forward.""" - D = query.shape[-1] - HD = D // num_heads - - Q = F.linear(query, w_q, b_q) - K = F.linear(key, w_k, b_k) - V = F.linear(value, w_v, b_v) - - # [BS, N, D] -> [BS, N, NH, HD] -> [BS, NH, N, HD] - BS = Q.shape[0] - N_q = Q.shape[1] - N_kv = K.shape[1] - - Q = Q.reshape(BS, N_q, num_heads, HD).permute(0, 2, 1, 3) - K = K.reshape(BS, N_kv, num_heads, HD).permute(0, 2, 1, 3) - V = V.reshape(BS, N_kv, num_heads, HD).permute(0, 2, 1, 3) - - combined_mask = attn_mask - if key_padding_mask is not None: - pad_bias = torch.zeros( - BS, 1, 1, N_kv, dtype=Q.dtype, device=Q.device - ) - pad_bias = pad_bias.masked_fill( - key_padding_mask[:, None, None, :], float("-inf") - ) - combined_mask = pad_bias if combined_mask is None else combined_mask + pad_bias - - # Attention - attn_output = F.scaled_dot_product_attention(Q, K, V, attn_mask=combined_mask) - - # [BS, NH, N_q, HD] -> [BS, N_q, D] - attn_output = attn_output.permute(0, 2, 1, 3).reshape(BS, N_q, D) - out = F.linear(attn_output, w_out, b_out) - return out - - -def fused_mha_forward(query, key, value, in_proj_w, in_proj_b, out_w, out_b, - num_heads, attn_mask=None, key_padding_mask=None): - """Multi-head attention with fused in_proj weights.""" - D = query.shape[-1] - w_q, w_k, w_v = in_proj_w[:D], in_proj_w[D:2*D], in_proj_w[2*D:] - b_q, b_k, b_v = in_proj_b[:D], in_proj_b[D:2*D], in_proj_b[2*D:] - return multihead_attention_forward(query, key, value, w_q, b_q, w_k, b_k, w_v, b_v, - out_w, out_b, num_heads, attn_mask, key_padding_mask) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--image", required=True) - parser.add_argument("--outdir", default="tests/ref_phase5") - parser.add_argument("--text", default="yellow school bus") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" - D = 256 - NH = 8 - HD = D // NH - NQ = 200 - T = 32 # text context length - H = 72 # image feature spatial dim (1008 / 14) - - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - # ── Load required weights from specific modules ───────────────────── - - # We need the image features & text features as inputs. - # To get them, we first need to run the ViT backbone + neck + text encoder. - # These were already verified in prior phases, so we load cached reference tensors. - # But we need fresh tensors to match exact C++ inputs. - - # Let's first generate fresh image features using dump_vit_reference approach - print(f"Loading image: {args.image}") - img = Image.open(args.image).convert("RGB") - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - img_tensor = v2.functional.to_image(img) - img_preprocessed = transform(img_tensor).unsqueeze(0).to(device) - - # ── ViT forward (reuse from dump_vit_reference.py) ───────────────── - # ... (this is too long to inline, we'll load pre-computed neck features) - # Instead, we'll load from the reference directory if available - ref_dir = "tests/ref" - neck_det_path = os.path.join(ref_dir, "neck_det_2") - if os.path.exists(neck_det_path + ".bin"): - print("Loading pre-computed neck features from tests/ref/") - # Load neck features - def load_tensor(path): - shape = list(map(int, open(path + ".shape").read().strip().split(","))) - data = np.fromfile(path + ".bin", dtype=np.float32).reshape(shape) - return torch.tensor(data) - - # Neck det features: [1, 256, H, H] for scale 2 (72x72) - neck_det_2 = load_tensor(os.path.join(ref_dir, "neck_det_2")) # [1, 256, 72, 72] - neck_det_1 = load_tensor(os.path.join(ref_dir, "neck_det_1")) # [1, 256, 144, 144] - neck_det_0 = load_tensor(os.path.join(ref_dir, "neck_det_0")) # [1, 256, 288, 288] - else: - print("ERROR: Pre-computed neck features not found. Run dump_vit_reference.py first.") - sys.exit(1) - - # ── Text encoder forward ──────────────────────────────────────────── - # For simplicity, we need text features. The text encoder was verified in phase 4. - # We can compute them here with the same approach. - # For now, load from existing ref or compute inline. - - # Actually, let's compute text features inline since we need them. - # The text encoder is: embedding -> transformer -> projection - - # Tokenize - from tokenizers import Tokenizer - tokenizer_path = os.path.join(os.path.dirname(args.checkpoint), "tokenizer.json") - if os.path.exists(tokenizer_path): - tokenizer = Tokenizer.from_file(tokenizer_path) - else: - tokenizer = Tokenizer.from_file("raw_weights/tokenizer.json") - - encoded = tokenizer.encode(args.text) - # tokenizer already includes SOT/EOT - token_ids = list(encoded.ids) - while len(token_ids) < T: - token_ids.append(0) # pad - token_ids = token_ids[:T] - - tokens_tensor = torch.tensor([token_ids], dtype=torch.long, device=device) # [1, T] - print(f" Token IDs: {token_ids[:10]}...") - save_tensor_i32(os.path.join(args.outdir, "token_ids"), tokens_tensor) - with open(os.path.join(args.outdir, "prompt.txt"), "w") as f: - f.write(args.text + "\n") - - # Text encoder forward - text_prefix = "detector.backbone.language_backbone.encoder." - text_sd = {k[len(text_prefix):]: v for k, v in ckpt.items() if k.startswith(text_prefix)} - text_width = 1024 - text_heads = 16 - text_layers = 24 - - with torch.no_grad(): - # Token + positional embedding - tok_emb = text_sd["token_embedding.weight"] # [49408, 1024] - pos_emb = text_sd["positional_embedding"] # [32, 1024] - x = tok_emb[tokens_tensor] + pos_emb[:T] # [1, T, 1024] - x = x.permute(1, 0, 2) # [T, 1, 1024] - - # Causal mask - causal_mask = torch.full((T, T), float("-inf"), device=device) - causal_mask = torch.triu(causal_mask, diagonal=1) - - # Transformer blocks - for i in range(text_layers): - p = f"transformer.resblocks.{i}." - # Pre-norm SA - xn = F.layer_norm(x, [text_width], text_sd[p + "ln_1.weight"], text_sd[p + "ln_1.bias"]) - # Fused in_proj - qkv_w = text_sd[p + "attn.in_proj_weight"] - qkv_b = text_sd[p + "attn.in_proj_bias"] - out_w = text_sd[p + "attn.out_proj.weight"] - out_b = text_sd[p + "attn.out_proj.bias"] - - # Manual MHA (T, BS, D) -> seq-first - xn_bf = xn.permute(1, 0, 2) # [1, T, 1024] - sa_out = fused_mha_forward(xn_bf, xn_bf, xn_bf, qkv_w, qkv_b, out_w, out_b, - text_heads, attn_mask=causal_mask) - x = x + sa_out.permute(1, 0, 2) - - # Pre-norm FFN - xn = F.layer_norm(x, [text_width], text_sd[p + "ln_2.weight"], text_sd[p + "ln_2.bias"]) - fc1_w = text_sd[p + "mlp.c_fc.weight"] - fc1_b = text_sd[p + "mlp.c_fc.bias"] - fc2_w = text_sd[p + "mlp.c_proj.weight"] - fc2_b = text_sd[p + "mlp.c_proj.bias"] - h = F.linear(xn, fc1_w, fc1_b) - h = F.gelu(h, approximate="tanh") - h = F.linear(h, fc2_w, fc2_b) - x = x + h - - # Final LN - x = F.layer_norm(x, [text_width], text_sd["ln_final.weight"], text_sd["ln_final.bias"]) - # x: [T, 1, 1024] - - # Project to 256-dim via resizer (Linear(1024→256)) - resizer_prefix = "detector.backbone.language_backbone.resizer." - resizer_sd = {k[len(resizer_prefix):]: v for k, v in ckpt.items() if k.startswith(resizer_prefix)} - txt_projected = F.linear(x, resizer_sd["weight"], resizer_sd.get("bias")) - # txt_projected: [T, 1, 256] - - save_tensor(os.path.join(args.outdir, "text_features"), txt_projected) - print(f" Text features: {list(txt_projected.shape)}") - - # Positional encoding for image features (sinusoidal) - # Compute sinusoidal PE manually - def compute_sine_pe(h, w, num_pos_feats=256, temperature=10000, normalize=True, scale=2*math.pi): - """Sinusoidal PE matching SAM3's PositionEmbeddingSine.""" - not_mask = torch.ones(1, h, w, dtype=torch.float32) - y_embed = not_mask.cumsum(1, dtype=torch.float32) - x_embed = not_mask.cumsum(2, dtype=torch.float32) - if normalize: - eps = 1e-6 - y_embed = (y_embed - 0.5) / (y_embed[:, -1:, :] + eps) * scale - x_embed = (x_embed - 0.5) / (x_embed[:, :, -1:] + eps) * scale - - dim_t = torch.arange(num_pos_feats, dtype=torch.float32) - dim_t = temperature ** (2 * (dim_t // 2) / num_pos_feats) - - pos_x = x_embed[:, :, :, None] / dim_t # [1, H, W, D/2] - pos_y = y_embed[:, :, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) - pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) - pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) # [1, D, H, W] - return pos - - img_pe_72 = compute_sine_pe(H, H, num_pos_feats=D//2) # [1, 256, 72, 72] - save_tensor(os.path.join(args.outdir, "img_pe_72"), img_pe_72) - - # ═══════════════════════════════════════════════════════════════════════ - # Step 5.1: Fusion Encoder (6 layers) - # ═══════════════════════════════════════════════════════════════════════ - print("\n=== Fusion Encoder ===") - - fenc_prefix = "detector.transformer.encoder." - fenc_sd = {k[len("detector.transformer."):]: v for k, v in ckpt.items() - if k.startswith("detector.transformer.encoder.")} - - with torch.no_grad(): - # Image features: [1, 256, 72, 72] -> flatten to [1, 5184, 256] (batch-first) - img_feat_flat = neck_det_2.flatten(2).permute(0, 2, 1) # [1, 5184, 256] - img_pe_flat = img_pe_72.flatten(2).permute(0, 2, 1) # [1, 5184, 256] - - # Text features: [T, 1, 256] -> [1, T, 256] for batch-first - prompt_bf = txt_projected.permute(1, 0, 2) # [1, T, 256] - - # Text mask: True for padding (0 tokens), False for valid - txt_mask = torch.tensor([[tid == 0 for tid in token_ids]], dtype=torch.bool) # [1, T] - valid_scale = float(T) / max(1, sum(tid != 0 for tid in token_ids)) - text_valid_mask = torch.tensor( - [[[valid_scale if tid != 0 else 0.0] for tid in token_ids]], - dtype=torch.float32, - ).permute(1, 2, 0) # [T, 1, 1] - - save_tensor(os.path.join(args.outdir, "fenc_img_input"), img_feat_flat) - save_tensor(os.path.join(args.outdir, "fenc_pos_embed"), img_pe_flat) - save_tensor(os.path.join(args.outdir, "fenc_prompt"), prompt_bf) - save_tensor(os.path.join(args.outdir, "text_valid_mask"), text_valid_mask) - - # Forward through 6 fusion encoder layers - output = img_feat_flat # [1, HW, D] batch-first - - for layer_idx in range(6): - lp = f"encoder.layers.{layer_idx}." - - # 1. Self-attention (pre-norm, pos_enc_at_attn=True) - shortcut = output - xn = F.layer_norm(output, [D], - fenc_sd[lp + "norm1.weight"], fenc_sd[lp + "norm1.bias"]) - q = k = xn + img_pe_flat # add pos to Q and K - - # SA weights - sa_qkv_w = fenc_sd[lp + "self_attn.in_proj_weight"] - sa_qkv_b = fenc_sd[lp + "self_attn.in_proj_bias"] - sa_out_w = fenc_sd[lp + "self_attn.out_proj.weight"] - sa_out_b = fenc_sd[lp + "self_attn.out_proj.bias"] - - sa_out = fused_mha_forward(q, k, xn, sa_qkv_w, sa_qkv_b, sa_out_w, sa_out_b, NH) - output = shortcut + sa_out - - # 2. Cross-attention (pre-norm, no pos at CA) - shortcut = output - xn = F.layer_norm(output, [D], - fenc_sd[lp + "norm2.weight"], fenc_sd[lp + "norm2.bias"]) - # cross_attn_image: Q from image, K/V from text - ca_qkv_w = fenc_sd[lp + "cross_attn_image.in_proj_weight"] - ca_qkv_b = fenc_sd[lp + "cross_attn_image.in_proj_bias"] - ca_out_w = fenc_sd[lp + "cross_attn_image.out_proj.weight"] - ca_out_b = fenc_sd[lp + "cross_attn_image.out_proj.bias"] - - # Split in_proj into separate Q/K/V for cross-attention - w_q_ca = ca_qkv_w[:D] - w_k_ca = ca_qkv_w[D:2*D] - w_v_ca = ca_qkv_w[2*D:] - b_q_ca = ca_qkv_b[:D] - b_k_ca = ca_qkv_b[D:2*D] - b_v_ca = ca_qkv_b[2*D:] - - ca_out = multihead_attention_forward( - xn, prompt_bf, prompt_bf, - w_q_ca, b_q_ca, w_k_ca, b_k_ca, w_v_ca, b_v_ca, - ca_out_w, ca_out_b, NH, - key_padding_mask=txt_mask) - - output = shortcut + ca_out - - # 3. FFN (pre-norm, relu activation) - shortcut = output - xn = F.layer_norm(output, [D], - fenc_sd[lp + "norm3.weight"], fenc_sd[lp + "norm3.bias"]) - h = F.linear(xn, fenc_sd[lp + "linear1.weight"], fenc_sd[lp + "linear1.bias"]) - h = F.relu(h) - h = F.linear(h, fenc_sd[lp + "linear2.weight"], fenc_sd[lp + "linear2.bias"]) - output = shortcut + h - - save_tensor(os.path.join(args.outdir, f"fenc_layer{layer_idx}_out"), output) - - # Transpose to seq-first for downstream: [HW, 1, D] - fenc_output_sf = output.permute(1, 0, 2) # [5184, 1, D] seq-first - save_tensor(os.path.join(args.outdir, "fenc_output"), fenc_output_sf) - save_tensor(os.path.join(args.outdir, "fenc_output_bf"), output) # batch-first for decoder - print(f" fenc_output shape: {list(fenc_output_sf.shape)}") - - # ═══════════════════════════════════════════════════════════════════════ - # Step 5.2: DETR Decoder (6 layers) - # ═══════════════════════════════════════════════════════════════════════ - print("\n=== DETR Decoder ===") - - ddec_prefix = "detector.transformer.decoder." - ddec_sd = {k[len("detector.transformer."):]: v for k, v in ckpt.items() - if k.startswith("detector.transformer.decoder.")} - - with torch.no_grad(): - # Query embeddings and reference points - query_embed = ddec_sd["decoder.query_embed.weight"] # [NQ, D] - ref_pts_raw = ddec_sd["decoder.reference_points.weight"] # [NQ, 4] - reference_boxes = ref_pts_raw.sigmoid().unsqueeze(1) # [NQ, 1, 4] - - save_tensor(os.path.join(args.outdir, "ddec_query_embed"), query_embed) - save_tensor(os.path.join(args.outdir, "ddec_ref_pts_raw"), ref_pts_raw) - save_tensor(os.path.join(args.outdir, "ddec_ref_boxes_init"), reference_boxes) - - # Presence token - presence_token_w = ddec_sd["decoder.presence_token.weight"] # [1, D] - presence_out = presence_token_w[None].expand(1, 1, -1) # [1, 1, D] - save_tensor(os.path.join(args.outdir, "ddec_presence_token"), presence_out) - - # Encoder output is [HW, BS, D] (seq-first) for the decoder - enc_memory = fenc_output_sf # [5184, 1, D] - enc_pos = img_pe_flat.permute(1, 0, 2) # [5184, 1, D] seq-first - - # valid_ratios: [1, 1, 2] = ones (no masking) - valid_ratios = torch.ones(1, 1, 2, device=device) - spatial_shapes = torch.tensor([[H, H]], dtype=torch.long) - - # Init output (tgt) - output = query_embed.unsqueeze(1).repeat(1, 1, 1) # [NQ, 1, D] - - # Text features for cross-attention: seq-first [T, 1, D] - prompt_sf = txt_projected # already [T, 1, D] - - # ── RPB coordinates ───────────────────────────────────────── - coords_h = torch.arange(0, H, dtype=torch.float32) / H - coords_w = torch.arange(0, H, dtype=torch.float32) / H - - for layer_idx in range(6): - lp = f"decoder.layers.{layer_idx}." - - # Reference points input - reference_points_input = reference_boxes[:, :, None] * torch.cat([valid_ratios, valid_ratios], -1)[None, :] - query_sine_embed = gen_sineembed_for_position(reference_points_input[:, :, 0, :], D) - # query_sine_embed: [NQ, 1, 512] - - # Conditional query pos: ref_point_head MLP (512→256→256) - query_pos = mlp_forward(query_sine_embed, ddec_sd, - "decoder.ref_point_head", 2) # [NQ, 1, D] - - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_query_pos_0"), query_pos) - save_tensor(os.path.join(args.outdir, "ddec_query_sine_0"), query_sine_embed) - - # ── Box RPB ───────────────────────────────────────────── - boxes_xyxy = box_cxcywh_to_xyxy(reference_boxes).permute(1, 0, 2) # [1, NQ, 4] - bs_d, nq_d, _ = boxes_xyxy.shape - - deltas_y = coords_h.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 1:4:2] - deltas_y = deltas_y.view(bs_d, nq_d, -1, 2) # [1, NQ, H, 2] - deltas_x = coords_w.view(1, -1, 1) - boxes_xyxy.reshape(-1, 1, 4)[:, :, 0:3:2] - deltas_x = deltas_x.view(bs_d, nq_d, -1, 2) # [1, NQ, W, 2] - - # Log transform - deltas_x_log = deltas_x * 8 - deltas_x_log = torch.sign(deltas_x_log) * torch.log2(torch.abs(deltas_x_log) + 1.0) / np.log2(8) - deltas_y_log = deltas_y * 8 - deltas_y_log = torch.sign(deltas_y_log) * torch.log2(torch.abs(deltas_y_log) + 1.0) / np.log2(8) - deltas_x = deltas_x_log - deltas_y = deltas_y_log - - # RPB MLP: [1, NQ, W, 2] → [1, NQ, W, NH] - rpb_x = mlp_forward(deltas_x, ddec_sd, "decoder.boxRPB_embed_x", 2) - rpb_y = mlp_forward(deltas_y, ddec_sd, "decoder.boxRPB_embed_y", 2) - - # Outer sum: [1, NQ, H, W, NH] - B_rpb = rpb_y.unsqueeze(3) + rpb_x.unsqueeze(2) - B_rpb = B_rpb.flatten(2, 3) # [1, NQ, H*W, NH] - B_rpb = B_rpb.permute(0, 3, 1, 2).contiguous() # [1, NH, NQ, H*W] - - # For presence token: prepend zeros - pres_rpb = torch.zeros(1, NH, 1, H*H, device=device) - memory_mask = torch.cat([pres_rpb, B_rpb], dim=2) # [1, NH, NQ+1, H*W] - memory_mask = memory_mask.flatten(0, 1) # [NH, NQ+1, H*W] - - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_rpb_mask_0"), memory_mask) - - # ── Self-attention ────────────────────────────────────── - # Prepend presence token - tgt_with_pres = torch.cat([presence_out, output], dim=0) # [NQ+1, 1, D] - qpos_with_pres = torch.cat([torch.zeros_like(presence_out), query_pos], dim=0) - - q_sa = k_sa = tgt_with_pres + qpos_with_pres - - sa_w = ddec_sd[lp + "self_attn.in_proj_weight"] - sa_b = ddec_sd[lp + "self_attn.in_proj_bias"] - sa_ow = ddec_sd[lp + "self_attn.out_proj.weight"] - sa_ob = ddec_sd[lp + "self_attn.out_proj.bias"] - - # Convert to batch-first for MHA - q_sa_bf = q_sa.permute(1, 0, 2) # [1, NQ+1, D] - k_sa_bf = k_sa.permute(1, 0, 2) - v_sa_bf = tgt_with_pres.permute(1, 0, 2) - - sa_out = fused_mha_forward(q_sa_bf, k_sa_bf, v_sa_bf, sa_w, sa_b, sa_ow, sa_ob, NH) - tgt_with_pres = tgt_with_pres + sa_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm2.weight"], ddec_sd[lp + "norm2.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_sa"), tgt_with_pres) - - # ── Text cross-attention ───────────────────────────────── - tgt_q_ca_text = tgt_with_pres + qpos_with_pres - ca_text_w = ddec_sd[lp + "ca_text.in_proj_weight"] - ca_text_b = ddec_sd[lp + "ca_text.in_proj_bias"] - ca_text_ow = ddec_sd[lp + "ca_text.out_proj.weight"] - ca_text_ob = ddec_sd[lp + "ca_text.out_proj.bias"] - - # Q from queries, K/V from text - w_q_ct = ca_text_w[:D] - w_k_ct = ca_text_w[D:2*D] - w_v_ct = ca_text_w[2*D:] - b_q_ct = ca_text_b[:D] - b_k_ct = ca_text_b[D:2*D] - b_v_ct = ca_text_b[2*D:] - - tq_bf = tgt_q_ca_text.permute(1, 0, 2) # [1, NQ+1, D] - p_bf = prompt_sf.permute(1, 0, 2) # [1, T, D] - ct_out = multihead_attention_forward(tq_bf, p_bf, p_bf, - w_q_ct, b_q_ct, w_k_ct, b_k_ct, w_v_ct, b_v_ct, - ca_text_ow, ca_text_ob, NH, - key_padding_mask=txt_mask) - tgt_with_pres = tgt_with_pres + ct_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "catext_norm.weight"], - ddec_sd[lp + "catext_norm.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_text_ca"), tgt_with_pres) - - # ── Image cross-attention with RPB ────────────────────── - tgt_q_ca_img = tgt_with_pres + qpos_with_pres - mem_k = enc_memory + enc_pos # [5184, 1, D] - - ca_w = ddec_sd[lp + "cross_attn.in_proj_weight"] - ca_b = ddec_sd[lp + "cross_attn.in_proj_bias"] - ca_ow = ddec_sd[lp + "cross_attn.out_proj.weight"] - ca_ob = ddec_sd[lp + "cross_attn.out_proj.bias"] - - w_q_ci = ca_w[:D] - w_k_ci = ca_w[D:2*D] - w_v_ci = ca_w[2*D:] - b_q_ci = ca_b[:D] - b_k_ci = ca_b[D:2*D] - b_v_ci = ca_b[2*D:] - - # Batch first - tqi_bf = tgt_q_ca_img.permute(1, 0, 2) # [1, NQ+1, D] - mk_bf = mem_k.permute(1, 0, 2) # [1, 5184, D] - mv_bf = enc_memory.permute(1, 0, 2) # [1, 5184, D] - - # Need to handle RPB mask as attn_mask - # memory_mask: [NH, NQ+1, H*W] — for PyTorch SDPA, need [BS*NH, NQ+1, H*W] - # For BS=1, just use as is - ci_out = multihead_attention_forward(tqi_bf, mk_bf, mv_bf, - w_q_ci, b_q_ci, w_k_ci, b_k_ci, w_v_ci, b_v_ci, - ca_ow, ca_ob, NH, - attn_mask=memory_mask) - tgt_with_pres = tgt_with_pres + ci_out.permute(1, 0, 2) - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm1.weight"], ddec_sd[lp + "norm1.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_after_img_ca"), tgt_with_pres) - - # ── FFN ───────────────────────────────────────────────── - ffn = F.linear(tgt_with_pres, ddec_sd[lp + "linear1.weight"], ddec_sd[lp + "linear1.bias"]) - ffn = F.relu(ffn) - ffn = F.linear(ffn, ddec_sd[lp + "linear2.weight"], ddec_sd[lp + "linear2.bias"]) - tgt_with_pres = tgt_with_pres + ffn - tgt_with_pres = F.layer_norm(tgt_with_pres, [D], - ddec_sd[lp + "norm3.weight"], ddec_sd[lp + "norm3.bias"]) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, "ddec_layer0_full_out"), tgt_with_pres) - - # Split presence and queries - presence_out = tgt_with_pres[:1] # [1, 1, D] - output = tgt_with_pres[1:] # [NQ, 1, D] - - # Box refinement - reference_before_sigmoid = inverse_sigmoid(reference_boxes) - normed_output = F.layer_norm(output, [D], - ddec_sd["decoder.norm.weight"], ddec_sd["decoder.norm.bias"]) - delta_unsig = mlp_forward(normed_output, ddec_sd, "decoder.bbox_embed", 3) - outputs_unsig = delta_unsig + reference_before_sigmoid - new_reference_points = outputs_unsig.sigmoid() - reference_boxes = new_reference_points.detach() - - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_out"), output) - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_refboxes"), reference_boxes) - if layer_idx == 0: - save_tensor(os.path.join(args.outdir, f"ddec_layer{layer_idx}_presence"), presence_out) - - # Final normalization - normed_output = F.layer_norm(output, [D], - ddec_sd["decoder.norm.weight"], ddec_sd["decoder.norm.bias"]) - save_tensor(os.path.join(args.outdir, "ddec_normed_output"), normed_output) - save_tensor(os.path.join(args.outdir, "ddec_pred_boxes"), reference_boxes) - - # Presence head - pres_normed = F.layer_norm(presence_out, [D], - ddec_sd["decoder.presence_token_out_norm.weight"], - ddec_sd["decoder.presence_token_out_norm.bias"]) - pres_logit = mlp_forward(pres_normed, ddec_sd, "decoder.presence_token_head", 3) - save_tensor(os.path.join(args.outdir, "ddec_presence_logit"), pres_logit) - print(f" Presence logit: {pres_logit.item():.4f}") - - # ═══════════════════════════════════════════════════════════════════════ - # Step 5.2b: DotProductScoring - # ═══════════════════════════════════════════════════════════════════════ - print("\n=== DotProductScoring ===") - - scoring_prefix = "detector.dot_prod_scoring." - scoring_sd = {k[len(scoring_prefix):]: v for k, v in ckpt.items() - if k.startswith(scoring_prefix)} - - with torch.no_grad(): - # prompt_mlp: residual MLP + LayerNorm - prompt_for_scoring = prompt_sf.clone() # [T, 1, D] - prompt_mlp_out = prompt_for_scoring - orig = prompt_mlp_out.clone() - prompt_mlp_out = F.linear(prompt_mlp_out, scoring_sd["prompt_mlp.layers.0.weight"], - scoring_sd["prompt_mlp.layers.0.bias"]) - prompt_mlp_out = F.relu(prompt_mlp_out) - prompt_mlp_out = F.linear(prompt_mlp_out, scoring_sd["prompt_mlp.layers.1.weight"], - scoring_sd["prompt_mlp.layers.1.bias"]) - prompt_mlp_out = prompt_mlp_out + orig # residual - prompt_mlp_out = F.layer_norm(prompt_mlp_out, [D], - scoring_sd["prompt_mlp.out_norm.weight"], - scoring_sd["prompt_mlp.out_norm.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_prompt_mlp_out"), prompt_mlp_out) - - # Mean pool text - # prompt_mask: True for padding, False for valid - is_valid = (~txt_mask).float().permute(1, 0)[..., None] # [T, 1, 1] - num_valid = torch.clamp(is_valid.sum(dim=0), min=1.0) # [1, 1] - pooled = (prompt_mlp_out * is_valid).sum(dim=0) / num_valid # [1, D] - save_tensor(os.path.join(args.outdir, "scoring_pooled"), pooled) - - # Project - proj_pooled = F.linear(pooled, scoring_sd["prompt_proj.weight"], - scoring_sd["prompt_proj.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_proj_pooled"), proj_pooled) - - # Project hs - hs = normed_output.permute(1, 0, 2).unsqueeze(0) # [1, 1, NQ, D] - proj_hs = F.linear(hs, scoring_sd["hs_proj.weight"], scoring_sd["hs_proj.bias"]) - save_tensor(os.path.join(args.outdir, "scoring_proj_hs"), proj_hs) - - # Dot product scores - scores = torch.matmul(proj_hs, proj_pooled.unsqueeze(-1)) # [1, 1, NQ, 1] - scores = scores * (1.0 / math.sqrt(D)) - scores = scores.clamp(-12.0, 12.0) - save_tensor(os.path.join(args.outdir, "scoring_class_scores"), scores) - print(f" scores shape: {list(scores.shape)}, range: [{scores.min():.4f}, {scores.max():.4f}]") - - # ═══════════════════════════════════════════════════════════════════════ - # Step 5.3: Segmentation Head - # ═══════════════════════════════════════════════════════════════════════ - print("\n=== Segmentation Head ===") - - seg_prefix = "detector.segmentation_head." - seg_sd = {k[len(seg_prefix):]: v for k, v in ckpt.items() - if k.startswith(seg_prefix)} - - with torch.no_grad(): - # Cross-attend encoder hidden states to prompt - enc_hs = fenc_output_sf.clone() # [5184, 1, D] - tgt2 = F.layer_norm(enc_hs, [D], seg_sd["cross_attn_norm.weight"], seg_sd["cross_attn_norm.bias"]) - - ca_w = seg_sd["cross_attend_prompt.in_proj_weight"] - ca_b = seg_sd["cross_attend_prompt.in_proj_bias"] - ca_ow = seg_sd["cross_attend_prompt.out_proj.weight"] - ca_ob = seg_sd["cross_attend_prompt.out_proj.bias"] - - w_q = ca_w[:D]; w_k = ca_w[D:2*D]; w_v = ca_w[2*D:] - b_q = ca_b[:D]; b_k = ca_b[D:2*D]; b_v = ca_b[2*D:] - - tgt2_bf = tgt2.permute(1, 0, 2) - p_bf = prompt_sf.permute(1, 0, 2) - ca_out = multihead_attention_forward(tgt2_bf, p_bf, p_bf, - w_q, b_q, w_k, b_k, w_v, b_v, - ca_ow, ca_ob, NH, - key_padding_mask=txt_mask) - enc_hs = enc_hs + ca_out.permute(1, 0, 2) - save_tensor(os.path.join(args.outdir, "seg_enc_after_ca"), enc_hs) - - # Replace lowest-res FPN feat with encoder visual output - enc_visual = enc_hs.permute(1, 2, 0).reshape(1, D, H, H) - save_tensor(os.path.join(args.outdir, "seg_enc_visual"), enc_visual) - - # Pixel decoder - modified_feats = [neck_det_0.clone(), neck_det_1.clone(), enc_visual] - - prev_fpn = modified_feats[-1] # [1, D, 72, 72] - for pd_idx, bb_feat in enumerate(modified_feats[:-1][::-1]): - curr_fpn = bb_feat - prev_fpn = curr_fpn + F.interpolate(prev_fpn, size=curr_fpn.shape[-2:], mode="nearest") - prev_fpn = F.conv2d(prev_fpn, seg_sd[f"pixel_decoder.conv_layers.{pd_idx}.weight"], - seg_sd[f"pixel_decoder.conv_layers.{pd_idx}.bias"], padding=1) - prev_fpn = F.group_norm(prev_fpn, 8, - seg_sd[f"pixel_decoder.norms.{pd_idx}.weight"], - seg_sd[f"pixel_decoder.norms.{pd_idx}.bias"]) - prev_fpn = F.relu(prev_fpn) - save_tensor(os.path.join(args.outdir, f"seg_pixel_dec_stage{pd_idx}"), prev_fpn) - - pixel_embed = prev_fpn # [1, D, 288, 288] - save_tensor(os.path.join(args.outdir, "seg_pixel_decoder_out"), pixel_embed) - - # Instance seg head - instance_embed = F.conv2d(pixel_embed, seg_sd["instance_seg_head.weight"], - seg_sd["instance_seg_head.bias"]) - save_tensor(os.path.join(args.outdir, "seg_instance_embed"), instance_embed) - - # Mask predictor - obj_queries = normed_output.permute(1, 0, 2) # [1, NQ, D] - mask_embed = obj_queries - for j in range(3): - w = seg_sd[f"mask_predictor.mask_embed.layers.{j}.weight"] - b = seg_sd[f"mask_predictor.mask_embed.layers.{j}.bias"] - mask_embed = F.linear(mask_embed, w, b) - if j < 2: - mask_embed = F.relu(mask_embed) - save_tensor(os.path.join(args.outdir, "seg_mask_embed"), mask_embed) - - # Einsum: bqc,bchw -> bqhw - mask_preds = torch.einsum("bqc,bchw->bqhw", mask_embed, instance_embed) - save_tensor(os.path.join(args.outdir, "seg_mask_logits"), mask_preds) - print(f" mask_preds shape: {list(mask_preds.shape)}") - - # ═══════════════════════════════════════════════════════════════════════ - # End-to-end summary - # ═══════════════════════════════════════════════════════════════════════ - print("\n=== End-to-end Summary ===") - with torch.no_grad(): - class_logits = scores[-1, :, :, 0] # [1, NQ] - pres_sig = pres_logit.squeeze(-1).squeeze(0).sigmoid() # scalar - probs = class_logits.sigmoid() * pres_sig - top_k = min(10, NQ) - top_probs, top_idx = probs[0].topk(top_k) - print(f" Top {top_k} detections:") - for i in range(top_k): - qi = top_idx[i].item() - box = reference_boxes[qi, 0] - print(f" q{qi}: score={top_probs[i]:.4f}, box=({box[0]:.3f},{box[1]:.3f},{box[2]:.3f},{box[3]:.3f})") - - print(f"\nDone! Reference tensors saved to {args.outdir}") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_phase6_reference.py b/tests/dump_phase6_reference.py deleted file mode 100644 index e124125..0000000 --- a/tests/dump_phase6_reference.py +++ /dev/null @@ -1,493 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump Phase 6 (SAM prompt encoder + SAM mask decoder) reference tensors for -numerical comparison against sam3.cpp. - -The script reuses tracker neck features from Phase 3 so Phase 6 can be audited -in isolation and the fixtures can be reused by later tracker stages. - -Usage: - uv run python tests/dump_phase6_reference.py \ - --checkpoint raw_weights/sam3.pt \ - --prephase-ref tests/ref_phase3 \ - --cases tests/phase6_cases.tsv \ - --outdir tests/ref_phase6 -""" - -import argparse -import math -import os -from dataclasses import dataclass -from typing import Dict, List - -import numpy as np -import torch -import torch.nn.functional as F - - -@dataclass -class PromptCase: - case_id: str - multimask: bool - pos_points: List[List[float]] - neg_points: List[List[float]] - box: List[float] - - -def load_tensor(path: str) -> torch.Tensor: - with open(path + ".shape", "r", encoding="utf-8") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32).reshape(shape) - return torch.from_numpy(data) - - -def save_raw(path: str, arr: np.ndarray, shape) -> None: - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(arr.astype(np.float32, copy=False).tobytes()) - with open(path + ".shape", "w", encoding="utf-8") as f: - f.write(",".join(str(int(d)) for d in shape)) - - -def save_ggml_bd(path: str, x_bd: torch.Tensor) -> None: - assert x_bd.ndim == 2 - x_bd = x_bd.detach().cpu().float().contiguous() - b, d = x_bd.shape - save_raw(path, x_bd.numpy(), (d, b)) - - -def save_ggml_bnd(path: str, x_bnd: torch.Tensor) -> None: - assert x_bnd.ndim == 3 - x_bnd = x_bnd.detach().cpu().float().contiguous() - b, n, d = x_bnd.shape - save_raw(path, x_bnd.numpy(), (d, n, b)) - - -def save_ggml_bhwc(path: str, x_bhwc: torch.Tensor) -> None: - assert x_bhwc.ndim == 4 - x_bhwc = x_bhwc.detach().cpu().float().contiguous() - b, h, w, c = x_bhwc.shape - save_raw(path, x_bhwc.numpy(), (c, w, h, b)) - - -def save_ggml_nchw(path: str, x_nchw: torch.Tensor) -> None: - assert x_nchw.ndim == 4 - save_ggml_bhwc(path, x_nchw.permute(0, 2, 3, 1)) - - -def save_ggml_whcb_from_nchw(path: str, x_nchw: torch.Tensor) -> None: - assert x_nchw.ndim == 4 - x_nchw = x_nchw.detach().cpu().float().contiguous() - b, c, h, w = x_nchw.shape - save_raw(path, x_nchw.numpy(), (w, h, c, b)) - - -def save_ggml_masks(path: str, masks_bchw: torch.Tensor) -> None: - assert masks_bchw.ndim == 4 - b, c, h, w = masks_bchw.shape - save_ggml_bnd(path, masks_bchw.reshape(b, c, h * w)) - - -def parse_points(field: str) -> List[List[float]]: - if not field: - return [] - points = [] - for part in field.split("|"): - x_str, y_str = part.split(":") - points.append([float(x_str), float(y_str)]) - return points - - -def parse_box(field: str) -> List[float]: - if not field: - return [] - x0, y0, x1, y1 = field.split(":") - return [float(x0), float(y0), float(x1), float(y1)] - - -def load_cases(path: str) -> List[PromptCase]: - cases: List[PromptCase] = [] - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.rstrip("\n") - if not line: - continue - fields = line.split("\t") - while len(fields) < 5: - fields.append("") - cases.append( - PromptCase( - case_id=fields[0], - multimask=fields[1] == "1", - pos_points=parse_points(fields[2]), - neg_points=parse_points(fields[3]), - box=parse_box(fields[4]), - ) - ) - return cases - - -def build_point_inputs(case: PromptCase): - coords: List[List[float]] = [] - labels: List[int] = [] - - if case.box: - coords.append([case.box[0], case.box[1]]) - coords.append([case.box[2], case.box[3]]) - labels.extend([2, 3]) - - coords.extend(case.pos_points) - labels.extend([1] * len(case.pos_points)) - - coords.extend(case.neg_points) - labels.extend([0] * len(case.neg_points)) - - coords_t = torch.tensor([coords], dtype=torch.float32) if coords else torch.zeros(1, 0, 2) - labels_t = torch.tensor([labels], dtype=torch.int64) if labels else torch.zeros(1, 0, dtype=torch.int64) - return coords_t, labels_t - - -def position_encode(coords_norm: torch.Tensor, pe_gaussian: torch.Tensor) -> torch.Tensor: - coords = 2.0 * coords_norm - 1.0 - enc = coords @ pe_gaussian - enc = 2.0 * math.pi * enc - return torch.cat([torch.sin(enc), torch.cos(enc)], dim=-1) - - -def build_prompt_encoder_outputs(weights: Dict[str, torch.Tensor], case: PromptCase): - embed_dim = 256 - input_size = 1008 - feat_size = 72 - - pe_gaussian = weights["pe_layer.positional_encoding_gaussian_matrix"].float() - coords, labels = build_point_inputs(case) - - pad_point = torch.zeros((coords.shape[0], 1, 2), dtype=torch.float32) - pad_label = -torch.ones((labels.shape[0], 1), dtype=torch.int64) - coords = torch.cat([coords, pad_point], dim=1) - labels = torch.cat([labels, pad_label], dim=1) - - coords_centered = coords + 0.5 - coords_norm = coords_centered.clone() - coords_norm[:, :, 0] /= input_size - coords_norm[:, :, 1] /= input_size - point_embedding = position_encode(coords_norm, pe_gaussian) - - not_a_point = weights["not_a_point_embed.weight"].float()[0] - point_embeds = [ - weights[f"point_embeddings.{i}.weight"].float()[0] - for i in range(4) - ] - - sparse = point_embedding.clone() - for idx in range(labels.shape[1]): - label = int(labels[0, idx].item()) - if label == -1: - sparse[:, idx, :] = not_a_point - else: - sparse[:, idx, :] = sparse[:, idx, :] + point_embeds[label] - - ys = (torch.arange(feat_size, dtype=torch.float32) + 0.5) / feat_size - xs = (torch.arange(feat_size, dtype=torch.float32) + 0.5) / feat_size - yy, xx = torch.meshgrid(ys, xs, indexing="ij") - dense_coords = torch.stack([xx, yy], dim=-1) - image_pe = position_encode(dense_coords, pe_gaussian).permute(2, 0, 1).unsqueeze(0) - - no_mask = weights["no_mask_embed.weight"].float().view(1, embed_dim, 1, 1) - dense = no_mask.expand(1, embed_dim, feat_size, feat_size).contiguous() - return sparse, dense, image_pe - - -def attention_forward(q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - prefix: str, - weights: Dict[str, torch.Tensor], - num_heads: int) -> torch.Tensor: - q_w = weights[prefix + ".q_proj.weight"].float() - q_b = weights[prefix + ".q_proj.bias"].float() - k_w = weights[prefix + ".k_proj.weight"].float() - k_b = weights[prefix + ".k_proj.bias"].float() - v_w = weights[prefix + ".v_proj.weight"].float() - v_b = weights[prefix + ".v_proj.bias"].float() - out_w = weights[prefix + ".out_proj.weight"].float() - out_b = weights[prefix + ".out_proj.bias"].float() - - q_proj = F.linear(q, q_w, q_b) - k_proj = F.linear(k, k_w, k_b) - v_proj = F.linear(v, v_w, v_b) - - bsz, nq, dim = q_proj.shape - nk = k_proj.shape[1] - head_dim = dim // num_heads - - qh = q_proj.view(bsz, nq, num_heads, head_dim).permute(0, 2, 1, 3) - kh = k_proj.view(bsz, nk, num_heads, head_dim).permute(0, 2, 1, 3) - vh = v_proj.view(bsz, nk, num_heads, head_dim).permute(0, 2, 1, 3) - - out = F.scaled_dot_product_attention(qh, kh, vh) - out = out.permute(0, 2, 1, 3).reshape(bsz, nq, dim) - return F.linear(out, out_w, out_b) - - -def mlp_forward(x: torch.Tensor, - weights: Dict[str, torch.Tensor], - prefix: str, - num_layers: int, - sigmoid_output: bool = False) -> torch.Tensor: - for i in range(num_layers): - w = weights[f"{prefix}.layers.{i}.weight"].float() - b = weights[f"{prefix}.layers.{i}.bias"].float() - x = F.linear(x, w, b) - if i < num_layers - 1: - x = F.relu(x) - if sigmoid_output: - x = torch.sigmoid(x) - return x - - -def layer_norm_2d(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: - u = x.mean(1, keepdim=True) - s = (x - u).pow(2).mean(1, keepdim=True) - x = (x - u) / torch.sqrt(s + eps) - return weight[:, None, None] * x + bias[:, None, None] - - -def dump_case(case: PromptCase, - prompt_weights: Dict[str, torch.Tensor], - mask_weights: Dict[str, torch.Tensor], - no_mem_embed: torch.Tensor, - neck_trk_0: torch.Tensor, - neck_trk_1: torch.Tensor, - neck_trk_2: torch.Tensor, - outdir: str) -> None: - os.makedirs(outdir, exist_ok=True) - - sparse_embeddings, dense_embeddings, image_pe = build_prompt_encoder_outputs(prompt_weights, case) - - image_embeddings = neck_trk_2 + no_mem_embed.view(1, -1, 1, 1) - feat_s0 = F.conv2d( - neck_trk_0, - mask_weights["conv_s0.weight"].float(), - mask_weights["conv_s0.bias"].float(), - ) - feat_s1 = F.conv2d( - neck_trk_1, - mask_weights["conv_s1.weight"].float(), - mask_weights["conv_s1.bias"].float(), - ) - - save_ggml_bnd(os.path.join(outdir, "sam_pe_sparse"), sparse_embeddings) - save_ggml_nchw(os.path.join(outdir, "sam_pe_dense"), dense_embeddings) - save_ggml_nchw(os.path.join(outdir, "sam_pe_image_pe"), image_pe) - save_ggml_nchw(os.path.join(outdir, "sam_dec_image_feats"), image_embeddings) - - output_tokens = torch.cat( - [ - mask_weights["obj_score_token.weight"].float(), - mask_weights["iou_token.weight"].float(), - mask_weights["mask_tokens.weight"].float(), - ], - dim=0, - ).unsqueeze(0).expand(sparse_embeddings.size(0), -1, -1) - tokens = torch.cat((output_tokens, sparse_embeddings), dim=1) - save_ggml_bnd(os.path.join(outdir, "sam_dec_tokens_initial"), tokens) - - src = image_embeddings + dense_embeddings - pos_src = image_pe.repeat(tokens.shape[0], 1, 1, 1) - bsz, dim, h, w = src.shape - keys = src.flatten(2).permute(0, 2, 1) - key_pe = pos_src.flatten(2).permute(0, 2, 1) - queries = tokens - query_pe = tokens - - for idx in range(2): - prefix = f"transformer.layers.{idx}" - if idx == 0: - queries = attention_forward( - queries, queries, queries, prefix + ".self_attn", mask_weights, 8 - ) - else: - q = queries + query_pe - attn_out = attention_forward(q, q, queries, prefix + ".self_attn", mask_weights, 8) - queries = queries + attn_out - queries = F.layer_norm( - queries, - [dim], - mask_weights[prefix + ".norm1.weight"].float(), - mask_weights[prefix + ".norm1.bias"].float(), - ) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - q, k, keys, prefix + ".cross_attn_token_to_image", mask_weights, 8 - ) - queries = F.layer_norm( - queries + attn_out, - [dim], - mask_weights[prefix + ".norm2.weight"].float(), - mask_weights[prefix + ".norm2.bias"].float(), - ) - - mlp_hidden = F.linear( - queries, - mask_weights[prefix + ".mlp.lin1.weight"].float(), - mask_weights[prefix + ".mlp.lin1.bias"].float(), - ) - mlp_hidden = F.relu(mlp_hidden) - mlp_hidden = F.linear( - mlp_hidden, - mask_weights[prefix + ".mlp.lin2.weight"].float(), - mask_weights[prefix + ".mlp.lin2.bias"].float(), - ) - queries = F.layer_norm( - queries + mlp_hidden, - [dim], - mask_weights[prefix + ".norm3.weight"].float(), - mask_weights[prefix + ".norm3.bias"].float(), - ) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - k, q, queries, prefix + ".cross_attn_image_to_token", mask_weights, 8 - ) - keys = F.layer_norm( - keys + attn_out, - [dim], - mask_weights[prefix + ".norm4.weight"].float(), - mask_weights[prefix + ".norm4.bias"].float(), - ) - - save_ggml_bnd(os.path.join(outdir, f"sam_dec_block{idx}_queries"), queries) - save_ggml_bnd(os.path.join(outdir, f"sam_dec_block{idx}_keys"), keys) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - q, k, keys, "transformer.final_attn_token_to_image", mask_weights, 8 - ) - queries = F.layer_norm( - queries + attn_out, - [dim], - mask_weights["transformer.norm_final_attn.weight"].float(), - mask_weights["transformer.norm_final_attn.bias"].float(), - ) - save_ggml_bnd(os.path.join(outdir, "sam_dec_final_queries"), queries) - - iou_token_out = queries[:, 1, :] - mask_tokens_out = queries[:, 2:6, :] - obj_token_out = queries[:, 0, :] - save_ggml_bnd(os.path.join(outdir, "sam_dec_mask_tokens"), mask_tokens_out) - save_ggml_bd(os.path.join(outdir, "sam_dec_sam_token"), mask_tokens_out[:, 0, :]) - - src_img = keys.transpose(1, 2).view(bsz, dim, h, w) - up1 = F.conv_transpose2d( - src_img, - mask_weights["output_upscaling.0.weight"].float(), - mask_weights["output_upscaling.0.bias"].float(), - stride=2, - ) - up1 = up1 + feat_s1 - up1 = layer_norm_2d( - up1, - mask_weights["output_upscaling.1.weight"].float(), - mask_weights["output_upscaling.1.bias"].float(), - ) - up1 = F.gelu(up1) - - up2 = F.conv_transpose2d( - up1, - mask_weights["output_upscaling.3.weight"].float(), - mask_weights["output_upscaling.3.bias"].float(), - stride=2, - ) - up2 = F.gelu(up2 + feat_s0) - - save_ggml_whcb_from_nchw(os.path.join(outdir, "sam_dec_feat_s1_proj"), feat_s1) - save_ggml_whcb_from_nchw(os.path.join(outdir, "sam_dec_feat_s0_proj"), feat_s0) - save_ggml_nchw(os.path.join(outdir, "sam_dec_upscaled"), up2) - - hyper_in_list = [ - mlp_forward( - mask_tokens_out[:, i, :], - mask_weights, - f"output_hypernetworks_mlps.{i}", - 3, - ) - for i in range(4) - ] - hyper_in = torch.stack(hyper_in_list, dim=1) - bsz, c, h, w = up2.shape - masks = (hyper_in @ up2.view(bsz, c, h * w)).view(bsz, -1, h, w) - iou_pred = mlp_forward( - iou_token_out, - mask_weights, - "iou_prediction_head", - 3, - sigmoid_output=True, - ) - obj_score_logits = mlp_forward( - obj_token_out, - mask_weights, - "pred_obj_score_head", - 3, - sigmoid_output=False, - ) - - save_ggml_masks(os.path.join(outdir, "sam_dec_masks"), masks) - save_ggml_bd(os.path.join(outdir, "sam_dec_iou"), iou_pred) - save_ggml_bd(os.path.join(outdir, "sam_dec_obj_score"), obj_score_logits) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--prephase-ref", default="tests/ref_phase3") - parser.add_argument("--cases", default="tests/phase6_cases.tsv") - parser.add_argument("--outdir", default="tests/ref_phase6") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - prompt_weights = { - k[len("tracker.sam_prompt_encoder."):]: v - for k, v in ckpt.items() - if k.startswith("tracker.sam_prompt_encoder.") - } - mask_weights = { - k[len("tracker.sam_mask_decoder."):]: v - for k, v in ckpt.items() - if k.startswith("tracker.sam_mask_decoder.") - } - no_mem_embed = ckpt["tracker.no_mem_embed"].float() - - neck_trk_0 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_0")).float() - neck_trk_1 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_1")).float() - neck_trk_2 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_2")).float() - - cases = load_cases(args.cases) - for case in cases: - print(f"Dumping {case.case_id}...") - dump_case( - case, - prompt_weights, - mask_weights, - no_mem_embed, - neck_trk_0, - neck_trk_1, - neck_trk_2, - os.path.join(args.outdir, case.case_id), - ) - - print(f"All tensors saved to: {args.outdir}") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_phase7_reference.py b/tests/dump_phase7_reference.py deleted file mode 100644 index 08a3c3c..0000000 --- a/tests/dump_phase7_reference.py +++ /dev/null @@ -1,836 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump Phase 7 (video tracking) reference tensors for numerical comparison. - -This reuses Phase 3 tracker features and Phase 6 SAM outputs so later tracker work -can audit Phase 7 in isolation from earlier stages. - -Usage: - uv run python tests/dump_phase7_reference.py \ - --checkpoint raw_weights/sam3.pt \ - --prephase-ref tests/ref_phase3 \ - --phase6-ref tests/ref_phase6 \ - --cases tests/phase7_cases.tsv \ - --outdir tests/ref_phase7 -""" - -import argparse -import math -import os -from dataclasses import dataclass -from typing import Dict, List - -import numpy as np -import torch -import torch.nn.functional as F - - -@dataclass -class Phase7Case: - case_id: str - label: str - mem_cases: List[str] - spatial_tpos: List[int] - ptr_tpos: List[int] - - -def load_shape(path: str) -> List[int]: - with open(path + ".shape", "r", encoding="utf-8") as f: - return [int(x) for x in f.read().strip().split(",") if x] - - -def load_tensor(path: str) -> torch.Tensor: - shape = load_shape(path) - data = np.fromfile(path + ".bin", dtype=np.float32).reshape(shape) - return torch.from_numpy(data) - - -def load_ggml_bd(path: str) -> torch.Tensor: - d, b = load_shape(path) - data = np.fromfile(path + ".bin", dtype=np.float32).reshape((b, d)) - return torch.from_numpy(data) - - -def load_ggml_bnd(path: str) -> torch.Tensor: - d, n, b = load_shape(path) - data = np.fromfile(path + ".bin", dtype=np.float32).reshape((b, n, d)) - return torch.from_numpy(data) - - -def load_ggml_masks(path: str, h: int, w: int) -> torch.Tensor: - x = load_ggml_bnd(path) - b, n, hw = x.shape - assert hw == h * w - return x.view(b, n, h, w) - - -def save_raw(path: str, arr: np.ndarray, shape) -> None: - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(arr.astype(np.float32, copy=False).tobytes()) - with open(path + ".shape", "w", encoding="utf-8") as f: - f.write(",".join(str(int(d)) for d in shape)) - - -def save_tensor(path: str, t: torch.Tensor) -> None: - t = t.detach().cpu().float().contiguous() - save_raw(path, t.numpy(), t.shape) - - -def save_ggml_bd(path: str, x_bd: torch.Tensor) -> None: - assert x_bd.ndim == 2 - x_bd = x_bd.detach().cpu().float().contiguous() - b, d = x_bd.shape - save_raw(path, x_bd.numpy(), (d, b)) - - -def save_ggml_bnd(path: str, x_bnd: torch.Tensor) -> None: - assert x_bnd.ndim == 3 - x_bnd = x_bnd.detach().cpu().float().contiguous() - b, n, d = x_bnd.shape - save_raw(path, x_bnd.numpy(), (d, n, b)) - - -def save_ggml_bhwc(path: str, x_bhwc: torch.Tensor) -> None: - assert x_bhwc.ndim == 4 - x_bhwc = x_bhwc.detach().cpu().float().contiguous() - b, h, w, c = x_bhwc.shape - save_raw(path, x_bhwc.numpy(), (c, w, h, b)) - - -def save_ggml_nchw(path: str, x_nchw: torch.Tensor) -> None: - assert x_nchw.ndim == 4 - save_ggml_bhwc(path, x_nchw.permute(0, 2, 3, 1)) - - -def save_ggml_masks(path: str, masks_bchw: torch.Tensor) -> None: - assert masks_bchw.ndim == 4 - b, c, h, w = masks_bchw.shape - save_ggml_bnd(path, masks_bchw.reshape(b, c, h * w)) - - -def parse_int_list(field: str) -> List[int]: - if not field: - return [] - vals = [] - for part in field.split("|"): - vals.append(-1 if part == "-" else int(part)) - return vals - - -def load_cases(path: str) -> List[Phase7Case]: - cases: List[Phase7Case] = [] - with open(path, "r", encoding="utf-8") as f: - for line in f: - line = line.rstrip("\n") - if not line: - continue - fields = line.split("\t") - while len(fields) < 5: - fields.append("") - mem_cases = fields[2].split("|") if fields[2] else [] - spatial_tpos = parse_int_list(fields[3]) - ptr_tpos = parse_int_list(fields[4]) - assert len(mem_cases) == len(spatial_tpos) == len(ptr_tpos) - cases.append( - Phase7Case( - case_id=fields[0], - label=fields[1], - mem_cases=mem_cases, - spatial_tpos=spatial_tpos, - ptr_tpos=ptr_tpos, - ) - ) - return cases - - -def layer_norm_2d(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: - u = x.mean(1, keepdim=True) - s = (x - u).pow(2).mean(1, keepdim=True) - x = (x - u) / torch.sqrt(s + eps) - return weight[:, None, None] * x + bias[:, None, None] - - -def sinusoidal_pe(h: int, w: int, d_model: int) -> torch.Tensor: - half = d_model // 2 - scale = 2.0 * math.pi - temperature = 10000.0 - - ys = torch.arange(1, h + 1, dtype=torch.float32).view(h, 1).repeat(1, w) - xs = torch.arange(1, w + 1, dtype=torch.float32).view(1, w).repeat(h, 1) - ys = ys / h * scale - xs = xs / w * scale - - dim_t = torch.arange(half, dtype=torch.float32) - dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / half) - - pos_x = xs[:, :, None] / dim_t - pos_y = ys[:, :, None] / dim_t - pos_x = torch.stack((pos_x[:, :, 0::2].sin(), pos_x[:, :, 1::2].cos()), dim=3).flatten(2) - pos_y = torch.stack((pos_y[:, :, 0::2].sin(), pos_y[:, :, 1::2].cos()), dim=3).flatten(2) - return torch.cat((pos_y, pos_x), dim=2).permute(2, 0, 1).unsqueeze(0).contiguous() - - -def get_1d_sine_pe(pos_inds: torch.Tensor, dim: int, temperature: float = 10000.0) -> torch.Tensor: - pe_dim = dim // 2 - dim_t = torch.arange(pe_dim, dtype=torch.float32, device=pos_inds.device) - dim_t = temperature ** (2 * torch.div(dim_t, 2, rounding_mode="floor") / pe_dim) - pos_embed = pos_inds.unsqueeze(-1) / dim_t - return torch.cat([pos_embed.sin(), pos_embed.cos()], dim=-1) - - -def init_t_xy(end_x, end_y, scale=1.0, offset=0, device=None): - t = torch.arange(end_x * end_y, dtype=torch.float32, device=device) - t_x = (t % end_x).float() - t_y = torch.div(t, end_x, rounding_mode="floor").float() - return t_x * scale + offset, t_y * scale + offset - - -def compute_axial_cis(dim, end_x, end_y, theta=10000.0, scale_pos=1.0, offset=0, device=None): - freqs_x = 1.0 / (theta ** (torch.arange(0, dim, 4, device=device)[: (dim // 4)].float() / dim)) - freqs_y = 1.0 / (theta ** (torch.arange(0, dim, 4, device=device)[: (dim // 4)].float() / dim)) - t_x, t_y = init_t_xy(end_x, end_y, scale_pos, offset, device=device) - freqs_x = torch.outer(t_x, freqs_x) - freqs_y = torch.outer(t_y, freqs_y) - freqs_cis_x = torch.polar(torch.ones_like(freqs_x), freqs_x) - freqs_cis_y = torch.polar(torch.ones_like(freqs_y), freqs_y) - return torch.cat([freqs_cis_x, freqs_cis_y], dim=-1) - - -def reshape_for_broadcast(freqs_cis: torch.Tensor, x: torch.Tensor) -> torch.Tensor: - shape = [d if i >= x.ndim - 2 else 1 for i, d in enumerate(x.shape)] - return freqs_cis.view(*shape) - - -def apply_rotary_enc(xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor, repeat_freqs_k: bool = False): - xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) if xk.shape[-2] != 0 else None - freqs_cis = reshape_for_broadcast(freqs_cis, xq_) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) - if xk_ is None: - return xq_out.type_as(xq), xk - if repeat_freqs_k: - r = xk_.shape[-2] // xq_.shape[-2] - freqs_cis = freqs_cis.repeat(*([1] * (freqs_cis.ndim - 2)), r, 1) - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) - return xq_out.type_as(xq), xk_out.type_as(xk) - - -def attention_forward( - q_in: torch.Tensor, - k_in: torch.Tensor, - v_in: torch.Tensor, - prefix: str, - weights: Dict[str, torch.Tensor], - num_heads: int, - rope_cis: torch.Tensor | None = None, - repeat_freqs_k: bool = False, - num_k_exclude_rope: int = 0, -) -> torch.Tensor: - q = F.linear(q_in, weights[prefix + ".q_proj.weight"].float(), weights[prefix + ".q_proj.bias"].float()) - k = F.linear(k_in, weights[prefix + ".k_proj.weight"].float(), weights[prefix + ".k_proj.bias"].float()) - v = F.linear(v_in, weights[prefix + ".v_proj.weight"].float(), weights[prefix + ".v_proj.bias"].float()) - - bsz, nq, dim = q.shape - nk = k.shape[1] - head_dim = dim // num_heads - - qh = q.view(bsz, nq, num_heads, head_dim).permute(0, 2, 1, 3) - kh = k.view(bsz, nk, num_heads, head_dim).permute(0, 2, 1, 3) - vh = v.view(bsz, nk, num_heads, head_dim).permute(0, 2, 1, 3) - - if rope_cis is not None: - nk_rope = kh.shape[2] - num_k_exclude_rope - if nk_rope > 0: - qh, kh_rope = apply_rotary_enc( - qh, - kh[:, :, :nk_rope, :], - rope_cis, - repeat_freqs_k=repeat_freqs_k, - ) - if nk_rope < kh.shape[2]: - kh = torch.cat([kh_rope, kh[:, :, nk_rope:, :]], dim=2) - else: - kh = kh_rope - - out = F.scaled_dot_product_attention(qh, kh, vh) - out = out.permute(0, 2, 1, 3).reshape(bsz, nq, dim) - return F.linear(out, weights[prefix + ".out_proj.weight"].float(), weights[prefix + ".out_proj.bias"].float()) - - -def mlp_forward(x: torch.Tensor, weights: Dict[str, torch.Tensor], prefix: str, num_layers: int) -> torch.Tensor: - for i in range(num_layers): - w = weights[f"{prefix}.layers.{i}.weight"].float() - b = weights[f"{prefix}.layers.{i}.bias"].float() - x = F.linear(x, w, b) - if i < num_layers - 1: - x = F.relu(x) - return x - - -def build_no_point_prompt_encoder_outputs(prompt_weights: Dict[str, torch.Tensor]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - embed_dim = 256 - feat_size = 72 - - pe_gaussian = prompt_weights["pe_layer.positional_encoding_gaussian_matrix"].float() - coords = torch.zeros((1, 1, 2), dtype=torch.float32) - labels = -torch.ones((1, 1), dtype=torch.int64) - - coords_centered = coords + 0.5 - coords_norm = coords_centered.clone() - coords_norm[:, :, 0] /= 1008.0 - coords_norm[:, :, 1] /= 1008.0 - coords_enc = 2.0 * coords_norm - 1.0 - coords_enc = coords_enc @ pe_gaussian - coords_enc = 2.0 * math.pi * coords_enc - point_embedding = torch.cat([torch.sin(coords_enc), torch.cos(coords_enc)], dim=-1) - - not_a_point = prompt_weights["not_a_point_embed.weight"].float()[0] - sparse = point_embedding.clone() - sparse[:, 0, :] = not_a_point - - ys = (torch.arange(feat_size, dtype=torch.float32) + 0.5) / feat_size - xs = (torch.arange(feat_size, dtype=torch.float32) + 0.5) / feat_size - yy, xx = torch.meshgrid(ys, xs, indexing="ij") - dense_coords = torch.stack([xx, yy], dim=-1) - dense_coords = 2.0 * dense_coords - 1.0 - dense_pe = dense_coords @ pe_gaussian - dense_pe = 2.0 * math.pi * dense_pe - image_pe = torch.cat([torch.sin(dense_pe), torch.cos(dense_pe)], dim=-1).permute(2, 0, 1).unsqueeze(0) - - no_mask = prompt_weights["no_mask_embed.weight"].float().view(1, embed_dim, 1, 1) - dense = no_mask.expand(1, embed_dim, feat_size, feat_size).contiguous() - return sparse, dense, image_pe - - -def run_sam_decoder( - mask_weights: Dict[str, torch.Tensor], - prompt_weights: Dict[str, torch.Tensor], - image_embeddings: torch.Tensor, - neck_trk_0: torch.Tensor, - neck_trk_1: torch.Tensor, -) -> dict[str, torch.Tensor]: - sparse_embeddings, dense_embeddings, image_pe = build_no_point_prompt_encoder_outputs(prompt_weights) - - feat_s0 = F.conv2d( - neck_trk_0, - mask_weights["conv_s0.weight"].float(), - mask_weights["conv_s0.bias"].float(), - ) - feat_s1 = F.conv2d( - neck_trk_1, - mask_weights["conv_s1.weight"].float(), - mask_weights["conv_s1.bias"].float(), - ) - - output_tokens = torch.cat( - [ - mask_weights["obj_score_token.weight"].float(), - mask_weights["iou_token.weight"].float(), - mask_weights["mask_tokens.weight"].float(), - ], - dim=0, - ).unsqueeze(0) - tokens = torch.cat((output_tokens, sparse_embeddings), dim=1) - - src = image_embeddings + dense_embeddings - pos_src = image_pe - bsz, dim, h, w = src.shape - keys = src.flatten(2).permute(0, 2, 1) - key_pe = pos_src.flatten(2).permute(0, 2, 1) - queries = tokens - query_pe = tokens - - for idx in range(2): - prefix = f"transformer.layers.{idx}" - if idx == 0: - queries = attention_forward( - queries, queries, queries, prefix + ".self_attn", mask_weights, 8 - ) - else: - q = queries + query_pe - attn_out = attention_forward(q, q, queries, prefix + ".self_attn", mask_weights, 8) - queries = queries + attn_out - queries = F.layer_norm( - queries, - [dim], - mask_weights[prefix + ".norm1.weight"].float(), - mask_weights[prefix + ".norm1.bias"].float(), - ) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - q, k, keys, prefix + ".cross_attn_token_to_image", mask_weights, 8 - ) - queries = F.layer_norm( - queries + attn_out, - [dim], - mask_weights[prefix + ".norm2.weight"].float(), - mask_weights[prefix + ".norm2.bias"].float(), - ) - - mlp_hidden = F.linear( - queries, - mask_weights[prefix + ".mlp.lin1.weight"].float(), - mask_weights[prefix + ".mlp.lin1.bias"].float(), - ) - mlp_hidden = F.relu(mlp_hidden) - mlp_hidden = F.linear( - mlp_hidden, - mask_weights[prefix + ".mlp.lin2.weight"].float(), - mask_weights[prefix + ".mlp.lin2.bias"].float(), - ) - queries = F.layer_norm( - queries + mlp_hidden, - [dim], - mask_weights[prefix + ".norm3.weight"].float(), - mask_weights[prefix + ".norm3.bias"].float(), - ) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - k, q, queries, prefix + ".cross_attn_image_to_token", mask_weights, 8 - ) - keys = F.layer_norm( - keys + attn_out, - [dim], - mask_weights[prefix + ".norm4.weight"].float(), - mask_weights[prefix + ".norm4.bias"].float(), - ) - - q = queries + query_pe - k = keys + key_pe - attn_out = attention_forward( - q, k, keys, "transformer.final_attn_token_to_image", mask_weights, 8 - ) - queries = F.layer_norm( - queries + attn_out, - [dim], - mask_weights["transformer.norm_final_attn.weight"].float(), - mask_weights["transformer.norm_final_attn.bias"].float(), - ) - - iou_token_out = queries[:, 1, :] - mask_tokens_out = queries[:, 2:6, :] - obj_token_out = queries[:, 0, :] - - src_img = keys.transpose(1, 2).view(bsz, dim, h, w) - up1 = F.conv_transpose2d( - src_img, - mask_weights["output_upscaling.0.weight"].float(), - mask_weights["output_upscaling.0.bias"].float(), - stride=2, - ) - up1 = up1 + feat_s1 - up1 = layer_norm_2d( - up1, - mask_weights["output_upscaling.1.weight"].float(), - mask_weights["output_upscaling.1.bias"].float(), - ) - up1 = F.gelu(up1) - - up2 = F.conv_transpose2d( - up1, - mask_weights["output_upscaling.3.weight"].float(), - mask_weights["output_upscaling.3.bias"].float(), - stride=2, - ) - up2 = F.gelu(up2 + feat_s0) - - hyper_in_list = [ - mlp_forward(mask_tokens_out[:, i, :], mask_weights, f"output_hypernetworks_mlps.{i}", 3) - for i in range(4) - ] - hyper_in = torch.stack(hyper_in_list, dim=1) - _, c, uh, uw = up2.shape - masks = (hyper_in @ up2.view(bsz, c, uh * uw)).view(bsz, -1, uh, uw) - iou_pred = mlp_forward(iou_token_out, mask_weights, "iou_prediction_head", 3).sigmoid() - obj_score_logits = mlp_forward(obj_token_out, mask_weights, "pred_obj_score_head", 3) - - return { - "sparse": sparse_embeddings, - "dense": dense_embeddings, - "image_pe": image_pe, - "mask_logits": masks[:, :1, :, :].contiguous(), - "iou": iou_pred[:, :1].contiguous(), - "obj_score": obj_score_logits.contiguous(), - "sam_token": mask_tokens_out[:, 0, :].contiguous(), - } - - -def run_memory_encoder( - tracker_weights: Dict[str, torch.Tensor], - neck_trk_2: torch.Tensor, - low_res_mask_logits: torch.Tensor, - obj_score_logit: torch.Tensor, -) -> dict[str, torch.Tensor]: - high_res_mask = F.interpolate( - low_res_mask_logits.float(), - size=(1008, 1008), - mode="bilinear", - align_corners=False, - ) - mask_for_mem = torch.sigmoid(high_res_mask) - mask_for_mem = mask_for_mem * 20.0 - 10.0 - - x = F.interpolate( - mask_for_mem.float(), - size=(1152, 1152), - mode="bilinear", - align_corners=False, - antialias=True, - ) - - conv_ids = [0, 3, 6, 9] - norm_ids = [1, 4, 7, 10] - for conv_id, norm_id in zip(conv_ids, norm_ids): - x = F.conv2d( - x, - tracker_weights[f"maskmem_backbone.mask_downsampler.encoder.{conv_id}.weight"].float(), - tracker_weights[f"maskmem_backbone.mask_downsampler.encoder.{conv_id}.bias"].float(), - stride=2, - padding=1, - ) - x = layer_norm_2d( - x, - tracker_weights[f"maskmem_backbone.mask_downsampler.encoder.{norm_id}.weight"].float(), - tracker_weights[f"maskmem_backbone.mask_downsampler.encoder.{norm_id}.bias"].float(), - ) - x = F.gelu(x) - - x = F.conv2d( - x, - tracker_weights["maskmem_backbone.mask_downsampler.encoder.12.weight"].float(), - tracker_weights["maskmem_backbone.mask_downsampler.encoder.12.bias"].float(), - ) - - pix_proj = F.conv2d( - neck_trk_2, - tracker_weights["maskmem_backbone.pix_feat_proj.weight"].float(), - tracker_weights["maskmem_backbone.pix_feat_proj.bias"].float(), - ) - fused = pix_proj + x - - block_outs = [] - h = fused - for idx in range(2): - prefix = f"maskmem_backbone.fuser.layers.{idx}" - residual = h - h = F.conv2d( - h, - tracker_weights[prefix + ".dwconv.weight"].float(), - tracker_weights[prefix + ".dwconv.bias"].float(), - padding=3, - groups=h.shape[1], - ) - h = layer_norm_2d( - h, - tracker_weights[prefix + ".norm.weight"].float(), - tracker_weights[prefix + ".norm.bias"].float(), - ) - h = h.permute(0, 2, 3, 1) - h = F.linear( - h, - tracker_weights[prefix + ".pwconv1.weight"].float(), - tracker_weights[prefix + ".pwconv1.bias"].float(), - ) - h = F.gelu(h) - h = F.linear( - h, - tracker_weights[prefix + ".pwconv2.weight"].float(), - tracker_weights[prefix + ".pwconv2.bias"].float(), - ) - gamma = tracker_weights[prefix + ".gamma"].float().view(1, 1, 1, -1) - h = residual + (gamma * h).permute(0, 3, 1, 2) - block_outs.append(h) - - out = F.conv2d( - h, - tracker_weights["maskmem_backbone.out_proj.weight"].float(), - tracker_weights["maskmem_backbone.out_proj.bias"].float(), - ) - - is_obj_appearing = (obj_score_logit > 0).float().view(-1, 1, 1, 1) - no_obj_embed = tracker_weights["no_obj_embed_spatial"].float().view(1, -1, 1, 1) - out = out + (1.0 - is_obj_appearing) * no_obj_embed - pos = sinusoidal_pe(72, 72, 64) - - return { - "high_res_mask": high_res_mask, - "mask_scaled": mask_for_mem, - "mask_downsampled": x, - "pix_proj": pix_proj, - "fused_input": fused, - "fuser0": block_outs[0], - "fuser1": block_outs[1], - "output": out, - "pos": pos, - } - - -def run_obj_ptr( - tracker_weights: Dict[str, torch.Tensor], - sam_token: torch.Tensor, - obj_score_logit: torch.Tensor, -) -> torch.Tensor: - h = sam_token - for i in range(3): - h = F.linear( - h, - tracker_weights[f"obj_ptr_proj.layers.{i}.weight"].float(), - tracker_weights[f"obj_ptr_proj.layers.{i}.bias"].float(), - ) - if i < 2: - h = F.relu(h) - no_obj_ptr = tracker_weights["no_obj_ptr"].float() - is_obj = (obj_score_logit > 0).float() - return is_obj * h + (1.0 - is_obj) * no_obj_ptr - - -def build_prompt_and_pos( - tracker_weights: Dict[str, torch.Tensor], - mem_outputs: List[dict[str, torch.Tensor]], - obj_ptrs: List[torch.Tensor], - case: Phase7Case, -) -> tuple[torch.Tensor, torch.Tensor, int]: - prompt_tokens: List[torch.Tensor] = [] - prompt_pos: List[torch.Tensor] = [] - - for idx, mem_out in enumerate(mem_outputs): - feats = mem_out["output"].flatten(2).permute(2, 0, 1).contiguous() - pos = mem_out["pos"].flatten(2).permute(2, 0, 1).contiguous() - tpos = case.spatial_tpos[idx] - if tpos > 0: - add = tracker_weights["maskmem_tpos_enc"].float()[7 - tpos - 1].view(1, 1, -1) - pos = pos + add - prompt_tokens.append(feats) - prompt_pos.append(pos) - - num_obj_ptr_tokens = 0 - ptr_tokens_list: List[torch.Tensor] = [] - ptr_pos_list: List[torch.Tensor] = [] - for idx, obj_ptr in enumerate(obj_ptrs): - ptr_rel = case.ptr_tpos[idx] - if ptr_rel < 0: - continue - pos_ind = torch.tensor([ptr_rel / 15.0], dtype=torch.float32) - obj_pos = get_1d_sine_pe(pos_ind, dim=256) - obj_pos = F.linear( - obj_pos, - tracker_weights["obj_ptr_tpos_proj.weight"].float(), - tracker_weights["obj_ptr_tpos_proj.bias"].float(), - ) - ptr = obj_ptr.view(1, 1, 4, 64).permute(1, 2, 0, 3).reshape(4, 1, 64) - pos = obj_pos.view(1, 1, 64).repeat_interleave(4, dim=0) - ptr_tokens_list.append(ptr) - ptr_pos_list.append(pos) - - if ptr_tokens_list: - ptr_tokens = torch.cat(ptr_tokens_list, dim=0) - ptr_pos = torch.cat(ptr_pos_list, dim=0) - prompt_tokens.append(ptr_tokens) - prompt_pos.append(ptr_pos) - num_obj_ptr_tokens = ptr_tokens.shape[0] - - return torch.cat(prompt_tokens, dim=0), torch.cat(prompt_pos, dim=0), num_obj_ptr_tokens - - -def run_memory_attention( - tracker_weights: Dict[str, torch.Tensor], - curr_neck_trk_2: torch.Tensor, - prompt: torch.Tensor, - prompt_pos: torch.Tensor, - num_obj_ptr_tokens: int, -) -> dict[str, torch.Tensor]: - src = curr_neck_trk_2.flatten(2).permute(2, 0, 1).contiguous() - src_pos = sinusoidal_pe(72, 72, 256).flatten(2).permute(2, 0, 1).contiguous() - - x = src.permute(1, 0, 2).contiguous() + 0.1 * src_pos.permute(1, 0, 2).contiguous() - mem = prompt.permute(1, 0, 2).contiguous() - mem_pos = prompt_pos.permute(1, 0, 2).contiguous() - - rope_cis = compute_axial_cis(dim=256, end_x=72, end_y=72) - out = {"input": x} - for idx in range(4): - prefix = f"transformer.encoder.layers.{idx}" - - x_norm = F.layer_norm( - x, - [256], - tracker_weights[prefix + ".norm1.weight"].float(), - tracker_weights[prefix + ".norm1.bias"].float(), - ) - sa = attention_forward( - x_norm, - x_norm, - x_norm, - prefix + ".self_attn", - tracker_weights, - 1, - rope_cis=rope_cis, - ) - x = x + sa - out[f"layer{idx}_after_sa"] = x - - x_norm = F.layer_norm( - x, - [256], - tracker_weights[prefix + ".norm2.weight"].float(), - tracker_weights[prefix + ".norm2.bias"].float(), - ) - ca = attention_forward( - x_norm, - mem + mem_pos, - mem, - prefix + ".cross_attn_image", - tracker_weights, - 1, - rope_cis=rope_cis, - repeat_freqs_k=True, - num_k_exclude_rope=num_obj_ptr_tokens, - ) - x = x + ca - out[f"layer{idx}_after_ca"] = x - - x_norm = F.layer_norm( - x, - [256], - tracker_weights[prefix + ".norm3.weight"].float(), - tracker_weights[prefix + ".norm3.bias"].float(), - ) - ffn = F.linear( - x_norm, - tracker_weights[prefix + ".linear1.weight"].float(), - tracker_weights[prefix + ".linear1.bias"].float(), - ) - ffn = F.relu(ffn) - ffn = F.linear( - ffn, - tracker_weights[prefix + ".linear2.weight"].float(), - tracker_weights[prefix + ".linear2.bias"].float(), - ) - x = x + ffn - out[f"layer{idx}_after_ffn"] = x - - x = F.layer_norm( - x, - [256], - tracker_weights["transformer.encoder.norm.weight"].float(), - tracker_weights["transformer.encoder.norm.bias"].float(), - ) - out["output"] = x - return out - - -def write_meta(case_dir: str, case: Phase7Case) -> None: - with open(os.path.join(case_dir, "meta.txt"), "w", encoding="utf-8") as f: - f.write(f"num_slots={len(case.mem_cases)}\n") - for i, tpos in enumerate(case.spatial_tpos): - f.write(f"slot{i}_spatial_tpos={tpos}\n") - f.write(f"slot{i}_ptr_tpos={case.ptr_tpos[i]}\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--prephase-ref", default="tests/ref_phase3") - parser.add_argument("--phase6-ref", default="tests/ref_phase6") - parser.add_argument("--cases", default="tests/phase7_cases.tsv") - parser.add_argument("--outdir", default="tests/ref_phase7") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - - ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - tracker_weights = { - k[len("tracker."):]: v - for k, v in ckpt.items() - if k.startswith("tracker.") - } - prompt_weights = { - k[len("tracker.sam_prompt_encoder."):]: v - for k, v in ckpt.items() - if k.startswith("tracker.sam_prompt_encoder.") - } - mask_weights = { - k[len("tracker.sam_mask_decoder."):]: v - for k, v in ckpt.items() - if k.startswith("tracker.sam_mask_decoder.") - } - - neck_trk_0 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_0")).float() - neck_trk_1 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_1")).float() - neck_trk_2 = load_tensor(os.path.join(args.prephase_ref, "neck_trk_2")).float() - - cases = load_cases(args.cases) - for case in cases: - case_dir = os.path.join(args.outdir, case.case_id) - os.makedirs(case_dir, exist_ok=True) - write_meta(case_dir, case) - - save_tensor(os.path.join(case_dir, "input_curr_neck_trk_0"), neck_trk_0) - save_tensor(os.path.join(case_dir, "input_curr_neck_trk_1"), neck_trk_1) - save_tensor(os.path.join(case_dir, "input_curr_neck_trk_2"), neck_trk_2) - - mem_outputs = [] - obj_ptrs = [] - for idx, mem_case in enumerate(case.mem_cases): - masks = load_ggml_masks(os.path.join(args.phase6_ref, mem_case, "sam_dec_masks"), 288, 288).float() - low_res_mask = masks[:, :1, :, :].contiguous() - sam_token = load_ggml_bd(os.path.join(args.phase6_ref, mem_case, "sam_dec_sam_token")).float() - obj_score = load_ggml_bd(os.path.join(args.phase6_ref, mem_case, "sam_dec_obj_score")).float() - - save_tensor(os.path.join(case_dir, f"input_mem_mask_logits_{idx}"), low_res_mask) - save_tensor(os.path.join(case_dir, f"input_mem_sam_token_{idx}"), sam_token) - save_tensor(os.path.join(case_dir, f"input_mem_obj_score_{idx}"), obj_score) - - mem_out = run_memory_encoder(tracker_weights, neck_trk_2, low_res_mask, obj_score) - obj_ptr = run_obj_ptr(tracker_weights, sam_token, obj_score) - mem_outputs.append(mem_out) - obj_ptrs.append(obj_ptr) - - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_high_res_mask"), mem_out["high_res_mask"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_mask_scaled"), mem_out["mask_scaled"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_mask_downsampled"), mem_out["mask_downsampled"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_pix_proj"), mem_out["pix_proj"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_fused_input"), mem_out["fused_input"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_fuser0"), mem_out["fuser0"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_fuser1"), mem_out["fuser1"]) - save_ggml_nchw(os.path.join(case_dir, f"phase7_mem{idx}_output"), mem_out["output"]) - save_ggml_bd(os.path.join(case_dir, f"phase7_obj_ptr{idx}"), obj_ptr) - - prompt, prompt_pos, num_obj_ptr_tokens = build_prompt_and_pos( - tracker_weights, mem_outputs, obj_ptrs, case - ) - mem_attn = run_memory_attention( - tracker_weights, - neck_trk_2, - prompt, - prompt_pos, - num_obj_ptr_tokens, - ) - - save_ggml_bnd(os.path.join(case_dir, "phase7_mem_attn_input"), mem_attn["input"]) - for idx in range(4): - save_ggml_bnd(os.path.join(case_dir, f"phase7_mem_attn_layer{idx}_after_sa"), mem_attn[f"layer{idx}_after_sa"]) - save_ggml_bnd(os.path.join(case_dir, f"phase7_mem_attn_layer{idx}_after_ca"), mem_attn[f"layer{idx}_after_ca"]) - save_ggml_bnd(os.path.join(case_dir, f"phase7_mem_attn_layer{idx}_after_ffn"), mem_attn[f"layer{idx}_after_ffn"]) - save_ggml_bnd(os.path.join(case_dir, "phase7_mem_attn_output"), mem_attn["output"]) - - conditioned = mem_attn["output"].permute(0, 2, 1).reshape(1, 256, 72, 72).contiguous() - prop = run_sam_decoder(mask_weights, prompt_weights, conditioned, neck_trk_0, neck_trk_1) - save_ggml_masks(os.path.join(case_dir, "phase7_prop_masks"), prop["mask_logits"]) - save_ggml_bd(os.path.join(case_dir, "phase7_prop_iou"), prop["iou"]) - save_ggml_bd(os.path.join(case_dir, "phase7_prop_obj_score"), prop["obj_score"]) - save_ggml_bd(os.path.join(case_dir, "phase7_prop_sam_token"), prop["sam_token"]) - - print(f"Saved Phase 7 references to {args.outdir}") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_reference.py b/tests/dump_reference.py deleted file mode 100644 index 85b57f2..0000000 --- a/tests/dump_reference.py +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump reference intermediate tensors from the SAM3 Python model for C++ verification. -Saves .bin files with raw float32 data and .shape files with tensor dimensions. - -Usage: - python tests/dump_reference.py --checkpoint raw_weights/sam3.pt --image tests/test_truck.jpg --outdir tests/ref/ -""" -import argparse -import os -import sys -import struct -import math - -import numpy as np -import torch -import torch.nn.functional as F -from PIL import Image -from torchvision.transforms import v2 - -# Add SAM3 repo to path -sys.path.insert(0, os.path.expanduser("~/Documents/sam3")) -from sam3.model.vitdet import ViT, compute_axial_cis, window_partition, window_unpartition -from sam3.model_builder import _create_vit_backbone - - -def save_tensor(path, t): - """Save a tensor as raw float32 binary + shape file.""" - t = t.detach().cpu().float().contiguous() - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)} bytes={t.numel()*4}") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True, help="Path to sam3.pt") - parser.add_argument("--image", required=True, help="Path to test image") - parser.add_argument("--outdir", default="tests/ref", help="Output directory") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" # Use CPU for reproducibility - - # ── Load model ───────────────────────────────────────────────────────── - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - state_dict = ckpt if isinstance(ckpt, dict) and "model" not in ckpt else ckpt.get("model", ckpt) - - # Extract ViT weights - vit_prefix = "detector.backbone.visual.trunk." - vit_state = {k[len(vit_prefix):]: v for k, v in state_dict.items() if k.startswith(vit_prefix)} - - print(f" Found {len(vit_state)} ViT parameters") - - # ── Create ViT backbone ──────────────────────────────────────────────── - print("Creating ViT backbone...") - vit = _create_vit_backbone(compile_mode=None) - vit.use_act_checkpoint = False # Disable for inference - missing, unexpected = vit.load_state_dict(vit_state, strict=False) - if missing: - print(f" WARNING: Missing keys: {missing[:5]}...") - if unexpected: - print(f" WARNING: Unexpected keys: {unexpected[:5]}...") - vit.eval() - vit.to(device) - - # ── Load and preprocess image ────────────────────────────────────────── - print(f"Loading image: {args.image}") - img = Image.open(args.image).convert("RGB") - print(f" Original size: {img.size}") - - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - - img_tensor = v2.functional.to_image(img) - img_preprocessed = transform(img_tensor).unsqueeze(0).to(device) - print(f" Preprocessed shape: {list(img_preprocessed.shape)}") - save_tensor(os.path.join(args.outdir, "preprocessed"), img_preprocessed) - - # ── Step-by-step forward pass ────────────────────────────────────────── - with torch.no_grad(): - # 1. Patch embedding - x = vit.patch_embed(img_preprocessed) - print(f" After patch_embed: {list(x.shape)}") # [1, 72, 72, 1024] - save_tensor(os.path.join(args.outdir, "patch_embed"), x) - - # 2. Positional embedding - h, w = x.shape[1], x.shape[2] - from sam3.model.vitdet import get_abs_pos - pos = get_abs_pos(vit.pos_embed, vit.pretrain_use_cls_token, (h, w), - retain_cls_token=False, tiling=vit.tile_abs_pos) - print(f" Pos embed (tiled): {list(pos.shape)}") # [1, 72, 72, 1024] - save_tensor(os.path.join(args.outdir, "pos_embed_tiled"), pos) - - x = x + pos - save_tensor(os.path.join(args.outdir, "after_pos_embed"), x) - - # 3. ln_pre - x = vit.ln_pre(x) - save_tensor(os.path.join(args.outdir, "after_ln_pre"), x) - - # 4. Block 0 output - x = vit.blocks[0](x) - save_tensor(os.path.join(args.outdir, "block_0_out"), x) - - # 5. All remaining blocks - for i in range(1, len(vit.blocks)): - x = vit.blocks[i](x) - - save_tensor(os.path.join(args.outdir, "vit_final"), x) - - # 6. Permute to [B, C, H, W] - feats = x.permute(0, 3, 1, 2) - save_tensor(os.path.join(args.outdir, "vit_output_bchw"), feats) - - # ── RoPE reference ───────────────────────────────────────────────────── - print("\nDumping RoPE reference values...") - - # Window attention RoPE (24x24, rope_pt_size=24, no interp needed since pt_size == input_size) - # But wait — for window blocks, input_size is (24, 24), rope_pt_size is from the config - # Looking at the model_builder: rope_pt_size=None → defaults to (window_size, window_size) = (24, 24) - # And rope_interp=True, scale_pos = 24/24 = 1.0 → no scaling - freqs_cis_window = compute_axial_cis(dim=64, end_x=24, end_y=24, theta=10000.0, scale_pos=1.0) - save_tensor(os.path.join(args.outdir, "rope_window_complex"), freqs_cis_window) - # Convert to real (cos, sin) pairs: freqs_cis is complex [576, 32] - freqs_real_window = torch.view_as_real(freqs_cis_window) # [576, 32, 2] - save_tensor(os.path.join(args.outdir, "rope_window_real"), freqs_real_window) - - # Global attention RoPE (72x72, rope_pt_size=24, rope_interp=True → scale_pos=24/72=1/3) - # For global blocks: input_size = (72, 72), rope_pt_size = (24, 24) (from None default) - # Wait — actually, looking at the code: - # Block.__init__: input_size=input_size if window_size == 0 else (window_size, window_size) - # For global blocks, window_size=0, so input_size=(72, 72) - # Attention.__init__: rope_pt_size from Block, which is (24,24) for rope_pt_size=None - # scale_pos = rope_pt_size[0] / input_size[0] = 24 / 72 = 1/3 - scale_pos_global = 24.0 / 72.0 - freqs_cis_global = compute_axial_cis(dim=64, end_x=72, end_y=72, theta=10000.0, scale_pos=scale_pos_global) - freqs_real_global = torch.view_as_real(freqs_cis_global) # [5184, 32, 2] - save_tensor(os.path.join(args.outdir, "rope_global_real"), freqs_real_global) - - # ── Neck reference ───────────────────────────────────────────────────── - print("\nDumping neck reference values...") - # Load neck weights - det_neck_prefix = "detector.backbone.visual.convs." - # We need to instantiate the neck too, but for now just save the ViT output - # The neck test will be done separately once we verify ViT outputs match - - # ── Sinusoidal PE reference ──────────────────────────────────────────── - print("\nDumping sinusoidal PE reference values...") - from sam3.model.position_encoding import PositionEmbeddingSine - pe = PositionEmbeddingSine(num_pos_feats=256, normalize=True, scale=None, temperature=10000) - - for H, W, name in [(288, 288, "pe_288"), (144, 144, "pe_144"), (72, 72, "pe_72"), (36, 36, "pe_36")]: - dummy = torch.zeros(1, 1, H, W) - pe_out = pe(dummy) # [1, 256, H, W] - save_tensor(os.path.join(args.outdir, name), pe_out) - - print("\nDone! Reference tensors saved to", args.outdir) - - -if __name__ == "__main__": - main() diff --git a/tests/dump_sam2_pvs_reference.py b/tests/dump_sam2_pvs_reference.py deleted file mode 100644 index 0f20fbe..0000000 --- a/tests/dump_sam2_pvs_reference.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python3 -"""Dump SAM2 PVS decoder reference tensors for C++ comparison. - -Usage: - cd ~/Documents/sam2 - SAM2_DUMP_DIR=/tmp/debug_sam2_pvs \ - python /path/to/dump_sam2_pvs_reference.py \ - --checkpoint /path/to/sam2.1_hiera_base_plus.pt \ - --config configs/sam2.1/sam2.1_hiera_b+.yaml \ - --image /path/to/test_image.jpg \ - --point-x 600 --point-y 599 -""" -import argparse, os, sys, numpy as np -sys.path.insert(0, os.path.expanduser("~/Documents/sam2")) -import torch -torch.set_default_dtype(torch.float32) -from PIL import Image -from sam2.build_sam import build_sam2 -from sam2.sam2_image_predictor import SAM2ImagePredictor - -DUMP_DIR = os.environ.get("SAM2_DUMP_DIR", "/tmp/debug_sam2_pvs") - -def save(name, t): - os.makedirs(DUMP_DIR, exist_ok=True) - t = t.detach().cpu().float().contiguous() - path = os.path.join(DUMP_DIR, name) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" {name}: {tuple(t.shape)}") - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--config", required=True) - parser.add_argument("--image", required=True) - parser.add_argument("--point-x", type=float, default=600) - parser.add_argument("--point-y", type=float, default=599) - args = parser.parse_args() - - model = build_sam2(args.config, args.checkpoint, device="cpu") - model = model.float().eval() - predictor = SAM2ImagePredictor(model) - - image = np.array(Image.open(args.image).convert("RGB")) - print(f"Image: {args.image} ({image.shape[1]}x{image.shape[0]})") - - # Set image and get features - with torch.no_grad(): - predictor.set_image(image) - - # Dump features stored after set_image - print("\n=== Features after set_image ===") - save("features_image_embed", predictor._features["image_embed"]) - for i, f in enumerate(predictor._features["high_res_feats"]): - save(f"features_high_res_{i}", f) - - # Run PVS prediction - point_coords = np.array([[args.point_x, args.point_y]], dtype=np.float32) - point_labels = np.array([1], dtype=np.int32) - - print(f"\n=== PVS with point ({args.point_x}, {args.point_y}) ===") - - # Manually trace the prediction path for dumps - with torch.no_grad(): - # Step 1: Coordinate normalization (matching _prep_prompts) - unnorm_coords = predictor._transforms.transform_coords( - torch.from_numpy(point_coords).unsqueeze(0).float(), - normalize=True, - orig_hw=(image.shape[0], image.shape[1]) - ) - labels = torch.from_numpy(point_labels).unsqueeze(0).int() - concat_points = (unnorm_coords, labels) - save("unnorm_coords", unnorm_coords.squeeze(0)) - print(f" unnorm_coords: {unnorm_coords.squeeze(0).numpy()}") - - # Step 2: Prompt encoder - sparse_emb, dense_emb = model.sam_prompt_encoder( - points=concat_points, boxes=None, masks=None - ) - save("sparse_embeddings", sparse_emb) - save("dense_embeddings", dense_emb) - - # Step 3: Get dense PE - image_pe = model.sam_prompt_encoder.get_dense_pe() - save("image_pe", image_pe) - - # Step 4: Prepare high-res features - high_res_features = [ - feat_level[0].unsqueeze(0) - for feat_level in predictor._features["high_res_feats"] - ] - for i, hrf in enumerate(high_res_features): - save(f"high_res_for_decoder_{i}", hrf) - - # Step 5: Run mask decoder - image_embed = predictor._features["image_embed"][0].unsqueeze(0) - save("image_embed_for_decoder", image_embed) - - low_res_masks, iou_predictions, _, _ = model.sam_mask_decoder( - image_embeddings=image_embed, - image_pe=image_pe, - sparse_prompt_embeddings=sparse_emb, - dense_prompt_embeddings=dense_emb, - multimask_output=True, - repeat_image=False, - high_res_features=high_res_features, - ) - save("low_res_masks", low_res_masks) - save("iou_predictions", iou_predictions) - print(f" iou_predictions: {iou_predictions.numpy()}") - print(f" masks range: [{low_res_masks.min():.4f}, {low_res_masks.max():.4f}]") - - # Also run through the full predict API for comparison - with torch.no_grad(): - masks, scores, logits = predictor.predict( - point_coords=point_coords, - point_labels=point_labels, - multimask_output=True, - normalize_coords=True, - ) - save("api_masks", torch.from_numpy(masks).float()) - save("api_scores", torch.from_numpy(scores).float()) - save("api_logits", torch.from_numpy(logits).float()) - print(f"\n API scores: {scores}") - print(f" API masks shape: {masks.shape}") - - print(f"\nDone. Tensors in {DUMP_DIR}/") - -if __name__ == "__main__": - main() diff --git a/tests/dump_sam2_reference.py b/tests/dump_sam2_reference.py deleted file mode 100644 index db216ff..0000000 --- a/tests/dump_sam2_reference.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python3 -"""Dump SAM2 reference tensors for C++ comparison. - -Usage: - cd ~/Documents/sam2 - SAM2_DUMP_DIR=/tmp/debug_sam2_pipeline \ - python /path/to/dump_sam2_reference.py \ - --checkpoint /path/to/sam2.1_hiera_base_plus.pt \ - --config configs/sam2.1/sam2.1_hiera_b+.yaml \ - --image /path/to/test_image.jpg \ - --stage all -""" - -import argparse -import os -import sys -import numpy as np - -# Add SAM2 to path -sys.path.insert(0, os.path.expanduser("~/Documents/sam2")) - -import torch -torch.set_default_dtype(torch.float32) - -from PIL import Image -from sam2.build_sam import build_sam2 -from sam2.sam2_image_predictor import SAM2ImagePredictor - -# Import dump utilities -sys.path.insert(0, os.path.expanduser("~/Documents/sam2")) -from debug_pipeline_utils import save_tensor, DUMP_DIR - - -def dump_preprocessing(predictor, image_np): - """Stage 0: Dump preprocessed image tensor.""" - print("\n=== Stage 0: Preprocessing ===") - # Run the transforms manually - from sam2.utils.transforms import SAM2Transforms - transforms = predictor._transforms - - # Get the preprocessed image - from torchvision.transforms import ToTensor - img_tensor = ToTensor()(Image.fromarray(image_np)) # [3, H, W] float [0,1] - - # Resize - from torchvision.transforms.functional import resize - img_resized = resize(img_tensor, [predictor.model.image_size, predictor.model.image_size], - antialias=True) - - # Normalize (ImageNet) - mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1) - std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1) - img_normalized = (img_resized - mean) / std - - save_tensor("preprocessed_image", img_normalized.unsqueeze(0)) - print(f" preprocessed_image: {img_normalized.shape}") - - # Also save as CHW float for C++ consumption - chw = img_normalized.numpy().astype(np.float32) - with open(os.path.join(DUMP_DIR, "preprocessed.bin"), "wb") as f: - f.write(chw.tobytes()) - print(f" preprocessed.bin: CHW float32, shape {chw.shape}") - - return img_normalized.unsqueeze(0) - - -def dump_backbone(predictor, img_batch): - """Stage 1: Dump Hiera backbone outputs.""" - print("\n=== Stage 1: Hiera Backbone ===") - model = predictor.model - - # Run backbone trunk - with torch.no_grad(): - backbone_out = model.image_encoder.trunk(img_batch) - - # backbone_out is a list of intermediate features - # For Hiera, the output is typically from the trunk's forward which returns intermediates - print(f" Backbone output type: {type(backbone_out)}") - - if isinstance(backbone_out, (list, tuple)): - for i, feat in enumerate(backbone_out): - save_tensor(f"hiera_stage_{i}", feat) - print(f" hiera_stage_{i}: {feat.shape}") - elif isinstance(backbone_out, dict): - for k, v in backbone_out.items(): - if isinstance(v, torch.Tensor): - save_tensor(f"backbone_{k}", v) - print(f" backbone_{k}: {v.shape}") - - return backbone_out - - -def dump_fpn_neck(predictor, img_batch): - """Stage 2: Dump FPN neck outputs (backbone + neck together).""" - print("\n=== Stage 2: FPN Neck ===") - model = predictor.model - - with torch.no_grad(): - backbone_out = model.image_encoder(img_batch) - - # backbone_out is a dict with: - # "vision_features": src - # "vision_pos_enc": pos - # "backbone_fpn": features - for k, v in backbone_out.items(): - if isinstance(v, torch.Tensor): - save_tensor(f"imgenc_{k}", v) - print(f" imgenc_{k}: {v.shape}") - elif isinstance(v, (list, tuple)): - for i, item in enumerate(v): - if isinstance(item, torch.Tensor): - save_tensor(f"imgenc_{k}_{i}", item) - print(f" imgenc_{k}_{i}: {item.shape}") - - return backbone_out - - -def dump_image_features(predictor, image_np): - """Stage 3: Dump prepared image features (what set_image produces).""" - print("\n=== Stage 3: Image Features ===") - - with torch.no_grad(): - predictor.set_image(image_np) - - # Access cached features - features = predictor._features - for k, v in features.items(): - if isinstance(v, torch.Tensor): - save_tensor(f"features_{k}", v) - print(f" features_{k}: {v.shape}") - elif isinstance(v, (list, tuple)): - for i, item in enumerate(v): - if isinstance(item, torch.Tensor): - save_tensor(f"features_{k}_{i}", item) - print(f" features_{k}_{i}: {item.shape}") - - return features - - -def dump_pvs_inference(predictor, image_np, point_coords, point_labels): - """Stage 4-5: Dump prompt encoding + mask decoding.""" - print("\n=== Stage 4-5: PVS Inference ===") - - with torch.no_grad(): - if not predictor._is_image_set: - predictor.set_image(image_np) - - masks, scores, logits = predictor.predict( - point_coords=point_coords, - point_labels=point_labels, - multimask_output=True, - normalize_coords=True, - ) - - save_tensor("pvs_masks", torch.from_numpy(masks).float()) - save_tensor("pvs_scores", torch.from_numpy(scores).float()) - save_tensor("pvs_logits", torch.from_numpy(logits).float()) - print(f" pvs_masks: {masks.shape}") - print(f" pvs_scores: {scores.shape}, values: {scores}") - print(f" pvs_logits: {logits.shape}") - - return masks, scores, logits - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--config", required=True) - parser.add_argument("--image", required=True) - parser.add_argument("--stage", default="all", - help="all, preprocess, backbone, fpn, features, pvs") - parser.add_argument("--point-x", type=float, default=None) - parser.add_argument("--point-y", type=float, default=None) - args = parser.parse_args() - - os.makedirs(DUMP_DIR, exist_ok=True) - print(f"Dump directory: {DUMP_DIR}") - - # Load model - print(f"\nLoading model: {args.config}") - model = build_sam2( - config_file=args.config, - ckpt_path=args.checkpoint, - device="cpu", - ) - model = model.float() - model.eval() - - predictor = SAM2ImagePredictor(model) - - # Load image - image = np.array(Image.open(args.image).convert("RGB")) - print(f"Image: {args.image} ({image.shape[1]}x{image.shape[0]})") - - # Default point at center of image - px = args.point_x if args.point_x is not None else image.shape[1] / 2 - py = args.point_y if args.point_y is not None else image.shape[0] / 2 - point_coords = np.array([[px, py]], dtype=np.float32) - point_labels = np.array([1], dtype=np.int32) - print(f"Test point: ({px}, {py})") - - stages = args.stage.split(",") if args.stage != "all" else [ - "preprocess", "backbone", "fpn", "features", "pvs" - ] - - img_batch = None - - if "preprocess" in stages: - img_batch = dump_preprocessing(predictor, image) - - if "backbone" in stages: - if img_batch is None: - img_batch = dump_preprocessing(predictor, image) - dump_backbone(predictor, img_batch) - - if "fpn" in stages: - if img_batch is None: - img_batch = dump_preprocessing(predictor, image) - dump_fpn_neck(predictor, img_batch) - - if "features" in stages: - dump_image_features(predictor, image) - - if "pvs" in stages: - dump_pvs_inference(predictor, image, point_coords, point_labels) - - print(f"\nDone. Tensors saved to {DUMP_DIR}/") - - -if __name__ == "__main__": - main() diff --git a/tests/dump_vit_reference.py b/tests/dump_vit_reference.py deleted file mode 100644 index f88cea4..0000000 --- a/tests/dump_vit_reference.py +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env python3 -""" -Dump ViT backbone intermediate tensors from the SAM3 checkpoint for C++ verification. -Uses inlined functions to avoid heavy SAM3 dependencies. - -Usage: - uv run python tests/dump_vit_reference.py --checkpoint raw_weights/sam3.pt --image tests/test_random.jpg -""" -import argparse -import math -import os -import sys -from functools import partial -from typing import Tuple - -import numpy as np -import torch -import torch.nn as nn -import torch.nn.functional as F -from PIL import Image -from torchvision.transforms import v2 - - -# ── Inlined from vitdet.py ───────────────────────────────────────────────── - -def init_t_xy(end_x, end_y, scale=1.0, offset=0): - t = torch.arange(end_x * end_y, dtype=torch.float32) - t_x = (t % end_x).float() - t_y = torch.div(t, end_x, rounding_mode="floor").float() - return t_x * scale + offset, t_y * scale + offset - - -def compute_axial_cis(dim, end_x, end_y, theta=10000.0, scale_pos=1.0, offset=0): - freqs_x = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - freqs_y = 1.0 / (theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim)) - t_x, t_y = init_t_xy(end_x, end_y, scale_pos, offset) - freqs_x = torch.outer(t_x, freqs_x) - freqs_y = torch.outer(t_y, freqs_y) - freqs_cis_x = torch.polar(torch.ones_like(freqs_x), freqs_x) - freqs_cis_y = torch.polar(torch.ones_like(freqs_y), freqs_y) - return torch.cat([freqs_cis_x, freqs_cis_y], dim=-1) - - -def reshape_for_broadcast(freqs_cis, x): - ndim = x.ndim - shape = [d if i >= ndim - 2 else 1 for i, d in enumerate(x.shape)] - return freqs_cis.view(*shape) - - -def apply_rotary_enc(xq, xk, freqs_cis, repeat_freqs_k=False): - xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) if xk.shape[-2] != 0 else None - freqs_cis = reshape_for_broadcast(freqs_cis, xq_) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) - if xk_ is None: - return xq_out.type_as(xq), xk - if repeat_freqs_k: - r = xk_.shape[-2] // xq_.shape[-2] - freqs_cis = freqs_cis.repeat(*([1] * (freqs_cis.ndim - 2)), r, 1) - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) - return xq_out.type_as(xq), xk_out.type_as(xk) - - -def window_partition(x, window_size): - B, H, W, C = x.shape - pad_h = (window_size - H % window_size) % window_size - pad_w = (window_size - W % window_size) % window_size - if pad_h > 0 or pad_w > 0: - x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) - Hp, Wp = H + pad_h, W + pad_w - x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) - windows = x.permute(0, 1, 3, 2, 4, 5).reshape(-1, window_size, window_size, C) - return windows, (Hp, Wp) - - -def window_unpartition(windows, window_size, pad_hw, hw): - Hp, Wp = pad_hw - H, W = hw - B = windows.shape[0] // (Hp * Wp // window_size // window_size) - x = windows.reshape(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) - x = x.permute(0, 1, 3, 2, 4, 5).reshape(B, Hp, Wp, -1) - if Hp > H or Wp > W: - x = x[:, :H, :W, :] - return x - - -def get_abs_pos(abs_pos, has_cls_token, hw, tiling=False): - h, w = hw - if has_cls_token: - abs_pos = abs_pos[:, 1:] - xy_num = abs_pos.shape[1] - size = int(math.sqrt(xy_num)) - if size != h or size != w: - new_abs_pos = abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2) - if tiling: - new_abs_pos = new_abs_pos.tile( - [1, 1] + [x // y + 1 for x, y in zip((h, w), new_abs_pos.shape[2:])] - )[:, :, :h, :w] - else: - new_abs_pos = F.interpolate(new_abs_pos, size=(h, w), mode="bicubic", align_corners=False) - return new_abs_pos.permute(0, 2, 3, 1) - return abs_pos.reshape(1, h, w, -1) - - -# ── Minimal ViT block implementation ────────────────────────────────────── - -try: - from timm.layers import Mlp -except ImportError: - class Mlp(nn.Module): - def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): - super().__init__() - out_features = out_features or in_features - hidden_features = hidden_features or in_features - self.fc1 = nn.Linear(in_features, hidden_features) - self.act = act_layer() - self.fc2 = nn.Linear(hidden_features, out_features) - def forward(self, x): - return self.fc2(self.act(self.fc1(x))) - - -def save_tensor(path, t): - t = t.detach().cpu().float().contiguous() - os.makedirs(os.path.dirname(path) or ".", exist_ok=True) - with open(path + ".bin", "wb") as f: - f.write(t.numpy().tobytes()) - with open(path + ".shape", "w") as f: - f.write(",".join(str(d) for d in t.shape)) - print(f" saved {path} shape={list(t.shape)}") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--checkpoint", required=True) - parser.add_argument("--image", required=True) - parser.add_argument("--outdir", default="tests/ref") - args = parser.parse_args() - - os.makedirs(args.outdir, exist_ok=True) - device = "cpu" - - # ── Load checkpoint ───────────────────────────────────────────────── - print("Loading checkpoint...") - ckpt = torch.load(args.checkpoint, map_location=device, weights_only=False) - if "model" in ckpt: - ckpt = ckpt["model"] - - # Extract ViT weights - vit_prefix = "detector.backbone.vision_backbone.trunk." - vit_sd = {k[len(vit_prefix):]: v for k, v in ckpt.items() if k.startswith(vit_prefix)} - print(f" Found {len(vit_sd)} ViT keys") - - # ── Load and preprocess image ───────────────────────────────────── - print(f"Loading image: {args.image}") - img = Image.open(args.image).convert("RGB") - transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(1008, 1008)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), - ]) - img_tensor = v2.functional.to_image(img) - img_preprocessed = transform(img_tensor).unsqueeze(0).to(device) - save_tensor(os.path.join(args.outdir, "preprocessed"), img_preprocessed) - - # ── Manual ViT forward pass ─────────────────────────────────────── - with torch.no_grad(): - # Config - E = 1024 - NH = 16 - HD = 64 - depth = 32 - MLP_DIM = 4736 - WS = 24 - global_blocks = {7, 15, 23, 31} - - # 1. Patch embedding: Conv2d(3, 1024, 14, 14, bias=False) - patch_w = vit_sd["patch_embed.proj.weight"] # [1024, 3, 14, 14] - x = F.conv2d(img_preprocessed, patch_w, stride=14) # [1, 1024, 72, 72] - x = x.permute(0, 2, 3, 1) # [1, 72, 72, 1024] - save_tensor(os.path.join(args.outdir, "patch_embed"), x) - - # 2. Positional embedding (tiled) - pos_embed = vit_sd["pos_embed"] # [1, 577, 1024] - pos = get_abs_pos(pos_embed, has_cls_token=True, hw=(72, 72), tiling=True) - save_tensor(os.path.join(args.outdir, "pos_embed_tiled"), pos) - x = x + pos - save_tensor(os.path.join(args.outdir, "after_pos_embed"), x) - - # 3. ln_pre - ln_pre_w = vit_sd["ln_pre.weight"] - ln_pre_b = vit_sd["ln_pre.bias"] - x = F.layer_norm(x, [E], ln_pre_w, ln_pre_b, eps=1e-5) - save_tensor(os.path.join(args.outdir, "after_ln_pre"), x) - - # 4. Transformer blocks - for blk_idx in range(depth): - prefix = f"blocks.{blk_idx}." - is_global = blk_idx in global_blocks - ws = 0 if is_global else WS - - # Pre-norm - n1_w = vit_sd[prefix + "norm1.weight"] - n1_b = vit_sd[prefix + "norm1.bias"] - shortcut = x - xn = F.layer_norm(x, [E], n1_w, n1_b, eps=1e-5) - - # Window partition - if ws > 0: - H, W = xn.shape[1], xn.shape[2] - xn, pad_hw = window_partition(xn, ws) - - # Attention - B_cur, Hc, Wc, _ = xn.shape - L = Hc * Wc - qkv_w = vit_sd[prefix + "attn.qkv.weight"] # [3*E, E] - qkv_b = vit_sd[prefix + "attn.qkv.bias"] # [3*E] - qkv = F.linear(xn, qkv_w, qkv_b).reshape(B_cur, L, 3, NH, HD) - q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0) # each [B, NH, L, HD] - - # RoPE - freqs_key = prefix + "attn.freqs_cis" - if freqs_key in vit_sd: - freqs = vit_sd[freqs_key] - q, k = apply_rotary_enc(q, k, freqs) - - # Scaled dot-product attention - attn_out = F.scaled_dot_product_attention(q, k, v) - # [B, NH, L, HD] → [B, H, W, E] - attn_out = attn_out.view(B_cur, NH, Hc, Wc, HD).permute(0, 2, 3, 1, 4).reshape(B_cur, Hc, Wc, E) - - # Output projection - proj_w = vit_sd[prefix + "attn.proj.weight"] - proj_b = vit_sd[prefix + "attn.proj.bias"] - attn_out = F.linear(attn_out, proj_w, proj_b) - - # Window unpartition - if ws > 0: - attn_out = window_unpartition(attn_out, ws, pad_hw, (H, W)) - - # Residual - x = shortcut + attn_out - - # FFN - n2_w = vit_sd[prefix + "norm2.weight"] - n2_b = vit_sd[prefix + "norm2.bias"] - shortcut = x - xn = F.layer_norm(x, [E], n2_w, n2_b, eps=1e-5) - - fc1_w = vit_sd[prefix + "mlp.fc1.weight"] - fc1_b = vit_sd[prefix + "mlp.fc1.bias"] - fc2_w = vit_sd[prefix + "mlp.fc2.weight"] - fc2_b = vit_sd[prefix + "mlp.fc2.bias"] - h = F.linear(xn, fc1_w, fc1_b) - h = F.gelu(h) - h = F.linear(h, fc2_w, fc2_b) - - x = shortcut + h - - if blk_idx == 0: - save_tensor(os.path.join(args.outdir, "block_0_out"), x) - if blk_idx == 7: - save_tensor(os.path.join(args.outdir, "block_7_out"), x) - - # Final output (x is [1, 72, 72, 1024]) - save_tensor(os.path.join(args.outdir, "vit_final"), x) - - # Permute to [B, C, H, W] - feats = x.permute(0, 3, 1, 2) # [1, 1024, 72, 72] - save_tensor(os.path.join(args.outdir, "vit_output_bchw"), feats) - - # ── Neck (SimpleFPN) ────────────────────────────────────────────── - print("\nDumping neck outputs...") - neck_prefix = "detector.backbone.vision_backbone.convs." - neck_sd = {k[len(neck_prefix):]: v for k, v in ckpt.items() if k.startswith(neck_prefix)} - - with torch.no_grad(): - # Scale 0 (4x): ConvTranspose(1024→512, k=2, s=2) → GELU → ConvTranspose(512→256, k=2, s=2) → Conv1x1 → Conv3x3 - s0 = F.conv_transpose2d(feats, neck_sd["0.dconv_2x2_0.weight"], neck_sd["0.dconv_2x2_0.bias"], stride=2) - s0 = F.gelu(s0) - s0 = F.conv_transpose2d(s0, neck_sd["0.dconv_2x2_1.weight"], neck_sd["0.dconv_2x2_1.bias"], stride=2) - s0 = F.conv2d(s0, neck_sd["0.conv_1x1.weight"], neck_sd["0.conv_1x1.bias"]) - s0 = F.conv2d(s0, neck_sd["0.conv_3x3.weight"], neck_sd["0.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_0"), s0) - - # Scale 1 (2x) - s1 = F.conv_transpose2d(feats, neck_sd["1.dconv_2x2.weight"], neck_sd["1.dconv_2x2.bias"], stride=2) - s1 = F.conv2d(s1, neck_sd["1.conv_1x1.weight"], neck_sd["1.conv_1x1.bias"]) - s1 = F.conv2d(s1, neck_sd["1.conv_3x3.weight"], neck_sd["1.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_1"), s1) - - # Scale 2 (1x) - s2 = F.conv2d(feats, neck_sd["2.conv_1x1.weight"], neck_sd["2.conv_1x1.bias"]) - s2 = F.conv2d(s2, neck_sd["2.conv_3x3.weight"], neck_sd["2.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_2"), s2) - - # Scale 3 (0.5x) - s3 = F.max_pool2d(feats, kernel_size=2, stride=2) - s3 = F.conv2d(s3, neck_sd["3.conv_1x1.weight"], neck_sd["3.conv_1x1.bias"]) - s3 = F.conv2d(s3, neck_sd["3.conv_3x3.weight"], neck_sd["3.conv_3x3.bias"], padding=1) - save_tensor(os.path.join(args.outdir, "neck_det_3"), s3) - - print("\nDone!") - - -if __name__ == "__main__": - main() diff --git a/tests/llama.jpg b/tests/llama.jpg deleted file mode 100644 index 9d5116e807055481d0eb50bb053b3f6da1fea0ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 79236 zcmb5VWl$X7(*`=QxXThCxCMtmAR)osf-LUAg1fsV*ajAd1b1KDoe(^@YX}--Avi(r z{@#Dpt^4&()lSXM(ds@=_nD{rJS{$L07Oc1igEx73h=+{X$6o5&_QTuXdrat4ILdF z0}~Gm6L}Hh;$q_w6Oxb+6A}}XKBFcl1wSVvCZ=GZcuqq@M@L6W&dAJ2%S=s6NBdtV zD9EOm7?=cDSOm0SVleIh@9U`tAjU%JM=b@RFaoH=C?H~#ryl?f0HC0O{@>{$Cyfe1 zP7(PM3maLiLGvxEGO1JMr)2;SSre5QL=1=n zEg^8&T=~zrd{WPpZ?=6Jj1BF%L1X2NPg?j!R*0s&@ofFUj)RjT@NN zQ7zTuZKVio6iNWHauFTJ;*gvo8>l*zS|b-kmd`d+*-uHz=mR&812bzt(e1%9>Io7k z2A2hJVhO-Tg%XSrf)Vj$y}<@#lFU5ir=AWh6xqx_um!S#^eZbXbMdb8;OTRKb&*~3 zzCkWN?OZv>b6}*$UOO!0jlQ0+jyeEQx2hMw`c#=W<}85YRHIXGn2QgyDV4Wdyw^es zk|>wvPiQVGMzdu4VlFCK8}K$%2iX9MsRDRNhRDDWOS1ui4YP-_HOpo`$*8H5bc zB&vkY=~>b11jE6R5YnEjO8O~_^zUpa{A>zESb6l5s4_ZHMOc;n$QJYL#*O(6rD#!b z;H5{Lk ztQ>g$Q%6S~LkFUxjwxy{SUv*K+gmiEJNm1mKoE(N z-C|xo7^RH`PB1IsE`cUIg<^mjg9H2ko9dv*p#!$E6l4tsFWCUpb2)SeZVK=>;Pn*8 zRH$t8o}2^`r@u`Pd}~JmaD*d@_>f8gxE;9Jo*^ZSvAn+gvOy9l*bbfV(|JA!!xaha z@O;$dmH!?P`MBxo$f^JmT^@@DS`QO52`R^g-_W4)?ki9(C zP_^9iN&*X!$Ei#PC)*4mUKzBTGB9++HUMOxG!3l=Q%ZmY+}@E#qd`j@^R2;@fe{?r zzy&J>PF4g>pE7tK0$V0&=vo(6Pbr|M*Ya0SVNBzc(n&^hm2OI1IfIksqA84F8|cB2 z(vCJ^!5FzBm$$R1k)Eb;Q&NzVN>DxxDvp#D5UtKxPcesJF?9?O6dP|zE7f>ahmxcl+oegxoBZijzv=E zhAU@9J=l)BO(ZF}hAT>_X7*;b#*H-%lO(u?xJn3QqjW%l?Fx>dovj*KpE^;DCt1;N zN?9>nWS5Y4v|856tz7xvJEYw5R+Qt3ISu|(ieu3b7NU|<*%9to2+D+8O&O!x;7o6z zdCfyll@i8mpb{ki)junOP9{iTrR1?IZz6FZ1Gj;5BEg0oO47h$>xhI6ChC9fc5D=K zAs7ZEe98$Dqss9_j3Lnf6snGVfb+VY*(Y~s0CNoBY z0ScrQn~w;=1T8nn62tw}0iDuVLOAddc>yj2bG7TAz4O6Xv7zZ4&pFUvLQo87N=uQ( z*+D1LP5_M32+oI+A%DO!4n)>gmMuTzbyzaQK*ClQj1r#I@${aX2#F}94GR4qi~t6Evmh-q zo3Kg1yy;moE)qMWGMoQtuZbKsJ4uQw^DNHMfm`S$#&Sb+Wn}qCkW3*BjjTNw16VOwK^EEOzQ-nm^G~nX%i7>k?{FI0 znzJcQZ=)$q5*wqblNF&*>(nYGNJDL~k$TVtBOQc{`eq7aV_XWrj1+JyYRG@uK|+%_1a=!-^v_A*AZ6r2TD~YWR4n>4tRrNL{HO#8+5dF&4OyLw-L2sp zQe9@qSKt5Z38~{e3Z$7Mz4*@phKB<$>`jv2v)O>O|AR`8u#Qfo-x?~Dd6ZWkSh^AQ zv)Cn>O%MkX2}By@Y+4$n4gaT?{{Qsy->rvjQXQkG2-s}MPnMLR2#dk_LB}Bop5&7U z=i~IS&8lM{4VWVs!j_ANE0^E&1?d>fs|8GTdh$_21kj1dYLJQofZZrw0e^c_%@87_ zwF-g92MW(fOoFA;cV>&vDh)mwZ!s|mF_Z3}U;Ufa;HeG!rQXt*{O|iZ16%5=ha+nW zX1|wv;!3veYWZy!Yp&ziH7C%;kK_CV!ZZHAzQ+}{K z#IMP%gcluoEquKUGmHHZO!9i>Vxp7nwt-yl&Vsrw06Dket2@hDa<`GvOacNo#U2pN zuxMnJEW5Q^8p}`9aGjNj;B5IKbno{CBk%YO`>nK0N%-w;r14;~?9cFk`JU^1q=k4N zfzofqhV+CAY}uHNR?C`n9OM)lE()OO5D;009YKK9jH72p7SEYP;!6BV^th(b-ri&O z=gktu2yJH(ZHg5@dd>91fO7wjTZRL^{g6O~(73Hs;x$&rI@yNuzeb^x@IRdRqb zI02P&)FAJMjZq8NJrh=21I<1GT$eHm1gXjP??() ziU9_=iKF4P`a^5&v{hZrH^`ihBGx#fXPcMy-YQXrC4t0oBWIA=EoPv1z%^kgzu!0M zD#%#b_@GlxHFwV1KML9!UThxq>TjO(S?|D0(R!0i$-jhOyl*=Hd>I)^gi(AmAZe%t ziRwlHG$&?kT4f7YfbRNijEIiF@Y8CXbBfgN z0ooG~?Qy92POxo^Uu-6xf=BoM^r9@8d$-lOClYk8a(L@?ySJPm0-ZlMaYf_ndpd0dRCYed(bFpC+Mdg}FU_VD?*I*N6 zL*Z2-VV7cK0~9LZ=_5!V51Zv&j`GTtae%e;_o+fyfJlm_s4x=5TK6_iLerBX}>#Y-J7BSA(QJ1zEM6#C-;XDNB=1IFQiy)NmZ%{At9}3Z_&NlgXK{epT|oC7aJtf3Uzl zM0Xw1_t4*3+|HFviAv(HC)X>oe_+Madt_e5etStd=G)|76rd(Rl1L!ZVckkObom7I zD_kIScISuTj0DeQ7p0y6tl@z`Owlfn&f%{QhQHH2$*NV%^?%)`EglHl$D^c3WS+>3 zQ!a?(+(&jQUW`T^7o3|ro@RqW`z%^%xBls7-C7vQBrtbiF@~g~FV-tRQa^$bIXVd4 zs@RH$Vk%R|^`)g0k{49oG1e}0-!X!cm1zk@4+EKQUlfr2yJ2Q*OgK?oSE6r7b^a|S zzc5{HQgUiXki0;8XY*_(6p+h_^XoQskS@yrkNMs)6|Nj%%1|v3%BvthS5$lud0=Nv}Z?WMFmK9<~&sd z!;ACC%iH3@Pw#nJOoJ@_o}d4Ud)7Ai{wD1z1G~rYPhaeP1%FPk^vdl_Q1qNZGC6mZ zZ9R7+#87re#Q|~G{--`Qw&z&;w9oosJRshI=!W3+eULr2P>SEQRqJvs#4*WC&uL^))>SpIn zjPnPpJ*I9EZOEqO^7llU*Mp526njE%h3K+5vb#V0>GwLb^Tfgt(yaz`Gj zb}K#*Ka&fSa}@8yAK!^iMGvBp>o?UxE=wK*T3Ql$U%arT#>@I)>be5B#(gW5{o%SU z^B#@CxkY%VGp?rJbCZfjFb^gdf8`5KRJ@vK4$$&Ev^KgsZ)y5tpJhkEMfHKGx+H;^ z@!KwciZg>(o=IP~Q{Q?e4VA91qwddUPBD}K#SYX!?CXDS_7hhIb88L$=>pJI4#fJ) zEwsnrBGq?Pm4sP!zjMXx@#G3@Vu*WW|J^42n_rJCd#n6Rp=aUZYpGS~U!w)mK4*KD zv7f_ud!I(cL;d~X}K@qs)p zeWstUc1MEHL#pG87Gq@{F&SIIQ8K+*pvU#{{w%2R4}c-8ZGHE4pJdSkKeL%H=bhQ^ z&`C`F8*wa}WNTD7o_1*^vvwLIvwF>NkwPwGrhoz*1zZc}VjBuWw|lja{{2rI&DtR! zJDv&?y2xMb2Le8gR5GQR#U)5rWUVp8rRf>Ll2km8RYQ^w$^-um85EV`pCzq8Yp0-#W}iaggq&d@9vC(VH#}bF!T> zb#wY{au}r>7DB1<s| z|L^?oOIJ_B0ZBtpsH38(rxa1zsM??~^?xL;%*DtR$|uWPZek-wLw*iPHIzfqv1Pf! zLS;D`03~)=vO;h^Y!*;zFg4Xlsx1OomqXqb&2pI%Shp&Cw%{4}3a8y$Znxk#pwxRw zPb4`+e?@-_4ZW$hb$$_bO|m*_HPk{f(5o6qD0=$-5oAyL1X%SK8?9xem@6qz^J6k6 zHN>?QK2p25uDF&OM~mV}#eeWpv&?vE*Y4j5yOV5++WdqRwG z3WOe}fP@BC?90t^x@Bb<*b`7#*!6Yw8#nZ8{40J|k)|H6#2{ax%Ze}H{<&M<&KZ_y z^VrV`bz=8;F71;+%-Wc7`iTGl=#WD-BTt*ZVd zVpHcNLIkBkSF3)s&)$=7X4y_RqqJyoz3%k=H#;VdX;BhmnwT`WxY=$Wg)ygN>0p6F zK-eVaIkvkRnE%S#984H($$PCpNe?}$QGE7+xFW$A|A&{MYH;czu^R6_qboA|D z9^wgzGmz`pZp(gXA>jNxYfAd=0jdzSZg~eIEETc7UC@-OmvJoqS|}6q(m6W9hjmKv z?Ph$lLKUO#^6ALWH?nMf1mgH!D@#YVFRaa}xQG{Og%D@|Aj{fGkn?IHU6m@-ExV;o%*~Z>sOr4OFQ0u~XNj6h)Kx*W%dS&%H z_fE_7CHH5_8(3hl*v0uB_r;&2@?Tor-;|egcwQz@4#8rZ1}RQ$K0`#H{OMPi`_Q2`*ZspgqGuZsj(6A*?qB%oJ$QHYxL zkkYL{PIdgL`Q^XnzG@jPqMJNoM`U+uA`gB7aPfOUaTYl_qtIJo_vebZ zT;EpWq@m|&QJv&POP(nMCC`Uod;YyG{M(tS6nhD43b&Tpq8}5=3ETwy0yWzD(WC`Z zx<}M(?M4y@@M8royLC3L%mk4Vn(Y13YF(d)q!jXLQit8`JzU4r(>$HYBh44klOwHx_qDi zAPSy!XRxI5iZ6jw^eYBhHa5+~McM1vt_gX)6UKDCd(JS|Q{GGG59ehYIV`rnNmh36WLCVnBvhbBoO)4J zy5)l8kEo}MKSOQuE!ByY?k4>v1r#GO$@aRJXNR(0{?4rHs6{Yaq9h$Y1an-xrPda{ zQ?5$~xlSxiT_OR2g_Fy# z-($i?hwiq^bq8`9RX>Le_#&1Er`owbwTRY}T00^s52n+PzqZWR57&{_NJ>ZFf@pcJVN?6?|c+7>D%_Y&SqOYX#oeNh@%Xqh zf}5Y`fQBb|!6e!Z*Ac^b@4x`Fckg0i^F$d~+dCn2Jxti2$Ro498MRxWVR3z)(-l(` zcvGIy^C0%SBtowWj{Mk(RKGKMeYx7Q9VfR3BdjSZ`v<`W#17OViMJrM)06d_IxIBePlsA7A3ukF>7Ed9=g}0)|6CfHY07CNQrL%&TwY zZtT(Q8~?F{3^T=zr>84oYpZ(Gl-$^t6)Fe9j-1qVJmryR~N|mOsJd5&PyzA;clFjM785<@OcJcqv=Tj0{bo4uAqL%`vq=oL) z>THob8SUQ(iTyvQ;fOE^Bn=T^FI<{8! zi~&v6PJ0$ayF?TPst|t)#ifYfG@}92nm~P6^{KAvlm*Y#HI>t#Pl7b)*pm75AxK+q z;`JaFgYO4wz8m#`^FxgZVg$E`oHjrI+qKhoiIQS{Iis052Gs~BnX5zLahIQ}r{N&V z9@-x>H`dMf;>rd>_s7a>Pk;~&tJ(9^FqaIq{%#b>>tb%mG0zKrB$>k7AtoM0IC}gE z>G}olz2^Vc3)G3dimCljJ#k<5+Fx1l4ia-9k`k4Qibn4mw>jvL%`XgL=fVHOVnVfU zvM&3hwu4#fnEW>totvZBw~62c@c0&_vFly(=-0mAeWaguT;F*ae6Av z6!73M?91C6C9|2Tjqc89XD3#i>5wCA$8yoY;fqphfxq>w%*?5pFW%uFD*6hWD6*!a z@TGnTR}k$s7@Ylh&J0~PW4bpNX+hgsy&7}1dlQ4-pVHW>uSK@~_e+fXIKThi9^}Ts zL~~m)n5=V%0m`>2IH0O_N@wR$Jii(fm@)kDnmMwx(t%0Ymf3^O!w(@^pZ(Z60T&c2 zb1pF+v2M*21F;3PCW#FK#OKAL?j?H>r0(c{T8yY8L7_Q!R~ILWOJdb%c0wQHm>i(g zpaU_6!6Io+maM59;qAiBRLoHtlP5s0{o}Jo1H!0(RT!&&zdOVIlzW6DjbF0FW+9e` zmeP-wu?noHsfPx)25)pac5L0Oo1;$}4y~{x&Y60KC#q*eL#kD7$YILx zfap8$&QN-sd4Vn30;rzInQJhaT=2}b*Isa~qa2mZ8dOAmz%6$dmX-8V1sQ+QE8I(9 zh%WTPDd$%#9I{~gplQ;WoyC=4=cYC~-|;ZF+}UvL=)!HG(yDG2_2LMo zoOu>hTq4Hmsim9I@W*k@Tr;)hTVN;H>qFUY*_!L~U0J=73d_6c?Cu1(6zc>s=}Yn5b3`g4ya12MAG7}&r(N1S1n+o z@)^k=ie$2DfA@SA+;T*1r4@w1<|hMz7)b z0ir)x!IQ@_Gw??#%5q@+o~W-`=rM&$z-=dGuM)c9*FJystAp@wl~-1eE6nSBWv0`M z7B2H4yb#CjH$(vhYGwN4KNGBe@E}+{`~-+Mec!It<4qHmywHzy5d~TN(_-Ay@)s_t z>!?qE5wkIrbt5|H(szlbz1+|_8di9zoSBL~Q19%6ov^e7BjW496zh5dRN3$P9&9fH zZIrDUzZRP|6Dbq<6w>aAMqO}*i5}z|%LtdGxP!}eVK{vEFge@v66Z`MGS*&n_hkl6kOW z+4NsF@Q8ir-@pg`m=dv0;@OapJQp9OyeJDI@V6?Bqj^`vbBR?6rwN*{q5O7Dh8O61 zOh72svK8|&b(k%2fqJOuQvW?()17%Nnol&56Onv-m+v-n z@7nD|L#&{Kq=OG=NeARKQDj_SQ&NgsKZ}lSoELM)r=9|s{TD_+_g1muakhT-F)prc zUg&)dIkUnNb-;JR`O-oYA%Q>7T!RB`7{8gT)h|C2>4@p2ym@35%Z^4U=@}GARIzvw zrRlA?1yS;a1R3!VPDdraI~i`RZio{W^9TM@y^tdi5irPPjE$@;g$_@gFkSE}uY0m9_PzY@Pid(aQxosR?RG-1@ z;Mo3&`d!WzWO&eXuWf-;XyRBXA7jZ1zl8GX?& zjGSm}iHoiJ%Eatn!8-%oO~rvMPg`G1H>$l-P8`@TY|`7a`Q{##pVSPTVhi=st<{Qz zPp}&vsQ+o}tSTw99rkKEDvYMsl$}@;E0v!RV!~ufH$iMa|LNB;$sVa0IXsI<`+5x% z>8B0Zs-1X?Io;})J|=L8@zVTS*&_A{fP!Esn3FL6pwFl23L<>)kEWNrs&?;GtMgY5 z-m#2z_QzEG6=5bKEYI{ZTl_TU^|G4#hHTch(2if4Y&%t_t_EL3vDK7$!*^s+HzpfP zh_jEAoiImr$!M74>#@0>ljcc&qSkrrh?>wpTFH%D(%ftvC4Lx%r8M|-U6Fl)oS<%jxkTk@$IqQ5BF5*AOB-kXr-yuuXkcu z{H&D%*FuPG30xEg&;HE`PECsCE%+HXxK&`bp(Xw!uJR`lqLb{oolDv} zZa2TSwRbv`bO_J3H9qJ18Q8lw6S|!6@U@M*EwUnauCU48o)gmQUCYbqJZtJ#A0B5kD;qPeWA$Q10Yd}9yh`qpiiQKV1EnDxr9BiygLc!iHM(=P^NQoSN&-jFYxRnRleGsisR-->twx`f7j z?H)W5XIBRovW&N7R=JJ>S2#Qd?|`!;ONd)_m00L~VoT zY1wS|13ht+mQ^##sc?nqdx(kjYK}(;DA%HypC=tW7Y{R*cKU1k$h5DzZN}I>m)9wI z_G8cI379?z@+8%?r+l?@ir{g4w=R0tUAq%pW3;xeO#MoXiBjI@uaHphgxQ#|ZMs!a zVM|vgd*Yi3*FFNQiEi;dU9ELW30})-hrJmVa)oKVRcWF0=`EX4^x)_%myWMFE+2<= zh3F-YJ2(yt=9HWTZVXkUuPfloZA#?vg`k$ZKn^m)FTC2PJfYP(uYN5d?)bXa4Og%m zDR=Z)48vl4VtuEJy{;u!(!Lhr6xs0;<0Y}j6qgI@$Opq&1L2iX zjC4&z5^mXU^4EMf37>d+UM7YcL#H`R&X#!B508`2j7fig5AK`yK0OpQ3Z>Ojd5aD& z&m>}AM@BQV!PHl*wDp^Miu~O21Ml7~I2b0~@}9ly;N4Sn{21)aRQmlpGPYEk)}Bz(A66@q*sANX52pj?_T^p1}7Dn})0nlcVO zzTaLknfG0u3s7t_HD+sYoCnc%u87pMtFJ)P$oNXvX3BTJjcwDdeeING6E)_9?_EZ3 zZ!*02F>4-q)*F3qnwbr~?tRhziUxk7JA1I72>jsg&!~d4Qrav+aE;=CA{y8#h7hA8gM~j0Nq5 z%UDhc(g9jd##$AZ($!C#X<9l=nC85Fi?wg7g)hgT&EbB@P48)UxmBl)6{XMO!6{5s zgbZk~m%eB`N-b#7*~K;U3h&gfrSFBkP7`+W z;PzITwTwC@#{NODIDzs>q5EF`dC_hEtqM#%-MxB!?fq1 z;zAB#m@)DHtogzs*MdmzkqhS`v$3maAHjHatx9^}_XI%lZEwkne#Ftku)eL^mr$|} zOUIK1opV~$R%q;B44;54c}mKZX%MNw8Xm?21?D2KjGD%58L3F$80xY;{FTk#ZEHMC z7dzgmM?6DM5qRaVgXdORh0hpir2fV>;}Bqnk!bo^60szQJA71!y&tWB2S`j+I$q*t zj5!;uGo5dC&Nwn?jAfQb84Y9iZ26eFp>X)piW~}KJ9tK5*g93RjOvZPP&KDU@4YL+ z{H{A0=b%|JHD`DXrL^k0V_M#+wFsIdU*nunP*;7?ND_*_n$A?Z;Yaj@OesBF^r%w!Q_oxc=x?~%?b=h3upj$IlbJ@72r7qgja9P0{m$K=x~#vgnxTMnbZM&l|aYH*|K$F~>Ue+R^z z&#J{gDDTU9~bUFaDgs68u=Ue45{5K&E?_Aa)lu*yWa_54**ulgb; z46x;QJqpp49S8-BStzwn^?epPruE5(WN+a_vhLc_ycZqRin4I2u+tX#9ru|7G5D79 z7ZcsYzJ~wMwtq!OnV9iUSB~cIE)>L}NBq9msCmlfUIQu?lF*gnqDGF~uY0jIe2MkE z&hd2DB`RvFO{l%bOe^TSM^{`O@qB+=2zS}*4d%eMw<)XB-?;M=FTXc?s=iuz0(P$d zEnu41Yw8PDjx1#{*S9PhhI1ohUNHrcR09_#Q6fLyXGip1MeNMFIfy;Q(hJC&5JCwg(y;CGyA}Nez?ndHHoVA z!9;Z%S$xZ?TBp>x??ySP?(8GZqE`(WDMOeN{YJ~ayNfRq zW<;IQ@2SN4Up-{sQi`!HzotN|HYc<&sru6sT>?*3nqK(?yU>_F} zzF&Z{w%D(6jPE2Aw?y!A*nD;8P4zlGQdkUX6+K*7$5oiQko_dEdpHsuP&VRtZf4d> zxL{A1w<^lvN5>Yivz?JGFZ#z;icL>z8`V-L>N@`d?mr0-OUcsMh9fgGvB7zH*STnm zNN(J3@>U(ys_x5lT^c=qZUICuKUS6v8Jv|gkW@mEK>IHnOO_)n1RI?rEF75@sVN~y zJsrgc)+l8o&VVS`J#$@KZr~|sg4B+@7KZLaWIw^!pR1TbGmOYgI_DNT-}rHPK1)c5 zUgDQimDm#Bq-poM%56MsN_=q4Uia0kOqhLB?<1@RjtG9h?1#^9GCT_puuJ+pwnp0cCj=69TpQT1D)wV)Fzda`)k>}d z#`R=IOWDL02ir`3sRiflj|G=Ad?t$y+1UGOL^o4&zyHl|{5#vU?QsEAyT_tE#$wUW zRdr2!GEl>DL3iqs;@v?1Cf?N>buNt%Iq$rz@DS7+}|IEJF!@0TpDzZx;g1hOYd1Wo%Fq8OUo}ktK3) z#kl`MFa33L0FOT?AtrSERnLHHdZeeCn#Z)>m|)RgN$P#O`uBMjiB^L+s@Uq?q+%~I zgcCV3Jr-KcWVgGyP15-s$4E!JX&a(MTDfLb{vtd+GPz8MB8BWf%duBk)#xaB9Bl)i zY{=GdrM7r0{z3Ip`LMZ7w5xEEr;Mg|8@-7_qPI5o#|G%vTcN6T*g}Ke$xq$u1#4Bu zQCA0J11Z_;2ztHe%oWA<;xps!@n1^Qj zVLS8Jw^v4 zr0&eoO%i_;_-Zaz5a$zcLmZAVZ3{1X$8Sqil5*%+1vKToD77~g7M-RihBRdbVQK~( zlhhD`SzvCqjV8OfMA_aWl7O$(KQ3p-MDp}Qv6aoc$RY$H7Cv7jzymdwaW|IQ?h760 zW9*0uCb)`gQt{V11!ddN=xpxFPl}0rSa1;Oj_gPZd9Tnez z4Qy;3{Wyx%5xTWl{$-m$!2%(F70OGhpZA32tJvn!IZpr%M+M#OC}BdOJ5+yIS9{v? zov+Y1ud|rAhV-4lMYsCm0XuwOeWkC;t|mhTn@3`aM8Rt;n3480A47{SHhK`pe#U#9 zz|PBR{RvPwL|ek0>|dS!^07Syjr7^i+LszAo?_k6N^dsHZ;y93&vun7+M}yjQ<@dIs*G3f5`ABE zD&H9x{(HBRdGe8LDTNmIhE|*xf%TcIQe*YAtH z-pxJQ7ZjF5S;nAEx+Bl|N8T|nb@M>jhsoC}<-7cI9{t5kbkJ;4=)buYGtmbA;pzO% z*+EPCu-z@6HIXt3MF|J$QkNb&Qx3AcKF7S{jQ8%Y<#h<<%N`$8 zu8Gh|{lNWYFnKL$sZ-*5Ykz=qF`GtzrTYE;@q8KnT%|J}lpKR~)w?7(Sgpv#Lq77> z>Qf=EmO}9dG0xBN#muX@#4WcQdvr|DOj(kGXX+@Tr-y1udf(YRtIB9B@9g!Ugk&8$ zhwKr=mAX77yh(Q3&TFvOcT;${WOs-J8ar-)^wcYQ=f>&inPd=(G6UdXGH~7vfJD)S>sc zNOt#DR@Ylpm|y&xeb;MoB7Mb=w|CiV?YeYT%X_WAJmx#m6W`8;TC^yaZ9oG>HxlQM z$3KKxM@V2MPmJT3S4{NBriFXiPsu%G8c ztW2|~2!kwMg90*54$q(*?G(Q$6&3fRg*zG5zCwXDmY|tg-KORlynlY4$#+vjPr%uD z2l}<%C?BF#c%ekx8vgvCuaM{6g-o4YbF;GTo(P4qGTT+q0Y%T2m+K(FJDNVz^9w)8 zc%itp)3PmWMBPi}Z)K~|_NQT^->!#A2$PO9H5Fm$nIn&Oq1=W>dWx+*C;#w?fJFK6 z=JvWKp?!HmH}*}#z5-i~30v?&)QvYwz!`=z9>(Q3&Yhnh^fV``e%a{P2RnrMNr@e~ zD#9*^B*}X7&Db)D^9s9H7|*G-8G z15_eg-3v*I9^tEPp7-STo%%*E8KCijF7ht&XnxShl@aY|?#ydvO-+KdjxXM+2Q;oR zWJhXEqW15876je>_O|=%Z$b9C)#tW;DI0adRxsVzb|n{TO6{^9Ct} ze7{)|$5$J^sEcSTP!AGQ)hNLL(Ho&X0b8eXM2Je+XG&ZbqJ7a#c9wa+N> z1}rRknd0TE?Mgdd?i)LoUnXY@>C?m1X*E9coV~={iFtUX7#`cl95PmRcs0L_pFhey z_JdEd)tA-#%c<57g`Tv#Iv4h$phUJEXHlRE0y3`4r{pXNQ{8xcH*?h4BwfQ_0%{t8dW!A~BgB zp_kq;6Q{Cia*c-!G)YDUrhqVHUI=1wA@iLdAtE~&!~SE74UvdL4~Ue6e~UVFmG}T( zP)wbBO4m~jVn-wCiTrAAj`j2u9Z@|a!a!1aKo1Z^FHG>g)zX&7Gf)l36<7YBNrW@AK1p@LwVnJIEr1|h1RC#TO2yOUfb7-AKt@n3NXA%1c z=Dw&P{b3QdlN+CqsQ2TX8%Xod&VN8mL%P_=z6GR1tHm;RSt zeg87c>751KQQApayRH+>1v8ER2>V11c_<5?RIpk}T!~N#6J=i-d2kRr7wv&f zg)&bFc_Q@x&R5t%(V>AUuNC+26%dQxd>QM4dS z+C}=ml8hafn}eU~ZT-ZTJjx1Y8=?tNKMc1WQ^giG)V3IbgExkj^)`syRgZrfSw8-9 zO^kDx-K_t;welFpY4h%?Id_c;POdjAuoAkgteh6zeUlyhuHHke=D;{e-*ddc8DT89 zHQdyQimuO-iJCVgOZ5xW>2VJvT>HM(LpwFC>>)?Tu&ii^ld2R(B($?gqOiS9VE6AK zxy`Rn;{dK%aj894{0ZkFJ{gzVt!YE)G(=lB?%VKgL3nxfH}MQ4k!a?qrrs|6!8!Hn zK_nrIw7Bhy?qn6JN8r#bcki9{SpSstwOTK=xCcG88}_j~Lf7XLL2lEb*_D;;60R~; z>ej5*QO?F(3(-!3^c-6?DAeHfHCu-K(+W8CY@!#HAwNTLXvH+i_Tn=|< z8)gCv-f1bp^EVWxgp^bnpL(eS8LPF$n9sg(PU)m4lc)skrKxp#Um!weFTdEoXbGEp zk)S>%>i_eWr zSH*oc(@|Ys@ES{cB^v{o(o)tJKm~%*b84BY3r(KLa0199buYaNP#c6HtQHbf#l|{| z%I*vwhdAWVN@_SGPQ`ao23^%xZWK#1KG(&)xha~QHVjXkfAu%saBm#*to+yeeO!dH zQS(fmRFaWu)4@++lEt+P%84D5nnOdO>{W7Ej<+s>u8^*TeMO!sv+j;Yq5=gxyGj)7 z>?7fr`_|o3v3-$Oevfd^IhHq>_c6F_XmoOEo* z%^HhavieYmH7oMXN~+`SpkKrdd_yd+y(mQX0xVoi8?LNO&dIJ6*00=?%z5z|x1m_sXahjM|cqy**=8|;-J?zN@ zW>2#O9#c6OAp3e(6q;6`mcecw$_tV(AwpcQ)Q&0{%M8`f*z4U8-!3Tc84-(+*dz>N zCawPf1|s`MS!sf1SmSx43{?RG80R0C^EF)0r^y}UX)*$G$%;!od@?Nj;s)c*j)iyLbhJnMHG9OQ4#NFCTH{6C=i z)n#=Kqj==2Hr2~-V~mexK7-^xI^%V<$z3kv(7JTs>5p$13K^tnc3fOql^Nh-?$frfi>9>N`7lyr*@JS>z-FvM53nFyL}Z`tyTK4`A|r?KFwzIX?E#$G3Lwhy5LZ(;3vm7@Aoa12qess+eioQ91M|BBH(4TLx*WqT79+Bd; z_MfNTp|H0O=X!u+BRi8PmK?XxeJWGdUN&_0kw5l*i5Kr4cg&514WGmB-2Rol4?w)u z=C``kEaMPeNi;WM8_WIJ-y60LNNy10IRp;kyw&0-LUmtGbsXYHj{g8lc?yvfcq+b< zdXhopoc8pt$vh)~n#(O4FBWuNA6C57S5H}Fx|Fu^Mq&rJTzARIuAa2lJUa0QrDk=R zG+lNf=EUVb8C`kW#Cn`#ccDJD<_nA4xh>7Tq8aYqUn1Hx_?es3s`fUns{!mi%Ir|S7zDp;P*WC(A!$DXWGOF(qImYmR-_TV(O`Wkcv^Dq4XEq1> z#oH!%B#dL^1z6q>>D?!&Ya+|48hcxLRI1)vV0kP@1;_B`)B}&FW@&WT88=Kf4gp~q zDxV|jDf^_fNJacn$8!>IakRUB?EOgPdw1vZs&Tt5WYV`z;}?V1o)q*w*Geez^;>J( zW?zCx;waL_d0p5gN&f&4?kdCs6%6o>-s;Oi7B^B!zAR!kZNXHG@P5AB9E0mqU&C9q z;YDEPEp5%@0y&}eqaugWq(oqQ(2<%idLK$w16$b(i*Dd_nULv8^LNcXupOK~r zxxUsOw*OHS^Zsy;YI$f z@~ZDr@vl_ZbvT|)iWSDqzYgC{{{ZaqUH<^IuZ4YAuWMGChLx&Bw&vT*ed=09Wn#>7 zJ<9|C0AW-oP-xnZN=FPA*D>5QZb#YxARap$_9OYzqq4x+Zt`5xe$aOsP27-PSlMb; zZGlDLi)O+C>9)4pF}^L-eTr zy?Hg;ECNC!gy(1{miPJmsxKsrvit;E-&?>ev3dC7Y9B1LLN_73Q zzM=NupnahsyB}Gt5I^eCgI-kqom|uC zN>{7=EpDgRjm!)YNdrCpSgze?Q|XS1=`rE%mw1;`nLg8~O&pj8QUDQ*a6rHu_NmuN z+(_EIa+}7AJ79j%0gG^zaok{yr`J68uRHju;ZCX3-D!QU*y?j!MRhw)Bb9g|jz9-K znd2gXG|SyQ|3BK8tBJ#-$+(@Mkcf;hDdO+nSa1Z$))SOsx8j zy?R~=6=IGSec_21;^@OP_|`+40VHx%hvn2kI-*oi}DII)1{*K>{0eUPe7V zs!c3@Y>d=BusJVkHK0z$jcV&Egg+G+6eI4@j8XxEy%4PdL_1QjHU$vvS{>+xBLiZ& zZt>=irFD+g!KyKiSl_rm56Ypl@D7J3HS`hB5Do(YZTeR{m&-+k*eqgz~%Oe#nzfsBU7B#!wU@!J(Ner?d)V`J8RXW~_r z-ivo<1b{4d(n#fdU@6LvPWT^FRUb+D*OD#G=n6^536YPXsIN!KcdT2@b#{WwObh{6 zea$?X&RMWHUibsK8P6DhlU=sAjM?emy_oU5c3-7IW5YCb(8L?}BrRn=1Ki4&|WOqi@iPVp@u!|1IsBK zg;SA}?eg{c44R=~K*8H1$x?TcxZlsI{vVgAu4AfK+iQO3YtgOlp)X-QF3RO07dMwu z-ALIHDlQR@dyi71_>X@;E563&dmH7N>Of>{;Z)_-bNs8HQr4riEffU|?Tzls#7pVN z^k2->dh=0e=Thk-%W;N8AeJEWnGestJ%IXFRBM+e&vI*W_|bZb?$Ykb+VLmT?=KQE z>@s0#Fsz`Dx;KzUat>G$IH;eDw=i334(+r!em8a>?=>FjOmx=svILgCy17_QUX^2pDpIPYG1 zu#)B-a%p2nc^N|p--7FsG8EvEnsVxnsJgM~o6BFd0dsIXq9Y@6ZvG-T9B1Jl%hyjP zG*fQ~hR>*_{OfOKSs?^&U_Ai93O<}vpGklM6|3l(XOOE zSjcYKPg={>w%O5i`qVikHj6Z1h`+qtL&V;r*B+z&_wQ4$4Z3t{7hh>u7Huv`HcXqc z@UiWmVc-7%6#<6n<5hCIqrgzu`-4Ap`XA4=TK))im7h)PEheM_>9;tJ;r3+5&I6I` z4}Z?9Y2z7Kl(;PGWaCqCJ!*Opa<~d@8z$S!1JZ-ikJ5zG(13bSdQygf&;S||IHRo^ z0ByVgPq>WZkw{;CzI1V>O>G^*C?ReTvNy<;Adjfcb>FXMt}iiRlg`Q8j4=6CJ6Ft2 z4bc3}sgv5NCMw%bU|n8pqZ0DWu7eP6Em+KWrLHYFQQ-VPXlzman z`cdSyyD`ZO;x*jA0Nf7@eF4Tdb_W^nUPS6UZAm=&Ww(|jUCdq17vH~Ze_BZH$Fjod zUZ=N$9k&0`xSEoDl9E48)!Ft0 zTV|A~B#at+Fo{S|h%vz9k@T%xzz*0)Z&OjR_v?H}Bw4xxAmQhKdrIb)8 zXeic&J!wT0(V?KCPUf@)5_dEX40ojmY6YbQ0Bm;@W3>nrY1q&K0a>W&>vv%BHb&*f zM{s=&Z9AHCC?d4CgUXaj@w?keW%EeB86H0f;bU|EEURY@G7_g`3?rBC_;JDY6bo36prCD6xzR2Nu9azn?;y4@>8soX~HMEn1w2`AAuFJS%Kk3a6dYstt8ba)Fj&mB4R!w+e(4zN#dVu?wm$!;*h*@ zIBDNJN=IS;07%Klxg`5kk`fCg`{8}G7n-i0pxVwZoZKT_Oq{6-e&{}_D!J{4NG1`!3wI>rBmV#u-!u`)i4=n4x5~Mvz4BLNiGnyBaw|RkONo^q zSom!l8RfI#WCzTIuH<3tSM=x86-o66K}V)eWNj&52OxRPv-HMs`PZ%QCrRUNj-?N` zKdGwkt!+}~85SgDAd+#;TmJwR9&y1mUU@vfsx8IK#~j5`AO&NFF|{Kez<)A(ns@0- zSdwLS#zFP;uI{PQ1-0AkKDp%bDZ0vbjBN+CHRYNWrHzTT?xKFqw(ziAjt{V_d-hs{ zORnDCC_5v!+ngQ=KBm0w70G7Xl5^|sDpS**B5K_SXFj8)Y6$~nOG6+GlY#hlpS*eO z3ZD2N)YQC5WRFu@^qvoOR;sP=yT(qL>i8kFy`DWES}Hf$&mhAdIYvFf$l&3*0~KkG zYWivLVM;ALGb=GzD>Lm%G#vi4l+woxZ6vZRXrnC5zyLmE*A}^~Y1$^8E}^Pj$7^jE z<>YK+`Y88f=m#devEyfq+K*7&wu7PD&HbG(?;wgU9`ObS>52a8Jd%3?Hsl=g87Zjp zSkvj}bHsn2_F8@Ax1F!e-|U303;fxB-Cy71d@A6 zL&X%08X#@EPB`y(o9M(pzAJo0MM{%U-S7(2dd=q-lylqcIi@mqf%LLqi5kIQQb9`nGgkY zDgDypoB{WU_Z|JRMK)^o(CT`3lOs*AFm3Tl!O^gAQHNunUtIIWW!%1e(%ncKY;pNk z>NyHW1oPjMNv*)K(b<%jnVgWPw_pZ+`(~noaB5L|i)xLLaFWU@t!t(<`+Z7gyVNhQ z3{COzxOu7t*{#RqSW9Wda-zo|@A$Kh-#!Wd020$0^<_3QCb!M#2-*W92)t*a6Z)tc9G<1KX)Rj_?7t}fESKHUtz5Tl-lhEV6(`Rg`BK$z^35hJ8j>v}>Lg8ICHv; z3!f~oW5LHBl`fG(DN;Bi6$t7=kjU}71Yy|w)qO(7PjwdmrWBfl8Oe0Hclr5w=6 z*mfY~eSURWO~-P6)Y0Wz9+RF)l(a0XjFQTG{JrXVqG`g$*;NtIRCXsBKBWBW#ntV^ ztjr()`|ZUr&1OM7as6psxdjA<>oUYM4l7qv1Vis#Ga!aM3TC-t&T@G5ret;r7lq97 zc!8our?nDmkSG;-C$V^kRf|qaCOLNCst5 z^QVHp4iD)~x^m`leTHew;P&*YdY6}i#VBRfEmeW;2|e@M{HhPs`eNKG`J8}0A69Rd z>sB@H3XsHL_XeXHy}si6HU>EZe_%Tk{{U}V&uen*UdNd{N$B`UmPp}<9$Vo6a(0n~6fSw240oa( zs6`O%Qjr9-s)|WzWTyaBN+@W@YHI^xh<2?C8;Rn!x0dE+*%U}*cE@D{B>sY$z-a3} zqtp6hL6=j$nkD=YM)Cynr9X86`TP0|nyb7n>Y26OcUu~T#4mjEiHvDGI>s@Lh!_G8 zV~#ydR{kmVEx%s%46@CAr5h*{Y!-&(O)5lJaA1lC9OEYqp2H(<4-Z92XVYC-r0a=x z%@lHNxQ;de$sC^m$UdsXa87tXTF<)d%L$jNqvg+&(RXHOvE{aVlZtA)yJDqew`n=# zrAcRm6d54k`gW%2pb8ELAb-AUHYy`iwCr&lMlq3rjA!2i)AOkQt(mR-tT6x(+fGLx zgnYey>S3s+_EkfceC_lD=kxmdQx2`BGR9UJLZ~NlY|eeYoS&(v@JN|W^;VSIo6Chs zIm>z<{QFgZb7Du?q4R zQhlmQqnD9&cBsWi+@JJn>)PGqT6LN)#CNKkZ9qQ}&mS_!IPN_`^#_LjB3<;fcAY(Q zeS6hT&xvs5-Vhg*haj|scRPq4pnz-5x-Oz1)6&;QogzjAKbQisJRB3A{fPEBBNb+N zIn%x&b#(1FPM_?ZAigHlq>su%Lu7@Xw>*==}}s zMbqD1Tuf%UxpgXJbLkQLMPn#x5!@ha!dwO=HI!8}88kAZ*P_xM(@}iL!_mMdOF$ccWjz_SmXGdLqkE_1R ztD%Qbv$)N|3}Q&Z_>_a+X~Tj!!S<~A;#m5grPDe*dS`s|+|tTeSxevb*zMQ!X0)-m zYk0tl(#K#U@dGTXqyGSjst2vQl&mmmY!-N}HyCZf_*zCeX(Yx8&TtpL0L}*o)g4bI zwZ~0$6!wycW3e0$;lS($F~@3n zi%_|=)nL7TdG+-(j|!NfYTyrX2ZyyVdEGsP6z4w(+$+7>h|XR zvlzom_IHev@h{@X1CR*z!LDiUWV+BTe8}I;x1CmEy|J+DVUkV>&VH3zT{=)vZE|f@ z)?s6ki{mTwI&H6^hG?0H2rwN$AbsA#oUiX~+zkCDOPynUD5??lk7g8r{z*tq|Am(sJ`9M^Qr1E#E&SqRUM&Fq?x(r<$!qn zsyD1&wD&HCUE5;`yBkS38PDbi=qP8pmg4cQE*ubpaVj>b#|oeh!~H3)V&$3e#FLV7 zn$)jL2tABDb(Y!72T`~av}mV@*BJu4bZeO{bvtW~#Q-~B1cQP;RN5DOoDYN?W8S+W zRw@Syp!N%#{{SpgW}=geWR=3%$x1QFVd$0?w|94NTirE64R6aHY7cpu^( zt>K?jt@fO;+-dhjMR#o>V3I}-NCb9}a#e^vqp>yb{+rfx?x4`D^{pc5F05}N62UTd zWj`-MIRqX-1Xs;SVn)*ZhRJ6|U954riOJ{lC-Sdfe#G}aUx&B4Q|l96yi<#nwvr8> zW=RVu!)M|QQ$20q) zy{4SYF2nx->Peq2Kg{RkYRWO2OOnR*sOMJo$4Ydu39svRmU5Amw=cS8`bN2L%z!J* zJ$2)!T=f)^L#N!wpy|z!BC=!Vy8PuhVt?rhKDAw0toK)KAbI3>KN|z_NZIF+$>-nt zRfuWlC+tFtrEUu!L}MPEWrhKla6X_rtrFF*QLzLFeAt)J9^Xod&#qbTWVu(3y*H2d{>5GTzMCD& zkW{fhJX8m%Iz}spV6hF&^7a1!O2U+yHzJcoe|M;9?1%S&oMxjMrQ`~y%fUF~=Tm)C z!|jS%7iiQjPDg5o>m4C=17)4Z9AxJoT1701;)sT$d>oVq*VdcVI~!uJw(m%qgq~@fC<5?o#J`a(}lLR zQm{gwaWnUuAH>{b`rvz3`Y)-qj*#kW9-+}LjlK1hzGO}4eHmDKs|Y01rwN-ifG901Ht2(GqBV=}?Fo z%Gs=?6yi}zAfZvsD1w0yY1)-hK$N0)G|*u!C_B(4D4w(m5{fkxDcsVD*w6u}qftV% zYA6F^y(l|Sj?_e|sb*lR21zFX4|0740;8(O0I@aHWKc7l`cxOC`sYs9dY4nvbkE+} z*eq(dSvx0kNPVN#SK%AIdB+$vFmf`db2fVrc=^>GQfhc*iX^2Rp)1yB8-s;K8?)|d zp6)2bZr*8Zj1p_M`rb(tl#r_*z4ICW08i49Xcjm(O(Bqx$HT@xpYOd?#^s#sKzg1e zP36j`pgVK%9{lMi0Mnj@;GBuEB3QI^cYKV5lp|>~m0!Ye9`A z^KI2*WMD_dyN=lJkT5;-O2-@)9w^%QcdBSQcC2C=PNOqLdnVtC-*yCZ4_&Hq7Rx3pdAI{4(Q31_p3fQSeguUrhq{n(yXaI-eqFNR0L_1Sh z20K$vRBPIIPUzC=8k@L{aH|?7=atU^6@5wl2Xlcz)ZI&^^z<+r`B|=R$s(w3sOvj@U&Um%)lsCowYHKQYimoU2?$^dC_F5#*a}Vua4~>QRn%Oo zgJhLg-z18=#7>F5S*|t3wQC(cq?Tyx6-C5$i0lAW__9zQ_#+#K0C7;c+fviwdxLN< zE#5Pj1ci%6yf!y-3S%WtY!6Dj^;xWM?eE2`@!dv?JP#c52wE9HDyL}}O`zZrgWL_= z0aEI^KBH;VR{FrSyta+4l%(=TNMP%?8;@{EU^wMY4sbD%UGkgxjaN#7$~|-8?@-zG z7ff5-THVEVV~rANBPthuz{7xWJ`xT+Gf#3ZK*=1Q5BgV{{{UpoLi1 z#$4gunRh$m&|t^U{p-?+FqSoVoHn1NNDt7_w zJ=f{lmEesc5v*vCZIG_=e(SrQ=eK`fudP6Jy(%V) zzZm@ezP*U1L7>5RF6fguLx)la>QDO=m?o-t-1Aq8cAwe$5s6D7xshN4J@d)q=nX=8 zdM~m0lABoO3hJfE`|R)w)sUzS1vE+*c`!TZm#%BR>vfxr~AGP*Wgs*x;PTygS$WucUg4Yu=o_ zYfGsiToBob$iWEcav4a-!TB76%|)fLMlH30wo@w_!Sr}>REO<(eY2)C4KFwsI;Wi{ z{Sd_Lf0-2_@Jq!je!tUAz4fx`n(ex7xV2Z@k`Be=KC6;A?lau?s3+}a?RC`sE2XLX z@myqq*gM3e{(y?JWBDEnT@Hypqg-UTdHpvj%@*cuJ4EUYWnXR75XEg94hppFcb~4% z!}G465UWN-%aY_sRxBeRk7x6`#V6sk_DtY7jF20PcK$BT-#5 zQ=3$=4R01KV&IKQJlNp?dh~`t|Ql>XF4|s@pj?wo5OXq+ovb z2mEL~bMY_Y?Yg((Jn1y6ByuE?+(a-(ZNnV>dsL3j*ky8zsr03(uE_JVYv|VLeHrI} z3*d(PPm&i>+fd5@X!nM6IV-ohp4sGPoqQ%9 zMb5owv9yNX`TV3+$?w4V3?JoBeMf2L&u^$(Aww!XvjxHcs@NR=01;Ol;QiB++Z`-x zR?`OfYai(Hb*PmZyCKFJYy6d*+Iu#n(MYi)x`vB(P?l~EeFhFe_NJX-Ev+v_t(vhA zg0e|2Gb8$(c2EZc*SR%tod7|m+iRlft>+Iafgx7{Rpm)vRVj`RJ|$8!lZX}Wn|E`l zop$aeaVXsigekp%+Y*fAV+ap(!ZuDwtawu>&PyB}Jhd>DQI>h%@*!6??2fj?C~@&f z3H)B)m2%95BRM!e-om95>5@lp`{9|s_U2KNNF4D^zu9|-#>5an;CHLenL;_iO07(8 zvM`I~rTXkOxl1S#KP_J6lslH?Q$2<~PxP#c<4~8Cypcz5-QjkY?@;nYb38~eAPuK05=&K^d0!f;;GcM{mM2qm%i-U)ao@A z_eQL!#p7?MxXC-*#)d<-!HPVJlm6@4ZA{Ao`yLTM(xcd56boQ*O#jq5s zdixK0Z!#FP%&5s}7X?2hUvplS z{fhN99c`{yOFrw1s}DOF;guaoAa>(>ex|&wXC}Y^Bp-UUKWDD9v*{la{{Xiycf1Lt z-A1v7Bh8Sg^3&?gfuBs~tHWo6YijX5G}}D``(XHoZ>DtM=(`)hFLeMPP)MAIOJkj= zNbTkk+rH3!#d%iFR%Ac|#~S^}m3E>Ja0eC3)>`JF)V*t|>Ux~!=IZX}&7HyE{3APk zly(Eq0qtEDOT6S~Xut;?X0~*&QAY`?pK~1)G{7*<^S~#Q^TGG`r|mw{cUN zvWrc&mPl>V&fH1mxVJ3ELC)VcLQ689I47L0agkn<@ST!Ah|?lnfUF=8r;g^cb+)LQVnlRvu=eKX2a+PM0wy^;BKTr3H!Di%6 zeL@IjiAKOZjw%z>wwCO;jYAR-aoVQ$0EmX;jO6`4-fA(AYq*mKmj0a8)UwLbcv&5^ z$jRDv6>vbxpYlISpZI6h{X^0kvAEFn#?ySm`{>zsTg%&tBRn7EeT_w};Mo*y+b`|% zu9+gt3d^_^AO;7KpYr)t^-d9(@5?=N(LOiZ&wKVxvWmk_f?}fIRJ3L`s>cIl;fXPG z+Dz>^!P-SzCzo z()~etw)Tet!LA!0W@6a#OCHgZM%?|FgM1p*3Gekwstj`VaRwHWO{1hlG9Af|4i>a8C_EgjU7%{qK?P# z^=oMj&ZiLg_9PL{Cf?a3fKfmL7|0}&2fa=GHJ(-q&s$DfQptPj{X?N>RhxpmA*d2o_D&XL-YE67W`vwTiTB;~SOob$z9qeY3QBRf;n zd*I^-f@9%5oyE1K{oSDul!!#Ku_Om#&JKHj2>Ir&T@vnVt4QuqgQUu=OB`c3J9FPZ zFHfykUKVJ>O6co5d06~I&tY;-)_WuB9 z=}P^m#}Aqqxfo)g^YpBYn>ZUa3wYHXgeP_e?%x-o;PLv@XI8V^7^dP z0AEU$KB)weuvZe1mB1Jwho{#R#jM`LZXeHvMUG5jXuv6tK)`=q57v-3JiYr(X%2Ob zIzWq9pEXR!LU!^ z%AUmIXHQ#|4okt2sI^ygV=cS0gVc4fIi#&;hUNsgx0*SjkYyx|!z&Mw92!|Eg@Ayi zM`}$4C_7LFz@{Fw>We;}vqdiyx|PWZXKK%bD;5irj&_21AaPGOQiv_3oTIpn3vEG` zY!1MCp2O!}d+MELPOj4KwZVHihBk_08W7EAwM#LDPBQUjRh6(LCA~t_7fq#z@;(tw z@4*b$y0=r-JYTdh-93czI5$^BiA-2t5-j84P{i%v6WkIBDr=fM=-qo^sr1XCnz>CH=e!W}~1 z+gXiRSzcM*`GaJKa-quuoa2z=EIZ^3WE#(wTrl~3vrA7(a+6>ER#F(XyIWa$qW4;d zW1x>W1jb|&2Kb&wRu~{06M#n?bInD3T+}r}hQ{XMj4|VEZ-FF{oDfJ+j02K8cjGkm z(mIclSsc7k+=5O9L?;Y+1A~Lxjw^@6>m|3*tzeP};CRy!uO>!}u5wRof)4|nHVGse zp|mfIUgnJGYMD=svyZtR@O`4w^{$`QX4Y-3%&gYmWOjFz!-AOLVHYfx%N;hoNd(+)8wz6i;tu9KLktVuoGJqa!~NDsWB! z+Iaw1qb;u^)wFwUP9wS-m{MftJJHk}AIzGcsV7{cWM3$`@al%55~kIZo_|sP!>jKv zV=ZeB+M~!;QN)nGFji$hb0qLaN7A`JO`O_S_WyrX<)b(rinluddzX?@X*Y8WPoaFMLoO+Nd zYVNXc4qFK>kEYWvQW48u{4c2)o;$;3HlcNH&@;R3Los8D|v3FjW9pIVXla}U_9 zlITXR_O_2c#`R;hRE+VET>eA2rcGjNiR_lnNpcb}i4Z4fX%1UpJT7ydJDzFdt~C_t zE`?as0eya=ayR|h@KP*e*OzR1hFa*O4%*aDH@@d(@|$_fWIpyub!~P+thW}rKrJFk z7Uqz<5=R~Iag6ruieBFCTW+F~E+LFYfbDWoi2x8d;Pc;*Hu~nH{u?B6TWWVxNXd3W zv<^dHHtetjak$||52jD0Jlb46*ICaCN+3f&o3)rR8yDfp^dpdQ-z0PEUT|(-Ocp4& zr!91bm2L0<@{PDO{cR*kMu(^LTYBWLH z)a)Za(yPWzMW%Kg2SXdRaj|h9^Wc_g_S?J)*2`yi+9GfdVh0%?tySGWc=5$7NgO_B zaPpjO$2@0@)y9kVPXn(bZQ#iyWU1|({OXHF7T2=|8BqI?zB&Y_`c+Iz7wTMsOQ?z|fEu8DOd zP{$l>RyWwkpTws>=6@qsM}eCBcHLw4M{F$97GF$~eX3nGow7)GQPM3&hEwIGmHH7+ z+FHh2Zxq&BDP)4~Hxk1%XKN!IatQ6(ob-OAyuRtniyclSj^;332wW>|;IjK?)Os9x zV+wxKpf`&l2k^#DeR!vv-8vum2T^k#ZefWY79V@h>;`>~SP|-^)1|zmzUFm0ctcGK z{K+i+c3-&z@E`2g9YxeqOE$$fJGXu;U=BWi>sE%N($ZYTFWBaQ2%RN%E1+y87=GrMjYm zP3n4ut;Ox7t4RfvUVY2t1y%VUQ|p1vN-k|RYpXrloar>LaK2iw18M`3Ngs#Tm*dJgkrmhiNPc_2hQ>4{A=Os`M+uS#NLqAMoamdaXoWeptzG`Rs}_ zq9QSMY_jYRBug=lGx(112R@wRnz*`UAOhn}NnUI4g7)RHjoHZDPo_vXUqw)TMOGaW zlYhm*L~1RPXFIbV;p#mZ$TgeOLax z*U;{?oBsf*+(lZNckamU-*lKJ{g8$MGq^St&49)}~IOu^ z1>Me_41R6YX=^c!qG@)-V`VO~u=T(Vj^A~A@m4SF>D6}==o@cczQ(e7c8UPb+~5zH zJfHkkch$luv~6zg)_$j|1dWmoKgQHqYz8GZlB_n`a953aPe>$wz6OKhX-CHwVOEj#k z#zGj6R`opl(q*{Td>iQf}Q1e52!iKE7l3+ zX;I){w1IKAw?FUot3#!{A=Y*7h=W>^^50L6ctNmrwcLBSz@;+{D{{U2X&8!KJjj(c9k}#ouEaU->YTNLN^P>75 zJN<91X4%xbb@N-ToXRGeHuCoSnBMMVk6LoFbh5q@S(4b?U3_M(FgbzfD* zttiB^WA5Fa)TG*#IXNdJ&>U5=xSeUc(KQFXGy$xwn#ynpprr?T1*H~|h=Nf{55+*8 z%_uw43KI6CS`aBj2Rs5#%78d}i&wPJGz-06_i={ZGqF#_Ba?(h`Lhr{T%Ss#HBDY! zYf-wnzet7D@&;iWd;w&M0^lFQtr+J&&ot}e?@*m{UOHW*gz4H_l-!3RYo5;EFub7o z?>@Bv)MhbU#fFYGxVCg4WIM9A!-Lz@cl!E)T2yKMIL61$e`6N?SF1VJxWoNNwBHGi zRrpLQHh{TOJAQxHBjsEN$xTKWrMM&%Nj4Gif{bvd)42p^jDzkgvfl7Egl?Du%OAp| z>kjmlly^xOJ=K%LVI_=eUyN;dHP3-ovUEBOoT`idMr6H3F z7j6LxbC5{;`2-P)i)p=e;d~d9F>P9VmGc;zh;pP6hv2aXjyV_@^{SH8_CK9rm+H{F z7s~R5Vet>Ex*o?(*0qbtZgh=S;$JUKorsVmK~-6E_W@D>QNaVBy;iSDb=IHNmiD%~ zebVOOD#0v6nC0xrBA#}G$xwR`bAety>aMxhdWT(#_DhYcZ8OPep)g`bIS770agSbo zO-npH>WiL})hs&8Ooj<;WRRuZq?p=|*|0gs2fjEB>}wj^VPvr-`JEP@O(;h(mmIIB zx$Ju=HF0>2bYfT&;hH53t;xXS7#@I*F`Dnay(~PGCD69e>Odkv$RCggkM$K8=|2)Q zp1AUtL1okRRR|-txR@f`4X6^G7;i_EfX>%M>=@Cgj;HfEJgfZlReMSNA@}zYw8ch$SH7=sm?%{&w z7@8>LcVq}j$r&D?6Orqj4Dm%vF9T-&T;z%9JwaupzOkxma9v9z%BgP@b9`40`P%?L zhXC*0&d?h*>mIXEP-HPNo?m$v-_ zd(^Pqt&CGML8J$K%ONg2$u{~s9oRmB2+jiLq7TU#Be(@5%Ti)iA2NVd|%v%x3?1GyWy#@;~(0=*~U-;Z|p zXtsTCV7hg{JAbrnk)*Vfj_kb256>Z&<}1dY9>`Owt<*N(GC2_E*CV&`t9(Kb#A6s< zLS$fYGsmyXKRi^LZ9FZjFJ?ok)lQpmmQ(P1X2Ry$_R=f6JBV&==1}oX3x`P*_A0}I zK9$mCC*`B#@f`9%=L8elvaOnSW7yQD z7=}w9N}3z)t+dsyo?{tRnna1QF{bb+C=uU%(0N$u+NsA0aC$R1co4m7{_(?s~g0PZqcEP zX%<&5C5a8T9eEHzJds5E5WlS>!zj7M*zzC8$+Lp4G=Get`bsMyDLXtwMDoDU#y>MA_25`(f9%mj1YdXE=t<-Sq zvd-623wyMA;PQj6-ziyep-SNp1_J~EPUPjHUHQ50tlKM1RyGOl<(HKjV=Om3J}xYsW2VzZVrx&YBYF2Cru#EqBlBQfw0{oTkRAlAOaA~8ZS@^O z{!7bCyid6jbQx4Y+yIS+#?kf3?OZoqO=Z-&W}~fj-l2JYugT_*I$3ZdLch9LB43B^ z_sAn|e(%CeA`+MkbI9a+*1nrT3b&T*=C!{O>MmWd*<%mSJpGG*qBQ6_hp6q|OO^i1 zvWQ<_T_-B3&OU#|^BC>Hs{a7Tje2c!ttLhV5g1}E;E2XP1Q_GtK%$*IGU{c~Bi3xL zBuK1o;9H3|Ha9#6DuV!@k9INgA3B0GcUrB@-m5Rh?$0Z)d}ovS59ljN&pWf4j5fk8 z9}F#}Uk$ZuumTvQOyDC3*cBU!{q92yZSFzsgHc@)N1v(hr-&Gs%)@$&n9l6v1N8i= zbKulXKSp&Qk=;D4l9pJU29R*M8?r$7w(-FC$cS`Dn*?`PHtQ2g?Va+0xkmosNyc;D zn5uJ=iX9u3NlISg8uw9MYyD?yYKa}arG%SehCURs{r4pD%vg6k@lu@{-K4yUa6xw5 zPp~y!bb__kwR3kP7A-DWvCdhEKhV{;;glb3>J?cOv~4P&$IXWypfyU=yOdGtKL|m` zO)ID6xB3$=6V^*V4;Il3Ox|p3@V*(>2a>+E=UrXQH)qU3gz$c1y$RyXR<;^khy=`E zgc3)L0qK+NUUAiK8sXgzLV|F#$*j!K>Q*ReRFmDd*2R&aMJ@jTL)lv@tL)!#Y(yJQ z{J51_$@qbeAVxR^6bx?Xv!e9|)B4Lvx-0_yq9RUlk&r2q!^^q9ur1TejiOnaHN%A^ z!v`4yEcrO#jASR$o^_2f<_kpEH!DVNR6(EJy~LpAQ5t$GfEeciLn`ukrFF@*D58f) z>hP+oDAY+MjHcce9?fs{CH^})P2(>ulvGAz>43SXU25i8dP3&WS#70Dc6WJ3#8Om} zT>1rEp2Xzf3WWI0uJ7>*Te}jUzqO730DOcn$_^BqVDQ;Km^}NXadUBh;Y=2jYH=IO z*E3v;!d0CanTR9;%t=_rIOE&a(x(`d-W0R?f7zX^_=Yv<-k(ngDMc;(W$pVM9t>N- zZ3MFYp=3jFcFLJh$haGTJb-!N;MMJ8EEjhq+PIR|B{8sG#(-H;mxp8P6FWxU18{yE*OpEiW#1;^u!e5y2`)A5Ox%9VDSDA=xb8 z`QxfnY&Et{*yFn+Y?=~7*HY@gXrIixwh*i!{n7>rBZdFq0e%Bucsdt=^) zZbJk2+usD|*XLXs$CGh&Z8OHi#@RyBU}W*0IQ?ol*IJ(VE{w7yMB1J4DwWzpZ3i8* zim@7%OEFkaE&a-mRXtRcWh}IkUuCflz3L^u)^0Gkw@=-v?|=x&`EkcJC9$-C=$99_ z7a!bPGKi#;YY8L`f&lJte>{q4@YhDPxVIN_K@^PbxJ4l&Xa&6o2Oas(rAc~`kP%@V z5yc!n$Bx+c&&!}(tXZA+%3sr~w&>d@$Ahe{-9@^WL}Hv| z`SwlkvDK``EiLrTJy<(MW+s#W0Om>YAI}>FM71J5(4(+a=Z}1!^QjK4a&C1y%Z!{B z*uOwKe=3V=54z=#Joe3L!5aNTwh>T|3E_Q^v}gBwPXzPYis?#2f-(E7^YiADwNo_f z9DI|2KDFJ|;b7G-AY286hzA|^;ZOen<5JfCbZ2VQ{+3evAbLLat*(md7}1&NushM<)n(TiCX5gM&qu zC@n2*r1MILX&Pr@L2z2TwX9HwOmL~(ga5L|WRF3PT zZFO6jt|Ps_)6f{ByVF#G_7YjSWB15>9jtNKh9HcTI)kMr=}UWdk59X~YuJv-@wzl~ z@hNs3kOGF=o^VESnz5$U)j0gWp1+^6O8W1u6=8QJzgdW zZ*v9ABm%1|+p_Lqk6zj2eznWiSAW`fP+HpFTtR4AF>w^GJmPpeaBv1YkTJ#vPDtSz zJ-zQj{f;AZJoC+Q9khmLX=FcWD-!R{;gG7QCxtouJj>k8uU1gu6U{bg?d+}^;$)5_ zjhx8G1ppaurUp$@o-&(@9)yZvZt$h;!lZlKWRfOXjDTYy2n)%2{}4d%5J2SGe*d4FklW zDl+nL4oNGLLC=0F8Ciz#RMzTy>XM4aQ9^L(HSZO=P#xaYnrh1b8`KDF6#o$hck zoGI>cNHUy&4=3E!Qw4A&;KyXr4(@O{{D=9}^P((78hjZ96C#s?+m3(Ap>Be0O5W6g zwmdt5$7;58J5MIw1d&NpQrofLj^FA2m7}1CGBUc2m`WorU)!+h@h8K*9S?k9mcZxp z6#oFxoqw#$)AMv}`o5uIp-RaNUt)=5xkq5D<(UuMg`B$(GDhMD1d2yq8qSreX}XM% z!)t9Vvs_1WdjV*QJ2XmgKM^YX0Qimp7zZLb)Owt*_gA*b4Xlslt;25p^JEdd!u%d% z9PT_|WkAN_)%DYi=iAN;Jvy0JUw6Ou8?_a-(t46{qTN5)?I&-t&%Ic(@&P+QQV7qW z$RmnuU}u&_UVbCGM(3ZIJ-WU0Gf(4>}W@NQ3W(aY*LFr2Q;dR1qnq)JX7js>3)!w;??%*S7h7S zW4gH+X)*o<$^QVTsP(Br*PeW5*T1&9{#$5Yb*7XJoGcH*WCt@p{`?Q5VytOt^=wJ- zw`EOj{*6tpUdWDU_tvbdVV^E&c-qJBf3Mf;No!742vh>tY&sx2omk;lBi(`%>Bl?@ z*0rQArDT^jk{EAb9}9uW`uvSErlT&QXFZ+tNScMPCTLiYuB2mlJ<5ZO{{TEz)Z>`A zob?apQvH81`$m)dS3{Fcwk7UXU|gu+jlUBNcflMVZ>>sn+j!$Zk*F&eBW~_S0rdI$ z{VE^RlUeBcjK=jE&hfTKEb7}cxih(yvl|TO8+pcg&jM4Y>ap4~-djm!sKU{<*rbK6 zN5#nPIbFvg=8DD`ra(422deg6h*AWdW zd#QJb?OFWAiZEG_kBLC;e)kydjGjAJoi(=We%{;`=+#*j9YS7l&T=($I^Oi>SjDHH znN?*00pzX-#AJ_C^7m8gUYO7`0Q0;p}_g7?COO#RYr&kHq4I=0I9PSjafSGJA_8CmkiL=24J zXFEvUY_O2j;aJJ!perEypayh{5 zz`z}jGJR^S@jF$w@iVTENa^?TTXd#?gY7ayhkMESAgYez;XM0ffyk%77<@vw)B0_? zTcbpQB_`f|Nt6S0-JMq-gmx|L0332@J!)6!UkbG)ws$wW#o}4Dz{;^*yXL|4;05E+ z&%IfzOyeI?NWr4`M#)hLx0d|p57~VY9`U;SP1O3z%RevPV@v-4np=YFwj`2Gpmqxzoc{nUew5ke;TPyC3tMu_@@`QT7-4uFIv*#h zEI>`fx#Kv`!fN=_ZKGxg6z~M>c9CCURU`Num!HJ6~GaN+!qHO{eEJ(JR^VGDb2VF) z9l=6W4mSYOHhcEigX(^jKB|53P+m&98oMU!mffiIZ|zyB+TTIbCeC(|Zf)A;+2R>2 z7_P@3mo-#$=SD?-tq!rM$8VJ`5(VB}fDFKL25@jk%;UE%ml~v+u8E;SvC8SALp%#E z+!Tz;j+y7MSpws>0VlO7)c#e5g{Vt2JhNI$7unj{LFF`yj5-o{P#c_j;BlJdwMuch z!5zkiF}F?Xcw3D+Mv>F2)Ymo|WV6j}X>l1wo&2XekSo6MJFmrn2cXU|R=%TUsq1=$ zjHcS&C{CL#vs|Ia+O}=mD<($R#~42r32bqS^X{G1n3C#ej@s(>>5;DG_|Szp4&}RK zhQI(E5Dz)UThG*RL(#fidTsfWSdljg;Y>$qW&tJ^&j+&-y}0f^5M3Il9H+Ti+X-XA z67e5Zj>^&WO)2-x>Qp!0O`wv`o<=!rlY_elo@z^V(~DuG>Gl@!8Kg@mm*dNs1~$sg zy8!qw|Z0{<|DyRZD!73wM_S(efC)JN(R*Y(TgH6#FMYO;E z(u&gVeMEr4S|weis33-7#9?~|_(|h5VZ~w~u-;h-u<5>#zUl6v>RzDM;jzBb=Cy{} zNoGd1k(F8YNSjN!d=()^R5&1armqw{f3xa4tuS=%mE}yf@<#LO69rh>ZH!moVoIja z2Ox3V(yCvz>w3ppjb7r)4MOHQ)>&Y?j4TSORlKJe_*VcfGn1SQlgyK^HGMZ+)EWN( zvuvimg-cC$GmCgacyyNXkoZMNVgCL4LZlWYQl1HA;CLO2fa4ydd8!pbmhNJ+3HU^!i1*yR2UKR z#Yro>1hHj}kHfgJ86XDvXBu>66`k&60gs)_=eo%#?YMEaFS?u-$;C0CLCrKVC<#kQY5xW%|kw>}QenFD5DX95l<4omTt;fo_{K|gj zMhoR5H<>Wls=eFa@~bn$T?ZOO(};Y_mpfqrfFZpfJ%$e;_Q)!GimiOh33|cW7?|nH zaCV@O^B6epdCA6pr_!xnn!K~J(k`^oKD#l(pQ<+TSWvN$ImpIWW1ptsa@X{;rVMUlfh&$JbZL}zdPAigrku>9$e`K-ea zh~0X95^!8>zcJ^aTFhvTWY;%w-i3RFMU8x=2m=ydcHmnJ*a7g2=DIbfdF|}A4}wVc z63|N^ZS2h7c<>cF9G`Xj#AG$u*lHRLm9C+v>34Hk>QM!XUoGQ`Ron^K#|8OJ5;pPg zQrn$NMb@;fF2Zh|nhj3yD$S|xAX^yRfMZn{0|O+nB!i6itFfP_Cndq=HqJNE7`(Zx z{+Y@{we_D%-)Q<@R%&;)^4v(R7+4Y76i;8I?!EP8m;cikbS}2$Iqn+S`1- zenFET2KmzFNVbf~qC0sN&-iyT zt6+S`>w;_4*V9EFooHh$6lc+72i0^HrQsP3d3#mKT z(;lBHs=DIhR-!6MTcC9{7IyHHAC|FWXvo5kiL1WU zz1@bXsuc;gGQ@4h3CZK~{{SkCbfjiIECIC1DhQNqJOlTRf0yQJV|54Ed_UHv+~YRC z7dcWu{{XnGt19&9oR_iEX?4-FsL@kfxv%g407VzlEWc*Bc4bY{G2F)ierM%Qm)EU# zszoHW_RyH%!22|X<81aLB;)B#{TXWo#+=dJ89bLQyUQ7V{g*zqiRC~iXj(%a>?v|L zet6?Ku2H`StMr;9EpMv%S^W+xhYRKs#DYmA4{qk8w=Ujuk}yg9vroE!Xsrydkma~W zQa3L-Vd;#Gr|VJ4n4Bja^HaocQPg)?+Pr^uNP0CCY8LZI3$L7|fbKvbpVt`t={~rP zpuD+OaPQ{)Ob$U~$-(vHn(gT*m|emnP|utyvX%!wA>Zjqc%YF*uWB~XoTL{%P(I~G z)BM4rxAT{Elqht)`N|vmAL721ah??O@+PhA5t`BqC5`aXO0Mm{V?(rt&PF!lj(rI~ z!YZEVcLP`Q$_L!Nk8TgGTzYdu_Q>VBw`+x26zG|<6>#ymj3`sbq1HK#Fz7^8oio2U0d6uL@%U~Kn!{12p7Ag5LEyL8xH0eA8|qk z$0xl<;<|$C?N_>o+g=-BWoaG1d2qm=F`VPOV*n3?000=O8XW|5w#siRJ~QLl7d1M$ zYHN*YrzE$>k;lVLHs0s1?(el-9{&JPw&~p|nbtUxRF36{b0aGA$|XmE@BgfXu~@^CVRpws=NPy+$b|rw z5X=~kdv*sMhkRAYmj)PQvbcvzxk=VfvKv`jEwyr4pEy-q9ofb^5)Mk$-WzFhBCe%0 zGdeVeJ7{Bll3dA-$bABmH$UMA+={Wa=&h`pGYqM5YjX@TdBQBl<%!yxGlCJ7!a@lI zp_W2H8J+4KRrbY|zoU)zcS#ca+g9t31+I;~!H-tyiA?6>`r0;!8he$1;SrMXcV$nyZ)Mn*_el~PBk+TL;Ukl0N*2=XDpMq%9G9G`!cP>%Z- z`+GKEb#3*Aw>^}4J*c<0wX_k)@*<&Jv0_-SsNqLBI3xfKG_Iei-d$Q~+8wpc#ogqK z7>YRaG`-IN@Du<}Gr;7U4HE7f+vK-_NVSOWB(4mt{6v3G%A_4N)K?ud)v2q;Y?oyt zlHH>xZs_EUtKqrY+?*U9-o~#(N!?SM-`Nx=tWdQ=JA1L;tvUmyG(8SGx7DwcQ(Fj4 z)uJLv6n7#q=3-KE%3HPv89b4SnEu3?qQlhvNud_NyRnM#7d&o?O|Aa`#!>oJN!5Do z&bcgsq5Cbq@XHWw%uirg;Ai9j#XWw{dfYuP;|_y=b0CV^=4P;)nuKQpKfO+B=LqLoA6j05!fk5A=IePO5DYF3MNY~tSLJozp- zD8L+p?5EXzd8hdHU`FREN5zwj{XgFn$FEz;CICXl5=e}w&d^6@_WpHhSZJeLah6cw zr%53ek8*9{nIKn@Bf~}KGT`J6Kvmy>ay>_SF>PUZuP}gG%`Qq$ZhW_3Mlta1!OngC z2N|wAiH@spCFf5{T0j;ns|lh8T;y`2(2<;D;cnpf0$1TrO5Sy5pkTH_BDDfpqqvl! zDD2A1f>5y-<9DxWq@`40nQmUPyJccfZDfJ#;`d8P?k*9p?G*F-yiBJ69l-*hepGJw zPD2i9r|nI0qFDAylWY+c#rT9_3oRf$Zc-zgpgfYyy{L9k^_3+ z5J4Va>@S1Rv7+(vx3{ z&V2s>*h5YK0E(|<(k+lYkj3XqAe1ao9G**)!()YBNzZI{oAmXa$4+UnX;u)V6YoY3 zcM+6Gg-K!>E>1zfV0a+$OkERb&{=BNR$_D^g}iJonG4QWb4h?k<^_T0wmqrePQJNz zmOE?fc7RSIg5LT9Bp(RN7LRGkAy+sfpIp>aa+7m$=i8FhAqYiOr1sx_GDyz*ZP66k(N* zBPRW>o zMHt%h58g`IEZe<7Jol<&sq|e-sXS1;w$trn)FQC8K*=F&4WoM!6mB1T?_dGXJ+nTo zP7z+C<(a((RNHKwNw*cx-`N_v)3o~zgj>j2l5uBtigE~)81P6v@s|8pJdx>DlYlty zine;9&r;L;Jii*AhXsU&5My<@FDB=B!-B2x4jDlwJo@uhIVW{xYGkD}(bs6BD8Z?7 zlfR_iI699{nnnQ@a9ihrSNuAzpY}wQUPm5RAiIuv0Oy`gc>e&^R3}QuJL|TeXN6sZ z9G2`47&*bmaaTR6>px(HO_j8YtF}e+g8`gyGYJ{SGCivHhQ}J-*)#f;D%%!bHaWVt zQV{9WNjYamLOj^;$-fy2et75es{5z)ShURw2bwXwM}**H4DRFVPEY0SUA`#X3p)f? zwu05J%%m7d1xHr4%3&5|98rfX^RFnOo{uu(w1p(<*9e zPiMRP7Po}ybE;}_>DLzt4BC~I!oe;`b;96t*bk39@xbqnI68@v+d}@%N#wYR)TDsn zfjdvW004u6xb>*!x35cY;zfnNjLwOrLLlICxR6+J>%h(_((_k}dxW0gnGAW`wSW?~ zdEjy*Ah5|9=RDRoP;1wJkBYHaoQY?Ht+~J_=>2<98~(ONN4@rY@uIb#=L=_x~Cj= z$3COt^{IzS;_JfgPSaApzlkEaU$EFLKf9HSW?31O5;4fmFs*}wyM&hMG)9&i$li4% zdxyEoH!wlN3taA%%Ngrrv)A2lr#4Kn? z$v=37_ccyNY$nsAeRs?I4Y*K}pmi`34&9@hk1$pXY)c4RV-ZsclvS**2HgH#TT%pPXQa#XfU1ITRhbCM6wn&|3r zqB2TSFzpb0v?K zijs$DnVo@goctM)vb2(*~WeQGI{PUc8VAM~;$9A7;!)0*J9wmzEHFU$3 z3~<8$;9z>!WY;s@*!2|f+1uP&J+G7KK+3S(Sn$@k{jFF_Xv_M9-$!sVN50f0fL z%}>Lwz*JQ&OR1(A8-?$tCH>9kTOJrSBU|&E~qUcPu;?}Bac9%804IK4Cc7Bp$()F ztgscHY>^wwla>P|q&?VxH#zQp72}Fi328OF-&Z%X0A- zvi-*BsZdL~ZvOyeEpe&ErD;-M>I6jrVq-)(F2EhC3Ga@6UbS9nx~99;T~}k%dP*20 zy0}&Hm$}|~B%WCJEKi^~_NQB)wDk?9r?1~3OB)MW4U_;9rG^57`H)Y!!0Zi1d^Ehd zgRbrGEhm;SVIz5ac+bLAll2+>YJQiYQogWh)?&p_pbTJYW^i z-Q$tRG~2Ap9S{lC8tg`qo;Gog>P`vz@yD%4wcTcUBaxtjSmTRwhMe#7M{q|U zT<5iS9Wsg0TUN)nK0KZ?@ea1GqkMm+lar4v@y^>bkY1bZag|U;OB1yh;ar};dw-QZ zb+1MBFIHRG+Gth^){#K2^IS@*4p@Rzk-3IJzyRY1n&i+J+ zK+yjYB}SdSk0>V zadW2xSnd3^aJ-X{P7Zqv=Qyf0^&9n?c}fX(<9GaLQK$HKL8IzbJ0|<{l)dqn?0NG; zhSGf@Vr^m@DNuq^KzSMD{{Yy3D(Sp4++QucvqwCyCOpW|(3|#&O%8J*zpU<@7sE9Wv8G+hX$ZvB%Yo{{YktrWAQNj5g*Wk8=F>CnGe+ z{qD?;H}D291vqQ15=ODAhLza4lR3in0DieYI^ebVrBzT|FKU_N232u#u|{0^`@PEp zr)|HnbOY3PGAd0bQ%r;LyY@2?=!xi0-fRKDCZPR7&9POED5#U)6sSJ0ODX zf<**OOf0TQk01bfEzbb|0GY);bY_=tebiR=Hv{b!amZ$s2*7lU1P{&3f6P}2nA?eA zg@YG(&SPvTM&uu20MF}EzY8rISLzK^u|>6lEjktcCoHo}e^QI-_f0=jW5;XZl5|}% zw^1M8^)5|OJe%DT#?s-u;X5;281he-8Nq*i78J)kmML$lLmMktwb@vd58adz#=n=Y zJx6+Y)>yw+>7GL|5!=pD^3w0PR zk`_e_t)AYXdUMCW6v?R@JIzhMv>{_|vqG{z#(|W|frD)zk+iV<>DE0#EP79;ZR9cS zw72_Aa^Sefmc>+#>_IpH_orp7rZ#QH-rRDC50Cv#8)?T&vvhR7oGWcs;ux?`>g138 zA3CJ=oaAG1xcz<=rfpTC06aTky65G%Qyuk;y&z80WvPf8T0K!$%sXsrH+(FZO&cu%vK*dv`Vg z5XbI$UCqc*&m^cXI!2uLajnGSXzfG!WFY)Cy=9)l0az%sN>1xk}`06p2(LB3vd@$=4JIZFk2$GM8?=(=UJTMT9^m=-0L-9e5uJAkx+ zWrhPRfsRh{3o$BxcySN3^z=sJRE@Ouc8sjJNT86YkgpkH7%p7$#z#3IS1^q(`g!5G z5XEz2A(0R`SmJ_0s>r+?C}oYAvzGYzR^Cj(tW1y3p=!t{LEgArn~JjEsdN_l_~@Gqrs)lg=`l+da*d z&79N2923CaOUUEPMD3jVl6e4j_BE^fsKt%lyUXTHa3oaU9`NHC$Zj_BPb1Q${v&ji zpGf#GX{+@VGsmpz@_BkajAX<1a6ox3Cm)Hn5CV2x<2gCS{{T%?lwxHoXKdW3$AYlw zKlxk95IRY*$PR?-_6RxhTVPF6g;f@FB05VViVB9E=)Q1t7$^6zY;y^VAK0NjQ4vHa3c{x#~>s?Ubqio^R^z;7!)vpFb#X}{WXCO$!)!PIhJaM@3 zfNV&VyQ&Tg5WlY#<~@FUZnW05`@&py2SR@d0)qm!QP))Xzc{FgW@^ZdV; z)}nn=f2pgWvzW&T0J5nWetplU`cuBGIf)A>z&@VX{{VlNe`2Zxt zTdB0UkzFLcvuTqgf-p(~>yyVh$phjf0Zm%u%CcJ6d11WRgDks<}BU(@Axmff`20XLH$=Gw8%)03FnOAhNl%iY9{P zApPm`0|mfvcmx8Z;Cv(kIT-Dp{;{-%;?jK)z<6#Wj@{Bya;7kLvXjFys2>5qZ1>}u z{{W#3@m=5l0BB8eP)H$VmMwsU;I`jG*dG0=NZBabp^dOa>Wr$C+utOEt^`W6!EDf& zu62chJgkARlQG6gWv#sv$*?Byl*wq$+vD)?2S0| z#vZIz_-x}Hfp7Dx`d!qY*ZUk-#M1fehqBM=z+`F7sTd3-RX;9ys2pA^^nR3@m4MUr z?N0XY{%5#IS#4rd=3pJ_Op=fkV|N396myyM^T%UheW=R0j^5T;+dhR5pYj#wPJ*-- zdfl02Su;3!6o98Chpv4&&mOf$szcVCVr~2w{R$^;Ie7gMG3zd*(7KPNFD~>OYpY0M zg3`>}T+8M>d7b1fyc{uR$!znU{+O+|WljmJFT_0-*GK5*k|>pKt}kGYW(ol%L$Lg? zct5XdtWO{1Sa#Rx%eUC?DYaV;_2iNCJPmQJ*`fkmmgfWvWOKM5m!>iASG3Syj*|G4^#0rBT?J7{@KSStZC$<9(&W6kGG{h^bVUAm!o~AM+q$GL}wTvk%RhY`BW3C`keJHzioMWYvjiSXUd2TIxGgmexC>fFB_lnYVAi_*I*>))>JZ`1y)!yVNf3 ztst|SBOI2KiIN%PEQcd&0zG)g)Gw_(^!)M33RzB!w)t5%fJxnq7F9jT7|ACCXs&l( zOsTgm=R5vKQPZ!5)g+cSOCM9RWOnHr4L07!Ew?t}d7WY_wX$2mR1h+C1gjkiABXAtCASZUsJ4In^^Z)6^9wZ0 zP6U};i1%aTImU5}3XE#^7ZY9IJ^MG>)kU@6oEalgwXlQ_iYW7(Mh}@UJGQAghfg-* z%Np25Q;Uzc=nc)y)K?C%%3{2?GDf~jVQoGi-op_*JeeQ7xjZfo(~v8#NscfnjHe!5xC(Z!8;D%-fG2Ca zWK~#SGd0qk7piMI74EPn@)F-N#L*q!Xku{X3v+@A%*T`XaJ!nJeM@y~Vbq#Vsc&a( zaTID4MCrO03PS`V)lVbzHG6cS*EalV77bJV7lkhi?RE zz`ot>w}t0{-{1Z7R5w>aZoN~hK?)TMbuumvcI^@N`S5D;>N$+^X_mPyB(lbF?x2JI zRYmn%dHqYQ$$_-$Hw=Ms%L0UOIKc0Lj>i>uq%{>JVtiX%lCM8fPvNbE<41tnDhDy6 zBTF6>FvDbmFu{~CIri=jY0Ih0>K$)=6P>n}7K;b&&RI9%nbl4`l=}U7z=ZUz=A?A> zu+!$8MqOO6ASn2VLQinHBr!Y=4slK$O)b`&pryOBA-j|8;wRc!B8+m3!1o2f9A_E% zRf;HUQQa;MUfUm|i1J3ic3i5dG!6zhR=4w5qd~s{Q$&IkHb{`JVSGn!S9CsVOL^X7suBWL<{fnd9 z%XfWre7N9@Wu)~#EC4b|;9!%*U7GL6%{09y7tTMyJkj+_)2Kx@gc{m;rc#mGL~|qd z04U|N+aEsF=&pd#HCe4K?zNAxrP0Z8YXoW{82l^-8HeJN?kV7IBBP!Vcy0dx57?vm z(cEA8A>_R|Fae%X4}UKi1%?_{{Td~Nz+Qdg{|hA2P-e#AT&gugJ3Mg zpS%keI3PEkF1lWBmDg^pwF7R~Hl&MffxL(<-HBF0-+RnlF>K`L;v==fu9Gz3U6j@LGs0Eg9AsWL(cEEtW@@I4T?B2nH0M zJ!#e3+Xn{lznd9)4wHVi%+nm>miYUP))!G6!bgs;$!=RUX{u-4h@ z4x@)kf<}i#wUBx8#>za#InE1yeEJNZQBjLAcOInHx`p-AhT$EYk+gAw2eJ4HeowJDj#}D;&K2eTfF8 z(OS~Ww3Zjk7nqUqbWmszov+2xZ8z=sTT z+ar%&MP7H>-PV_*rH#$ZvNfwa8Jy-kvC9wYIQGqSbmM6KNpzBHdb%0ojc%c`Nynbi zjleT{s<0|R#^aSg2sNdk)I&>mbK@sH@g0`+tun4DMX02`w*LTU2AQB}fzO8X;;^Cp5J}}4P3cTcjoNnZi=~84VN0T*l$kOIb-0}2TIStPONwN|}X)ZTG$vuyJQTQUr&&j zR#6V^gphdw$jJw507nL#JS(uf(Yie#)nzi?O(~vKR?49&iVz;hganb?ll7?Qgc^mM zwl~lzf+8*%-I>)@nTamr-*NRP&~_LVGSzyQr!;FxE^l?4DI|>=Pbp_v9zq8S6>@p) z`PUPwx>D|{D{TH}q3Ja7hi#JQ7~6uj@#9(k*S@jny*(CN!6(}8Po=u-AwfQbhTKQ6 zQQd_?y81Y6Iu7>S+nvB&$&ZKZp-CSy2s!+!HmWDou%jO@YPtM1u+;3dEA2-| z)V|L=QSNvllVp=9?g8(C-}0i#VI(u6Xg{jHb~Tj}5wGc5j-NM_W}9O!;FH7UNrUn{ zRC@74Ldzip`;m+vn5llgZr=3AMixO``Hu3fS9K9vu>ZGq%&K^Q+OTA!3p)~StZ zP|JUJoDJH3>wQWI{Ueky{c8UJ(4Hvt!*L@phBaB8lmb8;;15zxN9SI960js5`0rP~ zkt`APot#&DL>D%yk`uu#wNEFw7$={Qt^EYHl1G>INbd&D`&nb26x1&v5v<7fZ^<59 zLvUq0^Mb9$J&&zho(+g2@S{!DXSbG7bnkt44sc|7WG7(B$K7T2&IM3<)vUUAN*y{I zi6^|)?$*&HkOV$kG3)`rTyE#xiQpQ)H7$0>!_N(DBh}(}xPekDTR$@<=0w53CpbVh zlHP!T21uq`W81rww>Lf?)5Fx0hFW}klN~}yB-d=@kQI(ADO49=e6hJeLI*kb_x!3m zZzL&eEHX00TJGSYlFS?Thp{}7-xWY=J|gvYwW-TJ=DTri70Z=-dtcqa5x8XtJAw{5 zB%f1E9VM;xC7!Tg>9)U^msg(NIE;cY8ykGFl_SuT-;a8$R+XD;fe8!dOB(VyILROboSr!O)Ty;gEiw+A)$ZYr6tvTCZL675Ii$CKH*0^@EN;Cvh!Tr%FdRt$Q8^>x!w38G#`nGBc93#C%6qJ zjw59(cdrUdJK&D^`+= zD;WIPEtVLh^7gJbG;D-4Y6v?a;|$%&z{#!)Sfjbc%!78FZou244Eg7fuH)5mI3I}Q z_QqQZ-LmaO(y|(OxwE4}(<9VR+ch~Qj?W;&3h^Yq(lEwAA9ydf=uzscjY`}1P--)} zhx0tySP3_0JbqrI2LMy7O&}?EG4mPmm*1a<2LNZ9W9pl&?Za5D+YD|Bvbg*#4=P8` z6)e+jZY_Ft<*Z{DSNB5_vasxbKEu#_>ev0RdWS>S{6L4Ntn~dpN%Vn@Pf!r4npv0+ z+!q{`jc`aL{6`;#s&&)sQU|qon-FT(wv0Iu+<_{7OfYF{El*n0*o%l7c*aMYZ2QNj zb?-~*SLY%t?D2*@{>?W{)1EVE^WE)@B#LNTamL{3jz8bMdyR$|gXnQy61~?N>dGs- zc^BG>0!!^W3EU4yC^vud2CLm)V=QUxb}i8D zL+MJ9%>fkwWKW#dQF>NT0zE$r#`ODvpz`ao1S z(N}Yy`fJTPdmR!f63$bxxQ)qtvXb8I9T-vN^K_`4a1H*06i;PMwN8{xj;t0 z<@qr6&D}oL;2n3YHTx^nwCWdfSNo|D;x}J)+vl!h+w+oo&w7H6Ahl9=)v+3J3Qjao?0h z3YCkuiGP2wH0mv45~T!yGk4wV_5Qi_s9cw>x`mwZwq%NU**g=R!2Y$(tKJKnXzo)P z2M$N4x8+UxY{Y7IXLoCu@&5q4ReLSY#jQ0RxMjeCS=3##36WT?*3Qrw!QoFafsi=L zl79_vRF>Bf+edV}BLLh-Ap>i1l?&e|wmXBGTGfPlloG`$b&Ui}Kp;pTAh92G4j0f- z-9>2{MhWeY zdTOq1cTMfi!=06>cELRHmMyH<8@p#ooV%k8L(Wz;7-lRv03U@<$o(om3^H5CYokYN zYjYzid5X)u_Y;%x6S(A-$nDKM>dA0qHd8a3hPYUz8@Z2hA%WY=AqSFj2?OPhTj^rB z>KSgGkVcHcCM2*>6p}JWYz{c7V48B2-`I}M*(kyZr{_NZ075gTM0Fn%U140sZ4@vn zu6`a2gnqA%eX7KblH$hFNS}I0p_PZI_o%N!+gRG$-`-waN^fr3M|Fz<G~2$Jw3^ykQo;mf^Jf4)#9}f)ZtJY6CLfWnQ%iR6)%M4ha?|deSVcF@k>xT zg{G%)<-*-uTg4I)!a}8)S3Ubh&;9A2qyw}^$%Q8<0|i+M9DI*ENc1Du8LBou>m?`P z?0Q8?og6D(cl}2n7-D}S5cOo!c^Vo4$J)ToaL@^KmB$3I+Gy43$UrML?6~0L z!k6&FPqozbqxQCqG^>WjpqDdYhCm{Dz~z8or)ce;schWslxjsqa``o<^!A`SSamJL z{{UdLX(M|mr$;|%F61AZz0$WP)$lhDDW>;kdkl8p;KJH{}A77Vh zvnW?C+iUW$%GJuqIafaqC{x1?Qpx)E%4mkxn$mQgoCFTT50IeY01JVR_$Q79Th|wj zdwcemk(+h~=Eg_^kLg}x==mfX=AE`i3}ypAEFM0XuSu+5@+5*3tXal;4_y0KHPvB@ zQ};)v{3SIlGtY+q0DLg?*~j+n)%(~Pp|`dm+{-q^i24K`t-#6tVyOWVYC7Jb<~U5= zPa_@UfN{>@k%8}%{OZfLzP)RYCJC6wGi_ui?jSiW^7@LedTcP%Z{W-n3&p?vi>Klm3y}82}Eeh})@sJ543(GOc8;&YXCw!#mC(rmf z%ll%mhK~z$Tg`W=ZftCznZ)qCWz~YJ0td#;juaEg{JW41_2Gwxx<5(iHxXO7ZEj3M zd2w+U#T=o5SmZ&RMA*oP9nt~|oL~puiRlYoh_f19h?PadsB7r}dxpCTaT0(C^9cBn zJ4Q2|&&YhW%u$_8+pJeM{{Xw1#R!wg3`xr${`JmDInQjaaKM`u-rI4D*kmhMP53*Y zNw4lL?%8lqmX}R@Z;D{qS*CTyb{v4CE6VN9ahkO7=qmc%#nArGLO$N)49$W9x~T88 zwl<99e-Q&At6jmNTR&1=PL}#|>^ia|1`e13Bq#4XjNmH$@EUezN}nj5YPLX&To~ zg_)Ke!?5mHj_!MB(>NZau5^nyEeTb&RAt!_xy}jpRqUhnHBjhUg4@h8elY4r`nc1t7dwhhe=&DMilIq^DIwX>AS`Tdv&+IfpAt^ z3B-*T9>XW)%}x9mveT{B@zteU`QkA-E4B6?@Z*7yzP-oJxq3IIQ>Zmtduv!)&NVr? zxX;9>KJyQm-~fHcdbaJlPfFA7BD=PVXy%DbaR-xoA_2)~Je>FE*V}+4qCYlQ2QTqv zvY1uqN&N49zt7xcJvSBlnC^iwmsRq1jIhQJemNND{?zHy{V&q}9TbY+r-~hE<}+EDwtOmC17y1}%8YTwIl=biRTI`4*PSax87weK;83HUL%SRa_<9yeUGJ@1DXlg`3)nm3UiE}Ljj%#)7R%%tP&Sud+;f?Qu9r<12m0<0D9-rxwNv+ zRMgzzT8bX}6584IHM_lpPY`^Vix3~=anC<0yzI4(l)2K4ww2SC8eX8wwmC(_p@H-v znTYn_(+7sV5oc@C&wZ#}g}*8a+nD|oQ}~&fZ3o-G`ZjCVki|TZ?{`JO22cUcIl&&B zaaFCG?$3j@=si9Z>7wP8E?+C1_xB@RO$^q0iME~=id8=vk%B5Qe-qqK9kgl-%Gd+k z)SF$tH*?1<()pnw^72nA2VwbDhv9#Q?<7N8*Q}9I<5OxkZgmIo5!rGG@5s-kS<|$Z zSmCa5L_5 zc>e%m!@SVCV@uNKH)1(r36mQP1I|Z(e%R)quul$dwN1!$(I1&37b-t@y9>1bA%p(_ zx93|r8)5UddA`1>w+wwmx>id)OUF$f&iXU`m!ZJZU6I1Ylq9+NC<&jTre72~zJ9an z`;Av?MP!4?u(v&-fltCQ?tbz4pOrN9=A$jYM{9btj7Fu@w}uS*rK%9}`QVPMsPldWrGE`z~3MJZvA$C+yIG(__1z!h_~ba6_y#6B~eM zf#fabm}M!P@J>fRJaRMoS1zG-z9{YEIYs!K{{R!7=kn}7I!i{=?zN3x+f22=mU#j% z8Nom3Iii^32^BiE>s4cN{D(eoeK(|Z^@M7vA!uX^G-e~_PRze?GlI@{ zcHj(xeKV_zi>+MhQr%B8-r6!Nc{y_mDGF_%Q=HsvVdh9Pj1W#c*`LV{$H|dn^y~E3 zFOk0IPnO=|SWuvz+7I1%5B=ObauPCgnF@{4NKeDa8EQ)A;^tV8nGVR{RI~%1E)sH8 zGsyPba(Dwc7_F|Oa5TFc%QrUHHl?hV2r^bUq}*e{?e}+~Vo4=<^=fO=ty@!(ElQQK zjI?3n&+jA%NRQud9Fgom=OjTGO37hzRFvafd!Gj#-2VV*0I6lN53~YF^Lh0L@aI2M z`O?`oz&6q47X^?EgMv;<_WdzT(%u`H_AwhH{n%gO1A$$OaPlY%><8DM z;CA`{05e}j{6gwGzJu@*>s_^Q+J>E@TSaWr;QsxRIFX3;Ahzs!?XR6W5ol;a{0GS}D!p z7bMkj%PXxYPdv|YaU;zvu3hDkz9)7hmM7ejNi`r4lmz@;y|MoOlu&7=+SojYYi9%j z_lK|g)@pZZkq8qeQ;o!W5BJ)(BM!*m!kqi%lySYu53$rq4{RR7xvmvsZ#v3WUO52v z_x`o3i5SLFeRw0-dsAGzWX8l}uqVAoCQAf6$9o*p5j<)o&fUY*W7fT!`#wx(@K2>2 z{u>h?{>h|Y@~;aEun6-MmK+5GB>acHd-K9A67Q#cE$OWi{vj3Rrk@(zC-6wbyD9mC z_&+deozE#3sE#s8{b(AFgo>_O0nu+ zE|xJ`+9}V&7!fNU&vUWGMs$x4@8tomwb5#Q2Gh^dte+6Y zbaH^vFh4Pc`E4SxZ1t%Q%^ud?!X)@b&CGse#P-e)3OVctt!z_7ToY}-voLi!xhc4$ zm)#6nA4FbR%8v(92>`=q%OK07-s zT6`nIVdU&INiS$GvXHN${yG zY@Q{LS!k?nzRVOTwhAPXkapT4PzE^o#bs(Ci7YH5drNAR8*MhZ{{Uy5F2k;X8A}No z1uMo5a=7;SaZ1}c)MmA}l5L6@d6{2@dy+@UfKNPD+1f<4iP>6N7x#uWvA!jj4SbeDIX1wmcYmI6%FWZZZkaTJSHnhV@TsA zGUtQ4aX#QEC!PSO9Dlf;KtVoMEy{YxL}UJ`)MT~KbioE}N^J<0x*`?VG6AuB9KA;{-RB8?k*#8GpS{;kp7%P1%m|Xbmy`r;Jf<*INQbinxlTAI#KfU-o!o``5OOIsySuv9A-IXLGDsZ@90dw<-_z&ysW(yTu-)m=TF0kZ z;M|Av)({<6ZwsD2@7*J@0Znb`{aK|e9_-<|)mmyb4mxFsed6MtP+zi23 z5->1EdwNq9?9$21p9=AXAIhYE6UidFjlNTD$N*p;N@&rv%gt+5w$p4Odx@rX8&lDJ zSf0QE+@1w$*|N?~V!hu3=ZAei)!O9joiC(Z-x!$^SrRhI5pKngr>XpnTN2rI7f3;F zHG39&%a+^c$e-N$5Thp~k3+#6k~>w;;KrlUI!}k`rS#h(h#7>Ej!cpU0f;B>VYnx@ zc=fJd5kAv*YSOee*EX&)>CWJP!|XxloF8wUJ2g|3I}_Ns9V`_4@W|)02l=1)s{L-rA?5EeJT>Jao&47^&2x0FWM)LmV;cQ&z3W4F6TiZymBu8eV#MtyOS?_AvwG`TKysP#E!+a0uX z#;Y#Yd6ka*mGtl1xX2vyO8A=ga$m=DZ7i)M(#G*fa#0k9E?1qQc9iex)W_2Ds>!HC z6f#b+pM}DxM01?_{vZJu!5=(Rp#K2kcpW%6+LI(2-M)#S$!K7b;&}#Tm@vlZNOHb_ z0#(9}Jw1&_{7AjBz33Ty*Am}FaS;-;h6M~^SS~%86OwrcwMZ=Ei&$u_FKrfEkF!0i zA2EYORYuT2;NvF;(;#Egr9CmzT6Tdjy1I@%Pf%732)T`3J6sGaZN}D@9=(^)XPSzO zyIYK;+Y;*gl4)TTH$wD>Qs4DWhNOCPQ;S25!?f|)Fo{=gPTc+PQR|cIR(-Ce4g23& z+ZTnFU7=)CzXRWoPhnGm)>^imy82yCLaII?F%E=#6+C0*RhNi8b)z>_#j2ePWRoB_ z5ZlZ{t7i`)=kD%a%Gu<8bz40?2Bo<<#_yst#0D9U$I(G8{R$^ z;!fKWIoJ*kJ92T)v8w5(mrNSjw-G#zaAN_R8-6(kalzvpa6LH30%g&}p zJAi&Kx`V&5Zk7tSI_C4h{=a(E)m8PTlWi=l8tDBSQ0qR9mPqGxFfuQj4n|0k4%pOk z@~{VLX9t2#4r=J>?E_BfO)X*2_W73b8-Y5oGEeZ3QHU6OmQnz0ZI93Z2|b-Z4Zlw` zc2ESgvfR>G&9s#GD|4HP%E-EhT7@{xR1)Tkdvp|95Y9AjF}yM zosYt&;oMjsya06OqnOvnCKYg##HY^(R1jL;X)sRFNEYfg@?eFEyd3%xJ{Al7vq13PK`d%BoZ&8{%Jv!w7 z0QE`aV#XIywN87I?Ze{%PSOV_K3VZQ#4FDc_4kG;O`eNq8~v6UGNwShsbD^Tb$bl^ zl1f!qslfToL1mKevg*sesnjD|Smv|PEuEx-+aURB%u6#9$#PD4&d}HgIW)YFkwK{`Jm;N8==76u0tXT=nk-07AX9XDGZCvdKIpIzL&T0WJ zhc2CMdn9-Ek|II2NX|o?x{>l2Imri*J&j(CY6}B2`iC21H1UQVyGCN)7zYFp$2s|a zl=q_D9q9~(eld@s&u^tLD#>ZA!)(EPo2b*d`?)6!KTLfqwjI#6!Hk2E&UY#K`+Vx0 z;e*mTZ%}er^0_@};Qg#Re~0kujUSgK;U(QbEL9k)08gO7`SNRhL#~k_7Q+EaS7lSR z2*Jw}k52ypQBiLQI=T&8!_Vx-iVJCUx#hInW#!gBck*~Ac*xIl?TXf~4Xx7R`a>cg ziX~%$4tV+~1G&jLt1(!LNb}C6e6<>@Coe7k0CAgB>j|zI^xG(y-XLiNUS`x&*n;F? zl<|ymbB+&aQ|MP#a?Pk}>UDUFG?w;IF2#ZUU14L1Jv)%!T6fhapHS88W^-u;rJy2X z6l~8U?E?UD!N=>~vrxLayp3adWV9U%N}&>N`HU)O|T_o3(i_?U^>lvH&;$cHra>ah}-Xp}k$JSZemIEVgmMc)Xb- zTyOWt9DNTRspG_{ZSA!?h_tDvipAVYw$iR1+Du`TgWI1~;E$z1Zo^9%kQ@Px>OEGg zW#|_XoZ)R^Yx=d6!n7 zb?tj>PBo1;M866yP03x@L=G?*A5)GpeSSu&KNux#FGAGqU|%LVu7#3C=I?OB1@rX) zkbZsX+h5nUi0o#(jh+Tk@gpuU!@B!>)pM^s)!dLvs7~_T5^wHRy9p!!M8EqNdHE4m zDz(=cKNvC=m->q#1@P3nu+c0nrMtQyTfqcNmUkqRf%7Gfe>1?Py1PTvv<*)0 zMb&M@7uKbGriuRm?@Ow5{p3*E==WC9X!uctwh~#SaX9t}{#p5AsIA^x-0djpP*Rqb zsQj^u-|Ts@VrG56JXDLJbXhNw>sGjF;kC78ZSf{#mHfoSQIA(`$_Jp_Gr<73>3u7E z9<6k`oxI5nh3U1O5fss)*BrLu0X>r-o=T~i#v!-7)5n_~rKFaxWcN1Vwp*)6 zRbhj^WDZwzABoCjki50%%`MJ%TZKGD{r>>Lt$JDSH3q!7ns{WA+C8$yeAKa0%8dj} zN#Al(;umB+eY|e)2;6FBSN{7XJo;* zjIZ$p$$Vg7R z{d=|R*(HUig7PRLk@sRjfw-TDA7Sh2D_cuitv1X(Y~7Pl@QA*=L|Vk|q4Bj@v_ zG}(7x2KgEdQ4PXxY;GKP&(gSTBq=5s42*&)*~1x*=#p#aS?#UkU<$OHoMf&}KgfLf zs_ym~#q9CQGbC(7{IGFTxm?R>fXsG){CD~P07~G#wuI{u-K1cN;suYObvs)h%j;2F z2RS55Ow=OAzMIh{nt0`!NbREB14#Vr1oy}K)r+Fp$86Dt03hH3IKems4^%&|Z(I`1 zqs41ycN~%^nld(~c~w4uk5WCpl;UTZWZvwoT%F2OfuCP*`+iksh3W2!*UQy-Vw#-q z7sI)?eb1IhY#(aiw}E5$<>a3H;8vxVEYr&)IZ=V;pTao7_s6OA@9HVG>iMPMG1^W+ z^(55tu|3MMt*wL9=7>tmA1J5cKBRw^X{p$=9Fj@>N61mWN%z6^t9SNp@O`@HQabLv z42jbgQtfrgQ-3J$5})87jfedsl5t*h#^tKvosjeUDELhJ#;X^I9Ybr4F49fo%o4|ve_^kyE)Yt`-QPD%A9ia4T%7~p~^;ff>)9C4^sMFRm;4gdpy1z{CVPdk^R z9@T*|Ijv%k6fN4CNG3&P5fzjH$~#ua;-LnKC=nif{iFW?<$OZ3m=pqg2pzVZIVkcd z8TS7Gy)tzD^G?g>OvfK}hBpq}fIFXl*%bZzS?ISnTGvfq+q%Xjx0=bPO32H&FbvPealt0Osp4a+ANV!FqLb!c8@QuonG4h`tndfQr-P_y_^tHcKYZ|tf(_JI0 zr0H>>dG_ceT#~ZNj0evOr9tdLA6l_)v|AY@^O`k|Mar1}01FSNr_=PPM!7wS>LG6> zoRM8X6_B#!8Xo7LuWa%;s(P(FQtpfVjU67YZ{(K$0AKcGld~8uRFXu5H~^3VB$7Rk z%B0aqiPNhZ0uAip0Kv&zf%tojEBE9dtwc=D_Rzm4EUAnRdmc}=M{0VtoJVn{2H6}6 z%`Q-F8-D0L{ytdpa7GC2icZ}2q0yE*q&}&V;U$PdC6QBTjThpPvGCvy=lv^!eLjC9 z{%cP}EU|@mw*^}R2XV$#f%|Sdxo2^Ge*mW103Lv2*4-us`sI!I)=M>sZK!i z=6PL@rb#FCsAW7&C&JuI;U}k(j_u(ue6qvp3+QzH1|@Ds;ZNgie?jZ|delomu(;PX zPc~i53uK>qM_9d^T-E}|yUAaIB=%qRJ?hlxEeR}i<(1HGBR#9mektp{&Gx;P%l)3& z@GrwP{5{i?n*RWnd&&EICH_X;dRTNl7^>k&Ue!l+CZ#3a<8ob@hr|3$=stsw%+;0R z?fGkmiBH0VkyO%1B)e&3Ambzg2?U>~=lrSQPnRY()8%i@?bnN|1TVpvcr}eMh$++}6|SeHE9yj?R|THN9^ACehYW31E_3zyd^L z8O$*H0J-i)4&r$yb9w^frEX*CqL*!!F%ZoBd^pD^xfrV3!j87I=+rYsF3TIP7Q`Si zTQC`D*z#UbNz3|!lj;ScopYdd9+xG+*P)0IH=$q|PI0#$;PLJ%6(mVGwD&y8;hIQdm&qV(RUc`%L`r7;%TUmS7m{{VXRcU8YlM`bJAYL=@K zFU2FU!1pAdQ~rXgZlLPx8wC3oN$KlxCJ6gPpoQm+%Zw7;{#<&~)U&Z0So#r7YQ*Tm z8)>f|d0ITl512?npPyiO!ur*X;I@*MeL1M#T0sL&v6FE;*gIooU~s>SJaP5pW~(&P zZ#5gwI_7bEJ;4tw!eFe$xEqxB_3hXL+Z8hKr5{OcPR>i%uONoXOQRQ>N+T#_E1vmK z;k$9cBBPUD3sH_D`(FqR;FDB!Aj>N?uHo@}G^BO}_88;2^%Wp8gi`F=Km?M1t#YfV0aOK50aZs} z2LOIm$F947WX@BtLR;u6t6s#3`;fl1>c^Vk!EQ(hejo6A9D1?;0OCG%06LUY-8HuC zjmOQ9oR&VLKh)PQwRY=wG_L;u6J(x!OCxSM)K~otzs~4qqhknM<&HK+xJB?5HT7C1U5c5kP>(RQZQSj=yZjaOPx{@ zA5WRa#&iSWW;<~lD%eK@;&F~hVO^UpZ|x$zw2ljXR@L^nm3IF1zVdB7qAD`2jQz$P zt2yJj$t+~Oex}V%BPa65I9|*BnJ<~B!fY>E2i2|O+cc{YSS+C+G;^YZ@b20ec4Abg zU~4rER>H@puU}i5M2EvjlKcICoqzP)`If9r*Jnh@K;BU0u{x`kEPS?Vyou?O=vH0k_$J!x5es_Ra?f znJbI2e5F>@IS<9ov!AT=n~CJSxQZ)D!u^P$K`Q;v-fDb&c83*N3 zUkP-Gk4oyg)Qvxww^sJhq;jlb$UCG26Y-s-V0vJ%tMw%aG>B!1&MEo9VrrK1jRrZM z&c}7KrpTkYllMsK0r*D?yPioo$?s8ZJ5F@ft#!M7GS2@1QTs=c4V2jtTucw#Il&wd zdH6}&+zeMnt9Tkiq(>~a@kt1H-B^&(=i*#}lZ=u8&T~z6T6D5?4xedxG>97JGPCX@ z%8i+D>PINRdW?Wa%AiHU3ni~--+{grnVJq<|BRK9r$E`{F zXT@(aNi8+xXSQ;DnS%Y_U*2QC<@BZ-%`$CAQY#eJ!b#!s-3;o9kiQA%k3cc$icUV4 zh_tG+#_&ZB`6h|C$C24lhjH$G_^H&RjGX=5D(iI{r zPhti=Kc;FA*1bn{sJ`1nEwoBJ$HrBNAmjOSQD}7Y8#IZTk(nGYP^5`_T}) zwh_jwpWZ+|C-v=|4mO1)<=zz!|X>tchqEaOmjmdVM2%A55iRUH1pFsF14%N zl(V*$X#pN&hlPw0j!EsFMPo)Otk4+7@^nNXA=saWv$lI@1b6vWZQ_IJ z3Dmcj7WP9=XR`C|VL|Y*9IS{td_)}l&(5Kg6nHuv2U?x~042QRpL~4^i(1qpve)5g zqYCiCBbk))v1DY_JbhYFe)GcWjX>w)5sc!o!S%>yAAt&HbOe zF}d*@t0dNSsND4RrOB4|RvUiXKL^QNb{)54?yeWPq3}b%ZEvgvwzJeJHjSVhI+h!5 zZkO6MKXFh00Fl4+gWkQ7)7nm(q3IU7MvrL?m4&JZ;DRBAMgF9d$tRL|HBQSOFkZRU zq{^xHN$sC+`y63Vv4llbRTx!J08j=11JD8dtHeGrSq)>xZl~8FD}SZWcAsUI&Oso& z9#R6aus9oW@jqz&c`6Ot`%?8LyJGrhOj*6%j+3ZBX1utxC+!zwe$#G*0#O3|ay~dx z2RYhkiv(CmHz^ZMY7AEq*&ry$tQKo1n2-@vOB{~*$;+`MfGI1dD=ep{)z3{gIP%sG zgK<2XgX*$MZ?!vT7V}FU@A-sv^W%m9#PH!x`5)qK+rJ)JpH+lKZ*ivS?H~UDPW}Ca z$sXi}7Yp#lLHpZwIed~xkdDBY1(EG0vw}vCYP;z&TR|~c72}XRATh};C+<~`Cu*@R ziiGRBpt_t}Tfdf;@w`$^6O|rWb}JGL@q}eYKA`}~$2JsRa>%0G-Vznob*V01TYWlK ziq>FE&Zrl5{jBi}akW?=BkEf?nna%^5p)Mg4{}T5}5abNd3@q-}*@&pVFH(jl%vpS-~uTGj|@K zo;@gr_LOe&n{WpY$T{PWo@+ugZa6vX>+P|f@gSPPjy~_@*!y}@W%aW|D$ao5o`1;G z=8+3BOA{iDw+g(d1Y}_QWd8tKR`Sd}u`k9~?w&hS@sw8X?UO@xohUd5g*<*#j-(7$ zEX&BZkUxIo0Z;NMe_Bf4Fw5nzJRE&#+E~_A+T<0Lc1SbE4;lR%Kb1!F$s~4roH|ye zAC=`xpD2#QxcP(fr?r+pA(iA#+mVN2zKlPo%eS>PY1(F*s-^8BQ-Zsnk)K}m-*5e& z5|~j03f+n^DjTDs1?mn?<2X9z<5#EHHhR#1C#Hvxtpn*RVnO$(mD=AC4Q zPu@VFi34v?e*>@e`uc;KnD`UW7Cn2`X{X=nQP@}{M&?+=fo?949%6)hj(4}=9P&*^ z-|k4G>>q)C6Y5?eYd&nIYd)T|lWq0tj29!HCLeb`r?L)DI0xH0Hk)Ci=~lW{muVH9 zjif>=cp?~B&~^j*l6fSOc{Q3onWyNwRkoF+*+XSvYZ#6TcvoshzQg*Gc_)+2bee*5 za%Gz9QAQdPIj#ConqejZGz8IlP;gB0OS;7y$J0@JJ2F4Z`wxN%<%FH#f2n$ zO-dpQ4DIt3$~1xP767 z0yFF8dGVG0hxD&HT-ZT*46w+Ess|~2kM@4K^fgy$nm)e;>cweoCB%8!vL_3) zbICjdfzEISJu1o3;JmxhQRLJXdv)dGh^ixyNI1{m_q{ATIOpLNFPa840dHp+M z8K`WxzBrS@C#IiHS&GXlUZ1v_<_OYNc$^hAU`KKWamN|1OXzgzNi$qtTFTGH)dDBl0Qg88_CKJmRkM^`#|$pA<##Lk4E+01 zEJ?kG)S8aZ!0oPHK^~ENJZ#IGW(rx)#>_BD9*3UYy=otzmb1{duMc!`5BNR*05klD z&Yty{ZQ-?RYtkGx%tVI9#O7g}IqH&lweO$r9UX=V-K?16)m3S^Bh4-Oea;F53&aED! zXH9-WtCPQvu4}hg=-mTSxw6#bCC%(!eYTyvqe&nHye{~{rx{kxSROrCdWS=^(S&2+ zq+|SDqFRj{EKd$Uy7v2!o{G>ewX1)%%#uf89HuvM9{FW_e{~xEDG#9RJ$(VP%#i6q zD=SC@x^Pxw9Bvg1V1mpX@%FSTH%fR+d_91ylI}ZrgzzH1o+9#YBaMtoU@XCQZT{^6 zCoTt?>^>2Ulfc&zM$uRaVir<`lE9CI@ZSXTD!0QBA(hi4pP3mOw_Pi6yf5x~#-51R zpLQy<+S=~3n(13jyonE&hLl14dmlrZ=rsg~OB3F@wcU-Q` zIh7l0s*jVm3VSoCJDYVOb*Qzf`As1g5z6i!S0OE>0er|#(nus=pd2Z{P<|pA`cm>O zLi19yxx+ygo>69);#2Wq8c*UQJ3;&)`uC-(nOg9M-*hR9l)Maj#<4s%(pia}Bb5H} zNEwr1_+0-052*?^!sfjalSE3P@!cQawP%1xq8p(WVeEy}xI+z_w@F zUKPnDRk59?I3VN^-m+xuO|J&8bQ(jpm()66nW$UX7_Md)kln!Z5WA&C1y29~M$8P4 z#p~#Tc}sbDa)HC%T)oV@pLQ9};sN7;N&QLnrjs_81>Bc0s;$KCOGv{m?8npn1zG+a z_*2wfNb%k3@lT;?C*s{QV(M}K0O`hl;6KLW=%$L}W-*+MvQT@Wzr+5O)jGddHadRT zV21|o?nNaaQ``QgK8yLB4r%kPx)R$@wX-(IRf-$)acE=IpjhQuBRh8hycBP|qz#HTTKO&_fJB;OHrX&;+TLYf z`H0kd<#0Yhxv8yCrXMri@IwbHCGzlj{{X`O00{=2eKxJsP^HGb1ElsA1w^rh4}*X* zB$$DKr{RpU7UzL)#v`B1wv|-q+LN`wUo#k(oVH!KUBquPHh*{sQTJ&nAfDdwqPK_w ztkc>lll$nLk2)e(miUMvR|+#3$;EW-vXsVOciA-o=RrPE3^NXKkGh28a;Z2+R_;g# zHg>yO$)Os(HG>&1Zevz~cHh~Rj#(s>#Rm&qMR|5EBHtr9gOrI8@wrknow@E$a(O<~kc8&a@K050S(AGupZ@>^ zI{yHxdXuZQi<#zutu11g&5qFt3m)L1ij3eBfKEI2bZKU&s=J600G zmu;?XJpNwCY`$pHjp)6N?bXWfP6mn#Kf-MoFWW`+-PD(a&ZJw1 zQt^aU$KJvryd-UuK;oWwPfhlvM+|3UEJ1MlFe&x z8rfbn%l1K>uw3_0OEKKQ9qt7+$u!ApPPxWy9CBS5?rj><-d`@>7G+}0Re>b-B!Dtc z1NmfA)`(}0>dMulH&+ncg9KP+XxT`@s^<(b*kpS0Im~mVR%`83$&GF$jeNq&k@@Zy z2RK8^<2-;nXOBv-CDJXSj?Vd?sx2k7Rahq`AdnrT=gv&10nc;WBoYTAowHDBHf&k0 zS5oUx=gvki7j(Ea9e(QO!DF^q$PXJSj1Z_;kPjmObDrdkcfhNy-r8(6wZwBF>=g1n zvOAB`tPd1r)bDi7M^arzS(HFpStLlL4~4)PjNqOhkT^a4YLxpr000|GfrbYJRGNC7 zvCCxIUx*%xF|KjSUy~(ofFE|~*tX!y!2})%^#pgPS#PGg)2?21M7z0l6Kvoy1LACU z1-=q$3NQ6rU_6_rkw@W^3_qPMA6482#m$sW+i_f}IH$`^V&NOwPl)<_>{dy*ykm9m z^+c}X&&tyfzm|(BRourM7hLDqV>HLE>GDT3*D-*CMBVUkjtS!gas25GYpYW8Lt7ho z<$`6E9VAu*XdU?g9QOSwqSL80OJKodjj}LuyGi=v8T{(rny%RHa(@wPR;N&%Z8VpM zw=W-_4*M%RwsKw>Tp%p`auskhpQ!`$sb+^Hmev<;l1I8(Ruy?7EU&bE+>cz7`c!jF z>ib2Peg2fBmD#x%cFubd^7p6vexvED%Xlq}=H3)?;x~zu1|Wh#T#DP%xUM95?z%Qk zU9yA6zA}XxTU~Fd?KG>`7BKGt_JQqzx7eKbscy5<7gif>q|=z&DGL?EK(Q;HC!lSQzs#EqlZ4V?P=x>+6Q6}u?MzuR_>|NIzFmid7_RRmy~XjBe7h8#^vqUWAmoHC#S45E0>z?c`oh@ zgu@!)65IiTO97mb*a2Bnl;?#f!O+tvRHfLlo^kXbIwyy7-cGZvx7qD?WoNijxETW~ zy~+MvvsR6Uo2BS_RB%QX81Mv)09hQ2WRd)2_wSEPR|Wp9Wf4ZVxN&s(BMe!9BpjUa zoB@(C^Q!*0v`})E z&y4<_?QA%#DDmqmq9ryZi>!H13ViE;SpuR^Tl12$muj?F>8ZEx=F)OP&>Z zK7h@1++nd(x3!+hv3>D_l1@GOHAeNHiv3Ncc9E)k8+219MmM<7zw4{guyP8r7 zNW;2p+vu!Meq~+<_1)iC^xehlwXMwS`{j}gcZvwmk_s^-Zr9@5-mxh8; z2QBducopf5LdpC_@LFXE(`x#LkQo(*!v(^S-ca^qECC!b9I1>SUMH>mxb_`fC1jK$ve@6&`t`&zOQy{$oAc$z zsE)$k;t?b)vGIYqa9EBx} z{HnxLZ(~N7Mws@?l0gU$8S20Pg%RE-=H<>DjgJh%>-MD;S{DG0_o()}sar=?4 zmxOb^nH}m!aU^nrlRQNgca`B;K_kS^!XZp=_ico7s{$3h)o-9 z2bJ8iD;)dfart(xshO4`9H6UtTU+75Y&sG|v`ZILxu&09POe2hgrCc^{oO z-o|bu`${ju6byeMQb%aurzi7dkLoay`@xfr8w0qdBapSaqwqfI^gQQa(WZb#N zc8t*y z(e%$VOt9VNO7p4~WdO3MCoDi9p1^vVhH7_Fw5bd{#;izxRopuR{&ne}*k4>*`wph) zOTdq8_`Z%OP$< z2P1Ohf=3&0dvjL*0ED6SUl=-e_ z0Z;uA;ru;4;bq>p4BC8d&Lq?2fm%z)y8A(jFw$@n5AbY3@3^-jy$p~+YZ$e*h8Q4? zh)AJ8h`jTK00B^UIqpvgC^Jotah!T)fYF>#8hG0=*n>W$l0EiB#Aowhe=4S|WrhsQ>lDxCDus~zBYtoW z2I9ny5k4cI!Z2#-{i3Z!$4TjOB#q|9sGX6h#{9%`Oa292pq9bow;N3(^N!^D4`_wO-G@y* z#;dDLZ}l=t`Ad?h{-IyiPW<*>=RUQUP<0lou3s(Iqbc&C;wZO#C@=>e?SL}Jll-b1 zbqcla%M@uHuYHkg^j}}S*ICbjQ|uEQH4; z%=tph6=ESUI4)RZFXi*^UNN5bbtPjAsf_IdAm`WTnh)(=NWg9W*>tc-K{1INX&?0G z2C=S{mdm-ba+<2DIEp>Xq4ge*si*E5DG~+>jW8L>&p!Q!bMH%Q+D3=3$0e=9taogH zdFVz)PCz90UffrW`a7+4oeKRKFJws-b}Wq945yupJ&(Ou*4`^0HB9gw$O5qq$s}^# zz#I(V^H4_HD3)UI$-N5hSenWf>w7lojaBsBQ%<@rtB9DsAeAxQ2po=np!epYudO&F zS=+~K2*yxlzf)Jflc>Yumxb4wZRBI^(MRT64Y(p?w}BDPQtf@c#fK(K-&FraHT!L{#I>+5FZ>W{aafL}TB zZ*4aWi*b%l^*;iP$m{5)TMwj zsfeam?7o8`m-m?i?l-Ap>RL@I)7GH3!OUy=o!Z&=N$j{@5vgXlS*|6P-Z3f@7Nw`7 zW+mR(X@Y7}G>{bZ&t#;F*kF=!htf8ZR3Mc?=+@mU^?y_UB&2#Fy zoOY1HMa*CF(SZwuCM&o|+5iXp#|lETJ^&krGJsT0?(y$c0_81YW@sWTTVW^S%N|(# z&vHrQHKk2X$Ao0xYN{R2_{B9#xo#tl7Gf41sG*1_U>H)wcWhv1+W>GYG%Z22on{?C zt|4g5cNrNBq-{AEIlgv$^3efDH#qV!59eBOUCDJ6xV-!%3+oTu_?m-GR?it6~-=#W@Zw`yE+TWW=Hc#d52RS1vxDDz^ zATs0<2nLery<l=S&_+3ZXz4UDak=8xE}L z7HvCGoN4!QETdMAGP6j;Wm+-Z9G>0BBZHc*E-c>Ob7OUCG}bo@9j$~rq+46a@D^!+ z&fzcv3&>zbdFKEzEEHj~w69q8OtIa~btS}i8l~eG^eZ3u$Q*I_SMxtQ z(Swu13us`w9A095uM|X zeQK@gi-|n{05zm%W^K&Od1LBH^)+mCzNG6KkU<=9dkVQ+Jw1xHS={nPFZ?rfm(5}3 z0{;LD1~?>oW86{ObR2f^8CozRVYDzG56`#t_p30-w}gv@UoDRTSnm3o;J&kzqms`W z;~xv)0sdb~m%Z>o*oyN|&@{Mh-bs06_)8I$?dhI-Vv*4;G^<|^yylV=nnNSR@4qJu zz#^49S%%zvV~qOeH0RWi6y6+3zb9)E$ERWM^7W<*t#3oqx=}QzxSem~wN{o*p9*F| ze&_{o3mg;9aqKAxVHYg@5PcwRzqARuv9q{X*Un|QxOn4L1OQ8}J@8I(%~)Dph>}R< zvgs`=GfWcRNc9P)L)_zJXLAw(;AIT1f@O4;r25B$RMHhpT#ecJ3<0 z({Fla=c+C49@5&~<&gQQE276Eu_K?#y0puy%`)yg+bg;4WsQReib8~7=hA^! zYa01 zxjs)%&uee^ZY8)3S$#4vFnjjmsTlg>Q`0OO$4!dH*s;sa4Z4SK^8CQ39Yd_TqUTDm zEvno`vcW4KEbZJ?$jKuIwm2TuPE|bYTB)}d{lEMzi@hsa)zn)-wi`k*B&;^59f%#c z`qvEC{U_5O43|Rkc!6w1hAn~4Fme3FMDBH6e_6JSUSHl^DO{o!!i<6FFgd8+w#lb7 zY9Jy>321>O8wV(GFot*MvI*aro%jOW+sOD(n1>OpQ}nkf)qEQnQeC* z+7Koe3cvDJ=VVcw0I|s3n294o--PcS`&7U7v(Y@?4p*q-1MONx;>$hI&O{R884*8( zVUY399CK9v0D~HZ#jVAx_NvY({2{Yu`@X%KY;X>ekUhcU{7&N6xue69%R1DOEi{KW-W zgbKU3$W_P(G&=W&xBVTVSamN{#|cK%7TRqvOZ)X3e8Q!MKzsrS8QMyd?N;GpM?o5f{n&merxHFao>(l%B%?VCkN>r>zPoikhWA#szH|MX%#eAj8EmLxyc2-A`HG)-d#XL>OHRVw zOi|8by=5g@>u?1@e#!<;K<|J^?^o!#So=|(Zklw-Wu}bj{VXnntVsk=PqN#i)cKJ|8;ql1>}mHKxtvCA+jvCiTpksWeO#Tavp zY&;CwqFT9y_^R?3vqc&xWalrV3gWXRZ{j*42M5AoWj7T?Q z<)7)rJ}-7>%NRm=UEdFEf%NyL%|bsmBN_N{_edP${rXp9NMm*_kP8ux{rz~P9!TUM zu33j=13tdA*={;JJ%&fM!XLazqBV>_DC+DC8kx|yE!buOBGgTU;#Wx&uZB;B-1g1a9h0xr5C*q zr3?V{q4c6-L+MBWP_-AO2c-aNva+&(Ifpbr(2mqYwK!=E@o%NIZmH4q^wF=<*6&Zd zH%O8n-OAfQ0f%BZJQ6$i;;7D?&^qT`>Z`jLG-zxtZOn1pvplI4x-Y}au*hs=h3}5s zV!cg@%ZrF@ZDhE+kgct}Z7igo7By1LfA)h-kIOFaAmiU2)_9Bdk#v~5mua2k0XoCQq$KLJiV@-vf~tbmLV$1DNlgZ%5Ktabaor0VTcsV@%ejY{rI zmj}=}$Y05DKPpNVM~FBbkJ711UD9mQR6632jJS6>P7iW@&!qvY!CPPnh_btDoLEbBzyW(#*-v7wj@)KMow{0z8dru=U3gd ziyWfH5ASC^iS7r;5$jggyWy^Z4v9O#WfhBC{AS`IwTS~68R3`Qewe5)-Z;+XjXe8g zvbF=lT|imeqC+buz8k$-5Hio1Ny{%%My zo@>iKA+Fz5=1rr~I%}`fP74Hu6$L7Wq1{v0{Xd{B#8-FmtnlG+wV6zW{lZr`AOcCj z+w4yi?VxLMYI;q+ps`60+EH089Ohgyh8wZQ(ij!n_mqan2Nj6=PN`I`lv<17vURMc zRyyM>$#@*~Z?Vg!+(UYy?Besj^5_`nz>kSjUospVC)r8XB$;c3&=CVkDp*E-$k|aEk4#$oS7voK0^wF*+-}QJ?pOC zP0=dw02~3@n>Bl9)Ah@nS2$FY;68;9Do4nIE2`X;QR$x9tvw=1Wf;2mH!j(2j{}4( z3uI#;j>p$Exfn$$m3wCx?Z$ck06Iq5wuA8V!36pZ)G|KwF(Y}7#9(6|32sR~l|FHj zgi-6i*h*JD2h^6QnBmoQ&rH{?dfwJOSjMLi%%&`3%%I_P2E>yQX^DGA5 zc7H*WSC>I)(?h4(M<9%{#DIBVfsAqA*Wc$=HSMpMtfTP)E7E=)SVw)*`aYt75piP- zpPs-E^8|iX1YSv=!1V`N=$#9puA!{T%`AY6OE`%0u9$i*e(H}`JP(<`suy4Co}uf0 zwzJiGzSc&wUxv#}w`_fuHW^tWLUG3oWPTge_QdPj#i3yY;1MzbhlmOv0MN)}bz3E3UO0_eFXa(Bz%%YSHYhLfuL8MO%Q87{PN zZtW~Bu&r|%sVy4=$0Onl*(!HnWvkA84XaCWVl+FLrnb4Vh%JKK2Ya&!!>^RA+g4T}hdGG~@VO`c|tsSg!A9 zwKqO>xXCf>+@pd)9@xjNd6UP_5B1*;yhKfJq4cYzu+{Gp=E8e<2{2!?gXOpIrbbnJ zHcsL~;=PS?Y5k?3&tZDa8{JzNt_`%U8!2SNY<1;G$-x|s4J_dsBZ(}6n}f3!Bn)GoYux@P^hJa{M?JoxT0IkA z+9JJ^EG=(>h5h00i7}k|lHRzlHh72OKDX1_oZ>qv8RI#4mzF!tqz~h{Hf2}#i8EA^5aonF%^vyH~e^f5m zot+77xZ1-xUi^HR8uJNA~w z+DRvY*)V{*ka9Wn?dj=N{{Z|w99Pccm5<^HyYFL?KTbb7usUy8k{t;ht)5Gpr1BCd zfy+KH=i~tHeqxw4hBmimFr{1!0(%U9yi{?Vl$vFyTzhhh^QHV5>D@cP{-o*X)tvC7JUvVQqS-8coWgNCRzA z_&_;0_hM_wq<~FmRT#x=W?J8KOxW#OZ)(aCed=N{DFDTB$Ohw@?2y+a-RQ^w2RN}Sh(PDOtxLlatWtrw*o7nmmLMFQ3TsGoCn%z* zga8Ksab3P0#3$?io{5)meNF=2 zP3(%{63EwHtAd#s~iZ z8k2Ocp3Sw2nyadK9Y+KQs%<{S)l)!zHB#FV##KXM)L?RQ8?fZ#8LOki^(&%j9ZM`R zO!m{l*9PL*r9i9%5)zB?G00Zi+be^{NP~FGqe*S5*REwjrqrs(!4{t{$4?h1p!JheXR)P^+UHaAGaL-7kz z)3se>{iSYhY+LR$qB6VCk}(`At=KOhmhFLFyYQz}ms#-lNYo~@dq;SR`D}79tW*_0 ziAWqbsKKfW!oLIVdW)zg>Mp#~4ThMHwml?*t#s%}W{{tY%Pa{b;K)hrytk*|NaBei ziZqTiP>6`3Ra6WB0N?;P0;1H)J4G?DPmx+G5>-^QvZ{fSSl|FTB%eWEVDWeMjnr;# z?sT6HCAPEE;BySx{lt6x*Yy_2MhNzD2|d6AnypoCZwQ_cl-?uB=&rBouAb=q8TD?Z z)ot|4MFfK@d{Up`MDFM00k0SMz1K4Mo7F$tcNUMQ^yQ}I)2!PICB{hIE93D#7bn?5 zgTXZd)-}VZ>Mg12*IJKI>TTZZQ<%6PMakpk+vQDn+KtuhrLJY%-NKFlVfql+BE182^&f?x?03I0$T>gf&H1)dPak)oP)GM5^meTTEdNW@=?DlP7 z>-MObQ4*nkvX9TZ)InV1`@Y`F}6RdQr zN4}Tr^4=p#k^tL+oyX9g%Z%4{qZHQBMZL(9Wj6|{gcMRw83Qay2fsX3h-J$h8NEj8 z?NOTF_9J%&@=Hf)gixcYbdjVdXzlRr2mSF~a^7475yoCdBW_YXfZ%(-{{ZvpUACH^ z*-@?K0IMh}J`$`B?}X%i!dT=E{PCV?sI-!7hCen@7{MWOPjAyAom!J`F9iwq;b9!o zO*0aAM5OWw1<&*2`BHL;9V8Dc%Y_^k!hw!>9AcI7u;fb7uOARCxsN&QIi#eS1acUl zmQvq%kP%xQhCBA8agW?|T>Jp6kUTSccnv8Z3!B)(>(R#dl>v#LdafUAZX zAf7w(S8E;s&tZY`9@P_6I&*PnrkEljRB%+7Nh_@C`qlg4FG0nTCmoM zpbb`5y(=gKoOYsG5ILew=9P^Kp#K204x-v$f_+PI4tCb!e= zuq|1x6dkgb_;!)lk5ivoeOidvw&cdUODf)91>51>?XH2LNn>K3eas{mi4VuHgM)$h zhu#OjBeyj9_UnCSV{bEwt*1k}J8YL&$iW~E*z8xI&b;5Ibq)QFqHlc0cOL_S*e@CM z2j3uI`-+_C{a*#V>2+XU1a=;F1^^`T%sC+aGmk+|lvx~F@J`dhD~%=I?$1&Cb%T6R zx+uzmcnm!aNHiN+Y{8CEu%Q8%_8#9~m2>oOi1yOlAGSLTNBDVT$R8L38h zGDS735a*Jq@r4JUg-03ZxyA>2+rI-ZNJ#FbJgUlX)A^lq_xi+g!58vg*z zxU`CL4ZW(7nS8)bKJjMSeiaN07CcvHhJ8&HhO=^#Y8Oug%wGENoU_KvKxo;A?$u8l zkH*T+fC7P2-n`JQb^evOv1j`y|&zshH;;f(wW_Ujlx zHd_z781yauetqktppY=m?caUKQkx zV+G@Eg2p~L-H%LTpF>@`{{TVT==ww!x=ckQfqRbZeM$M6wwgpy$pW}9Wr5VIvYOFf z6KgJvE1UUa`~Ah~x*9ZhyI~#Q?e-|0Powmnrqj{^cA9>-0+d+iArfJn{_V~e>l=~J zD%|3t?cr}+!zs2+9%_aRI=O%{6vpT`u6%%lU>y^Aow#CRR+q0n^;^)v32aT5ovbPk=O(*FR)-+zq1*JAVihAica!>w-h z;V73Hph%?jkzY7O{0) zWXOIvahweKiWF!60LfJAmKkPiDD+;v(^_v-bjG98da`EAP=LcB112-wWMAcnvQ25GNV{gc#tOWnx<}aU`^-DwcWhH1 zCXZ!J409=tc#=iWD#{pSe2p#-YD{GPO&IM`ksSK2pJS@&)9Lz+tXFoI2vS(28CD-Z z>z?Dds`FLoeyY=L#ioaQr|bGse{o~2DKguS;Yo)ieMmnuR&eb@ZE*#~vfW$H6wo^F zSz~a>%kCI)kDW(4<00{zj|C?eNA--CUT^#h(@`)u-ElaPPwL0|RWIW&TEMc|rkkbM zY1c}x9m-n|+_6^SlzM-k2CdH;JWjBOs2b0rbt|1iTl*ZgR}w)!Z0+NAR%au#N;cpy zI8aY>#d*70(?+3oH2YF+?)PR{M*$8yXXWkrnyDx#yk{kt$-T)ZhT7b^)}M0sSQ6XM zvwkob6$hV~OYk_676{o-tB}fqE*x4 zmNv>m0>v8r>?q{t*DP>P&vVjg+>ZvEPWA zjLRaW{5S6o;lnCG6L3DcIUkjIP@jm3`Xl2unPZ@S&D~R@>DK9MWo6UzK?F(JR3-rb z0KciteC&|oM*7yIV}*ku%el4|Xj{EtG%RVr1epT4WhdH~mAI~H8iWF6s3MV{Ax1EH zW7vPKW3bI`++}g@X+Tv}w*xzUN$zW^HNX^{LYGlP18Yk|pCaW(?0I2`P(2xWC(sP% z`qE>H+zbm$rLa7QC5q$>VDf!3Mn6BDHG#o63hV|u|G!j_qE)VZ5L>^Az>LZgR?)KacY7eK+eIu^3v^2LjTL@Lp z?#aR0bK78Q1oSktcL-bg=-wpM!QhT49k6~tC? z!*cO6-C;yxSexK~W9CN~<2}I{$f&RE@1pCsy;7F_I+3l6UQCy_xBmd$q9G2=6l9E@ z%Gf^r_JLPjj)$T3KDX386?)=JDP@B3mKIWAkw|^N;UnTwJK&F%SN{ODPe{q3*>v86 z(^_rK#7Py+?Dq967oIqp4OV{>hL0^Qlq4a5=> zQb=T35cBBDNI30XiQ2q@`#SZ$snWe8C9SojO=%#KNp7xp2rX5NX#|hA-Tc)9jleHF zRj1(>ixcV}@ob$@t1R|9R+zepnPOO;6#%4&Hi6uMx4sXrpoUs`Cejf-hR@pHujaS4 zhr?|a>&d#*;qg=`osy(7<|lLtTHAL6er+&mMyM-)MAVjCcj7}~!708`x3+WxtD zt=?)n{-G?7f2>>=b0d~zh;WIM$-;#nl{CvMDiE!PJZ|>)HF}D>QcIIk6{;)Dy|T^? zUgyh{#c8o{aK(>v>?kZQHoA!v#`(&UG6^2u^cNazn=>4Nf$R@)TrtHhxjW=B4^iA2 z){TCd&+0WU_TF~zi2;FGNr;XK!Cn}T;XeF!q~(JHapaNw>z>n4iv!I&nNBya!-4OU zPB2Rp@$Zd;1MG2+txp^>gqw(tk0`O#EiY9`FiMhPP6+_=2ew8 z89SQVI}LK$BvJnE6bd*PP(R<+nDqd=jwAiy^$m_c&ZQTz417!qfxsQV@$;zesE@b4 z(egjj{b|-LErhMM_dQ+v4{Hy5;bx_zNX3qot2mE7k<5ZXKAVI7wR^FzEB?Xv#Wk+6 z(zEcrt)h(8a1yADuz94zbnb$n!S+4P|9r>6%I$6LCjZ zR3q#&&9CZn7Ts{eKGnx%CC;aN_IJ2s&*H{SWo2}_T|G(d6f;*jS6l6P9=d`%E|s>j zXl%vh&GcLR*_`1y_8gzo50!d9_BFn^)Vw~`n(NClF0CSmSk(z%DHsE^41jZt_Q|ZQ zu7^aDTOL8yH8$B}?v_ivYPtQU?BTfZJE$d)e0FVbbrJS>?o%JAW*?npWnP~Q%a13J zc7dO*55}N0QZ#Y~_#Jzr4nq%^ARo%Iva;U=)7!lnsY|W&mW!?2BBc73oo#t4uVC(g zwmw+zzlE(Gq*g|wXAA4uwLk~T0AJ1TAXlP`H`0`8$~OS zrz2}+7oL51=CZPp3ds_V0jO_U^)&q%9;c|xC)Hw+qm~`E)+MirYnJsj_l+&%4a2|Vyn;(!0DkeIYsyeqtdu>BnHaaz`VRWr71+=%@ z-)zc~SttrbKmLUj?kUB`urdUKY`(exZM=t*WRy^nDPWBcy)-aoRBMLYf92_4? z%F2O~$C1+5gYa!oBOGlQ&2(9^xZ3C#V0zY8Q%4HivqxF3~iqfHRLl$7LRXaambb=vx)-4mW(mX5?EUV_o*v46lNKQ%aNjc9Rme;haU1vemH2cls z86#+!qiwFa7~HHmR{4s`%7)#nSoRh_gpTN43DY`XNlVh!x_BCO72W{ zB&~nTAIP9}LQUjQ=L(R5KRV#LmDG$MZ|7NAT6%xPjUK800E$4Y(giN8K^#`#m+cu@ zP&cM)D=Spv*OQX+);Oe6y^9f#Y8yM5EgVS}JMs@=52a;gMGwSE_e7F-#_&PLK(5;u zS8fkD8SPnFR+lA5!1H4oA!Xa0y@{s=k8E5Xaz=4kSyIG-oL6cI$8DoI_o)7;fTQ>1 z6IoeJvw}*wEx*`mQobtda3}#M(3zMILX*Tt^)>7U?^#({^5oKo0)DhUm6epi!1_?P dYRbw01KNbvR!|1LD1QxQWdH%JtgNR2|JkJStOfu8 diff --git a/tests/phase4_prompts.tsv b/tests/phase4_prompts.tsv deleted file mode 100644 index 3d8fae8..0000000 --- a/tests/phase4_prompts.tsv +++ /dev/null @@ -1,5 +0,0 @@ -bus yellow school bus -cat a photo of a cat -apostrophe don't stop -empty -long a person riding a bicycle on a sunny day in the park diff --git a/tests/phase6_cases.tsv b/tests/phase6_cases.tsv deleted file mode 100644 index d4abaef..0000000 --- a/tests/phase6_cases.tsv +++ /dev/null @@ -1,3 +0,0 @@ -point_single 0 504:504 -box_only 0 180:160:820:860 -box_and_points 1 620:640 260:260 180:160:820:860 diff --git a/tests/phase7_cases.tsv b/tests/phase7_cases.tsv deleted file mode 100644 index 16c7a47..0000000 --- a/tests/phase7_cases.tsv +++ /dev/null @@ -1,3 +0,0 @@ -single_mem_no_ptr Single memory, no pointer point_single 0 - -cond_prev_with_ptr Conditioning and previous memory with pointers point_single|box_only 0|1 1|2 -two_prev_with_ptr Two previous memories with pointers point_single|box_only 1|2 1|2 diff --git a/tests/test_debug_encoder.cpp b/tests/test_debug_encoder.cpp deleted file mode 100644 index bd42f94..0000000 --- a/tests/test_debug_encoder.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/** - * Debug test for image encoder: loads Python-preprocessed image, - * runs through C++ ViT, dumps all intermediate tensors for comparison. - * - * Usage: - * ./test_debug_encoder - * - * ref_dir should contain preprocessed.bin/.shape from dump_phase3_reference.py. - * The test uses sam3_encode_image_from_preprocessed() to ensure identical input. - */ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -// Compare C++ ggml tensor (dumped as [ne0, ne1, ne2]) vs Python NHWC [1, H, W, E] -// ggml stores: flat[e + w*E + h*E*W] where ne0=E, ne1=W, ne2=H -// Python NHWC: flat[h*W*E + w*E + e] (with optional batch dim) -// These are actually the same flat layout! -static compare_result compare_ggml_vs_nhwc(const ref_tensor_f32& cpp, - const ref_tensor_f32& ref) { - return compare_tensors(cpp.data.data(), ref.data.data(), - std::min(cpp.numel(), ref.numel())); -} - -// Compare C++ ggml tensor (dumped as [C, W, H]) vs Python NCHW [1, C, H, W] -// ggml stores: flat[c + w*C + h*C*W] -// Python NCHW: flat[c*H*W + h*W + w] -// These differ! Need transposition. -static compare_result compare_ggml_vs_nchw(const ref_tensor_f32& cpp, - const ref_tensor_f32& ref) { - if (cpp.shape.size() < 3 || ref.data.empty()) { - compare_result r; - r.max_diff = 999.0f; - return r; - } - int C = cpp.shape[0], W = cpp.shape[1], H = cpp.shape[2]; - - // Transpose ggml → NCHW - std::vector transposed(C * W * H); - for (int c = 0; c < C; ++c) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - transposed[c * H * W + h * W + w] = cpp.data[c + w * C + h * C * W]; - - return compare_tensors(transposed.data(), ref.data.data(), - std::min((int)transposed.size(), ref.numel())); -} - -static void print_result(const char* name, const compare_result& r, float atol) { - const char* status = (r.max_diff < atol) ? "[PASS]" : "[FAIL]"; - fprintf(stderr, " %s %-40s max=%.6e mean=%.6e cos=%.8f\n", - status, name, r.max_diff, r.mean_diff, r.cosine_sim); - if (r.max_diff >= atol) { - fprintf(stderr, " worst_idx=%d a=%.6e b=%.6e\n", - r.worst_index, r.worst_a, r.worst_b); - } -} - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - fprintf(stderr, "\nref_dir: directory with Python reference tensors\n"); - fprintf(stderr, "output_dir: where to dump C++ tensors\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argv[2]; - const std::string out_dir = argv[3]; - ensure_dir(out_dir); - - // ═══ Load model ═══ - fprintf(stderr, "\n═══ Loading model ═══\n"); - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; // CPU for determinism - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state\n"); - return 1; - } - - // ═══ Stage 1: Load Python-preprocessed image ═══ - fprintf(stderr, "\n═══ Stage 1: Loading Python-preprocessed image ═══\n"); - auto ref_img = load_ref_f32(ref_dir + "/preprocessed"); - if (ref_img.data.empty()) { - fprintf(stderr, "Failed to load %s/preprocessed.bin\n", ref_dir.c_str()); - return 1; - } - fprintf(stderr, " Loaded preprocessed image: shape=["); - for (size_t i = 0; i < ref_img.shape.size(); ++i) { - if (i > 0) fprintf(stderr, ","); - fprintf(stderr, "%d", ref_img.shape[i]); - } - fprintf(stderr, "] (%d elements)\n", ref_img.numel()); - - const int img_size = 1008; - const int expected_elems = 3 * img_size * img_size; - if (ref_img.numel() != expected_elems) { - fprintf(stderr, " ERROR: expected %d elements, got %d\n", expected_elems, ref_img.numel()); - return 1; - } - - // ═══ Encode image using Python-preprocessed data ═══ - fprintf(stderr, "\n═══ Encoding image (from Python preprocessed data) ═══\n"); - bool ok = sam3_encode_image_from_preprocessed(*state, *model, ref_img.data.data(), img_size); - if (!ok) { - fprintf(stderr, "sam3_encode_image_from_preprocessed failed!\n"); - return 1; - } - - // ═══ Stage 2: Dump and compare ViT intermediate tensors ═══ - fprintf(stderr, "\n═══ Stage 2: Comparing ViT intermediate tensors ═══\n"); - - int n_pass = 0, n_fail = 0, n_skip = 0; - - // Helper to dump via sam3_dump_state_tensor and compare with ref in NHWC layout - auto check_nhwc = [&](const char* tensor_name, const char* ref_name, float atol) { - // Dump C++ tensor - sam3_dump_state_tensor(*state, tensor_name, out_dir + "/" + tensor_name); - - // Load both - auto cpp = load_ref_f32(out_dir + "/" + tensor_name); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - - if (cpp.data.empty()) { - fprintf(stderr, " [SKIP] %s — not found in C++ state\n", tensor_name); - n_skip++; - return; - } - if (ref.data.empty()) { - fprintf(stderr, " [SKIP] %s — no Python reference '%s'\n", tensor_name, ref_name); - n_skip++; - return; - } - - auto r = compare_ggml_vs_nhwc(cpp, ref); - print_result(ref_name, r, atol); - if (r.max_diff < atol) n_pass++; else n_fail++; - }; - - auto check_nchw = [&](const char* tensor_name, const char* ref_name, float atol) { - sam3_dump_state_tensor(*state, tensor_name, out_dir + "/" + tensor_name); - - auto cpp = load_ref_f32(out_dir + "/" + tensor_name); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - - if (cpp.data.empty()) { - fprintf(stderr, " [SKIP] %s — not found in C++ state\n", tensor_name); - n_skip++; - return; - } - if (ref.data.empty()) { - fprintf(stderr, " [SKIP] %s — no Python reference '%s'\n", tensor_name, ref_name); - n_skip++; - return; - } - - auto r = compare_ggml_vs_nchw(cpp, ref); - print_result(ref_name, r, atol); - if (r.max_diff < atol) n_pass++; else n_fail++; - }; - - // ── Patch Embedding ── - fprintf(stderr, "\n--- Patch Embedding ---\n"); - check_nhwc("dbg_patch_embed", "patch_embed", 1e-4f); - - // ── After Positional Embedding ── - fprintf(stderr, "\n--- After Positional Embedding ---\n"); - check_nhwc("dbg_after_pos_embed", "after_pos_embed", 1e-4f); - - // ── After LayerNorm Pre ── - fprintf(stderr, "\n--- After LayerNorm Pre ---\n"); - check_nhwc("dbg_after_ln_pre", "after_ln_pre", 1e-4f); - - // ── Per-block outputs ── - // Tolerance is progressive: early blocks have tiny error, later blocks accumulate - // f32 drift on values of magnitude ~100-270 across 32 blocks. - // Max absolute error grows but relative error stays < 0.1% and cosine ≈ 1.0. - fprintf(stderr, "\n--- ViT Block Outputs ---\n"); - for (int i = 0; i < 32; ++i) { - char cpp_name[64], ref_name[64]; - snprintf(cpp_name, sizeof(cpp_name), "dbg_block_%d_out", i); - snprintf(ref_name, sizeof(ref_name), "block_%d_out", i); - - // Check if Python reference exists for this block - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - if (ref.data.empty()) continue; - - // Realistic tolerance for f32 accumulation over transformer blocks. - // Values reach magnitude ~270 by block 20, so 0.05 absolute tolerance - // corresponds to ~0.02% relative error. - float atol = 1e-3f + i * 2e-3f; - check_nhwc(cpp_name, ref_name, atol); - } - - // ── Final ViT output (NCHW comparison) ── - fprintf(stderr, "\n--- Final ViT Output ---\n"); - check_nchw("vit_output", "vit_output_bchw", 0.05f); - - // ── Neck outputs (NCHW comparison) ── - fprintf(stderr, "\n--- Neck Outputs (detector) ---\n"); - for (int i = 0; i < 4; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_det_%d", i); - check_nchw(name, name, 5e-3f); - } - - fprintf(stderr, "\n--- Neck Outputs (tracker) ---\n"); - for (int i = 0; i < 4; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_trk_%d", i); - // Check if ref exists for tracker path - auto ref = load_ref_f32(ref_dir + "/" + name); - if (ref.data.empty()) continue; - check_nchw(name, name, 5e-3f); - } - - // ═══ Summary ═══ - fprintf(stderr, "\n═══════════════════════════════════════════\n"); - fprintf(stderr, "Results: %d passed, %d failed, %d skipped\n", n_pass, n_fail, n_skip); - fprintf(stderr, "═══════════════════════════════════════════\n"); - - state.reset(); - sam3_free_model(*model); - model.reset(); - - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/test_debug_fenc.cpp b/tests/test_debug_fenc.cpp deleted file mode 100644 index 2dd3343..0000000 --- a/tests/test_debug_fenc.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include - -struct tensor_case { - std::string name; - std::string label; - float atol; -}; - -static std::vector build_cases() { - std::vector cases; - for (int i = 0; i < 6; ++i) { - char name[64], label[64]; - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - snprintf(label, sizeof(label), "Fusion encoder layer %d", i); - // Tolerance: early layers should be tight, later layers allow more accumulation - float atol = (i < 3) ? 1e-4f : 2e-4f; - cases.push_back({name, label, atol}); - } - return cases; -} - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, - "Usage: %s [n_threads]\n" - "\n" - " ref_dir : directory with Python-dumped fenc tensors\n" - " (from dump_fenc_from_package.py)\n" - " model_path : path to sam3_f32.ggml weights\n" - " n_threads : optional (default 1)\n", - argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string model_path = argv[2]; - const int n_threads = (argc > 3) ? std::atoi(argv[3]) : 1; - - const std::string cpp_dir = ref_dir + "/cpp_out_fenc"; - if (!ensure_dir(cpp_dir)) { - fprintf(stderr, "Failed to create output dir %s\n", cpp_dir.c_str()); - return 1; - } - - // ── Load model ── - sam3_params params; - params.model_path = model_path; - params.n_threads = n_threads; - params.use_gpu = false; - - fprintf(stderr, "Loading model from %s...\n", model_path.c_str()); - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - // ── Run fenc-only test ── - fprintf(stderr, "\nRunning fusion encoder with Python-dumped inputs from %s\n", ref_dir.c_str()); - if (!sam3_test_fenc_only(*model, ref_dir, cpp_dir, n_threads)) { - fprintf(stderr, "sam3_test_fenc_only failed\n"); - return 1; - } - - // ── Compare per-layer outputs ── - fprintf(stderr, "\n══════════════════════════════════════════════════════════════\n"); - fprintf(stderr, " Fusion Encoder: Python vs C++ Comparison\n"); - fprintf(stderr, "══════════════════════════════════════════════════════════════\n\n"); - - // First verify inputs are loaded correctly - { - auto ref_img = load_ref_f32(ref_dir + "/fenc_input_tgt"); - auto cpp_img = load_ref_f32(cpp_dir + "/fenc_img_input"); - if (!ref_img.data.empty() && !cpp_img.data.empty() && - ref_img.numel() == cpp_img.numel()) { - auto r = compare_tensors(cpp_img.data.data(), ref_img.data.data(), - ref_img.numel(), 1e-6f); - fprintf(stderr, " Input check: img_feats max_diff=%.2e (should be ~0)\n", - r.max_diff); - } - - auto ref_pos = load_ref_f32(ref_dir + "/fenc_input_pos"); - auto cpp_pos = load_ref_f32(cpp_dir + "/fenc_pos_input"); - if (!ref_pos.data.empty() && !cpp_pos.data.empty() && - ref_pos.numel() == cpp_pos.numel()) { - auto r = compare_tensors(cpp_pos.data.data(), ref_pos.data.data(), - ref_pos.numel(), 1e-6f); - fprintf(stderr, " Input check: pos_embed max_diff=%.2e (should be ~0)\n", - r.max_diff); - } - - auto ref_prompt = load_ref_f32(ref_dir + "/fenc_input_prompt"); - auto cpp_prompt = load_ref_f32(cpp_dir + "/fenc_prompt_input"); - if (!ref_prompt.data.empty() && !cpp_prompt.data.empty() && - ref_prompt.numel() == cpp_prompt.numel()) { - auto r = compare_tensors(cpp_prompt.data.data(), ref_prompt.data.data(), - ref_prompt.numel(), 1e-6f); - fprintf(stderr, " Input check: prompt max_diff=%.2e (should be ~0)\n", - r.max_diff); - } - } - - fprintf(stderr, "\n"); - - // Compare per-layer outputs - const auto cases = build_cases(); - bool overall_ok = true; - int n_pass = 0; - - fprintf(stderr, "| %-30s | %10s | %10s | %10s | %10s | %6s |\n", - "Layer", "max_diff", "mean_diff", "cosine", "tolerance", "status"); - fprintf(stderr, "|%s|%s|%s|%s|%s|%s|\n", - "--------------------------------", "------------", "------------", - "------------", "------------", "--------"); - - for (const auto & tc : cases) { - auto ref = load_ref_f32(ref_dir + "/" + tc.name); - auto cpp = load_ref_f32(cpp_dir + "/" + tc.name); - - if (ref.data.empty()) { - fprintf(stderr, "| %-30s | %10s | %10s | %10s | %10.1e | %-6s |\n", - tc.label.c_str(), "N/A", "N/A", "N/A", tc.atol, "SKIP"); - fprintf(stderr, " (missing Python reference %s/%s)\n", ref_dir.c_str(), tc.name.c_str()); - continue; - } - if (cpp.data.empty()) { - fprintf(stderr, "| %-30s | %10s | %10s | %10s | %10.1e | %-6s |\n", - tc.label.c_str(), "N/A", "N/A", "N/A", tc.atol, "SKIP"); - fprintf(stderr, " (missing C++ output %s/%s)\n", cpp_dir.c_str(), tc.name.c_str()); - continue; - } - - if (ref.numel() != cpp.numel()) { - fprintf(stderr, "| %-30s | %10s | %10s | %10s | %10.1e | %-6s |\n", - tc.label.c_str(), "N/A", "N/A", "N/A", tc.atol, "FAIL"); - fprintf(stderr, " SHAPE MISMATCH: ref=%d cpp=%d\n", ref.numel(), cpp.numel()); - overall_ok = false; - continue; - } - - auto r = compare_tensors(cpp.data.data(), ref.data.data(), ref.numel(), tc.atol); - bool pass = r.max_diff <= tc.atol; - if (pass) n_pass++; - if (!pass) overall_ok = false; - - fprintf(stderr, "| %-30s | %10.3e | %10.3e | %10.8f | %10.1e | %-6s |\n", - tc.label.c_str(), - r.max_diff, r.mean_diff, r.cosine_sim, - tc.atol, pass ? "PASS" : "FAIL"); - - if (!pass) { - fprintf(stderr, " worst idx=%d ref=%.6g cpp=%.6g bad=%d/%d\n", - r.worst_index, r.worst_b, r.worst_a, r.n_bad, r.n_total); - } - } - - fprintf(stderr, "\n%d/%zu layers passed\n", n_pass, cases.size()); - fprintf(stderr, "Overall: %s\n", overall_ok ? "PASS" : "FAIL"); - - if (!overall_ok) { - fprintf(stderr, "\n>>> DIVERGENCE DETECTED. Next steps:\n"); - fprintf(stderr, " 1. If layer 0 fails: bug is in self-attention or cross-attention\n"); - fprintf(stderr, " 2. Add finer-grained dumps (after SA, after CA, after FFN)\n"); - fprintf(stderr, " 3. Check weight loading in convert_sam3_to_ggml.py\n"); - } - - return overall_ok ? 0 : 1; -} diff --git a/tests/test_e2e_pcs.cpp b/tests/test_e2e_pcs.cpp deleted file mode 100644 index 3b74e89..0000000 --- a/tests/test_e2e_pcs.cpp +++ /dev/null @@ -1,468 +0,0 @@ -/** - * End-to-end PCS pipeline test: loads Python-preprocessed image, - * runs through C++ ViT + neck + text encoder + fusion encoder + DETR decoder - * + segmentation head, dumps all intermediate tensors for comparison against - * Python reference. - * - * Usage: - * ./test_e2e_pcs - * - * ref_dir should contain pcs_ref/ with tensors from dump_e2e_pcs.py: - * - preprocessed_chw.bin/.shape - * - token_ids.bin/.shape - * - text_features.bin/.shape - * - neck_det_*.bin/.shape - * - fenc_layer*_out.bin/.shape - * - ddec_layer*_out.bin/.shape - * - pred_logits.bin/.shape, pred_boxes.bin/.shape - * - seg_pred_masks.bin/.shape - * etc. - */ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -// ── Metric row (same as test_e2e_pvs.cpp) ── - -struct metric_row { - std::string stage; - std::string name; - std::string shape_str; - float mae = 0.0f; - float max_abs = 0.0f; - float mean_rel = 0.0f; - float cosine = 0.0f; - float p95 = 0.0f; - float p99 = 0.0f; - bool ok = false; - float tolerance = 0.0f; -}; - -static metric_row compute_full_metrics(const float * a, const float * b, int n, float tol) { - metric_row m; - m.tolerance = tol; - if (n == 0) return m; - - double sum_diff = 0.0, sum_rel = 0.0; - double dot_ab = 0.0, dot_aa = 0.0, dot_bb = 0.0; - float max_d = 0.0f; - - std::vector diffs(n); - for (int i = 0; i < n; ++i) { - float d = fabsf(a[i] - b[i]); - diffs[i] = d; - sum_diff += d; - if (d > max_d) max_d = d; - float denom = fabsf(b[i]) + 1e-8f; - sum_rel += d / denom; - dot_ab += (double)a[i] * (double)b[i]; - dot_aa += (double)a[i] * (double)a[i]; - dot_bb += (double)b[i] * (double)b[i]; - } - - m.mae = (float)(sum_diff / n); - m.max_abs = max_d; - m.mean_rel = (float)(sum_rel / n); - double ddenom = sqrt(dot_aa) * sqrt(dot_bb); - m.cosine = ddenom > 0.0 ? (float)(dot_ab / ddenom) : 0.0f; - - std::sort(diffs.begin(), diffs.end()); - m.p95 = diffs[(int)(0.95 * n)]; - m.p99 = diffs[(int)(0.99 * n)]; - m.ok = max_d <= tol; - return m; -} - -// Compare raw flat data (same byte layout between Python and C++) -static metric_row compare_flat(const char * stage, const char * label, - const std::string & cpp_path, - const std::string & ref_path, - float atol) { - auto cpp = load_ref_f32(cpp_path); - auto ref = load_ref_f32(ref_path); - metric_row m; - m.stage = stage; - m.name = label; - m.tolerance = atol; - if (cpp.data.empty() || ref.data.empty()) { - fprintf(stderr, " [SKIP] %s (cpp=%s ref=%s)\n", label, - cpp.data.empty() ? "missing" : "ok", - ref.data.empty() ? "missing" : "ok"); - return m; - } - std::string s; - for (size_t i = 0; i < ref.shape.size(); ++i) { - if (i > 0) s += ","; - s += std::to_string(ref.shape[i]); - } - m.shape_str = s; - int n = std::min(cpp.numel(), ref.numel()); - return compute_full_metrics(cpp.data.data(), ref.data.data(), n, atol); -} - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - fprintf(stderr, "\nref_dir should contain pcs_ref/ from dump_e2e_pcs.py\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = std::string(argv[2]) + "/pcs_ref"; - const std::string cpp_out = std::string(argv[2]) + "/pcs_cpp"; - ensure_dir(cpp_out); - - // ═══ Load model ═══ - fprintf(stderr, "\n═══ Loading model ═══\n"); - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state\n"); - return 1; - } - - // ═══ Stage 1: Load Python-preprocessed image ═══ - fprintf(stderr, "\n═══ Stage 1: Loading Python-preprocessed image ═══\n"); - auto ref_img = load_ref_f32(ref_dir + "/preprocessed_chw"); - if (ref_img.data.empty()) { - fprintf(stderr, "Failed to load %s/preprocessed_chw.bin\n", ref_dir.c_str()); - return 1; - } - fprintf(stderr, " Loaded preprocessed image: %d elements\n", ref_img.numel()); - - const int img_size = 1008; - bool ok = sam3_encode_image_from_preprocessed(*state, *model, ref_img.data.data(), img_size); - if (!ok) { - fprintf(stderr, "sam3_encode_image_from_preprocessed failed!\n"); - return 1; - } - fprintf(stderr, " Image encoded successfully\n"); - - // ═══ Stage 2: Dump ViT block outputs (from state debug tensors) ═══ - fprintf(stderr, "\n═══ Stage 2: ViT Backbone ═══\n"); - std::vector report; - - // ViT block outputs are stored in state during encoding - for (int i : {0, 7, 15, 23, 31}) { - char cpp_name[64], ref_name[64]; - snprintf(cpp_name, sizeof(cpp_name), "dbg_block_%d_out", i); - snprintf(ref_name, sizeof(ref_name), "vit_block_%02d_out", i); - - sam3_dump_state_tensor(*state, cpp_name, cpp_out + "/" + cpp_name); - auto cpp = load_ref_f32(cpp_out + "/" + cpp_name); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - if (cpp.data.empty() || ref.data.empty()) { - fprintf(stderr, " [SKIP] %s\n", ref_name); - continue; - } - // f32 accumulation through 32 blocks: allow up to 0.07 max error - int n = std::min(cpp.numel(), ref.numel()); - auto m = compute_full_metrics(cpp.data.data(), ref.data.data(), n, 0.07f); - m.stage = "ViT"; - m.name = ref_name; - char shape_s[64]; - snprintf(shape_s, sizeof(shape_s), "%d,%d,%d,%d", - ref.shape.size()>0?ref.shape[0]:0, ref.shape.size()>1?ref.shape[1]:0, - ref.shape.size()>2?ref.shape[2]:0, ref.shape.size()>3?ref.shape[3]:0); - m.shape_str = shape_s; - report.push_back(m); - } - - // ═══ Stage 3: Neck ═══ - fprintf(stderr, "\n═══ Stage 3: Neck (SimpleFPN) ═══\n"); - // Dump neck detector features — these are stored in state - for (int i = 0; i < 3; ++i) { - char tensor_name[32], ref_name[32]; - snprintf(tensor_name, sizeof(tensor_name), "neck_det_%d", i); - snprintf(ref_name, sizeof(ref_name), "neck_det_%d", i); - - sam3_dump_state_tensor(*state, tensor_name, cpp_out + "/" + tensor_name); - auto cpp_t = load_ref_f32(cpp_out + "/" + tensor_name); - auto ref_t = load_ref_f32(ref_dir + "/" + ref_name); - if (cpp_t.data.empty() || ref_t.data.empty()) { - fprintf(stderr, " [SKIP] %s\n", ref_name); - continue; - } - - // ggml stores [C, W, H, B], Python dumps NCHW as [C, W, H, B] via ggml layout - // Both should match in flat layout now - int n = std::min(cpp_t.numel(), ref_t.numel()); - auto m = compute_full_metrics(cpp_t.data.data(), ref_t.data.data(), n, 5e-3f); - m.stage = "Neck"; - m.name = ref_name; - std::string s; - for (size_t j = 0; j < ref_t.shape.size(); ++j) { - if (j > 0) s += ","; - s += std::to_string(ref_t.shape[j]); - } - m.shape_str = s; - report.push_back(m); - } - - // ═══ Stage 4-9: Text + Fusion + DETR + Segmentation (via sam3_test_dump_phase5) ═══ - fprintf(stderr, "\n═══ Stage 4-9: Text Encoder + Fusion + DETR + Segmentation ═══\n"); - - // Load reference token IDs (saved as float32 by Python dump) - auto ref_tokens_f = load_ref_f32(ref_dir + "/token_ids"); - if (ref_tokens_f.data.empty()) { - fprintf(stderr, "Failed to load token IDs from ref_dir\n"); - return 1; - } - // Convert float→int32 - std::vector token_ids(ref_tokens_f.numel()); - for (int i = 0; i < ref_tokens_f.numel(); ++i) { - token_ids[i] = (int32_t)ref_tokens_f.data[i]; - } - // Pad to 32 if needed - while ((int)token_ids.size() < 32) token_ids.push_back(0); - fprintf(stderr, " Token IDs: ["); - for (int i = 0; i < 32; ++i) { - if (i > 0) fprintf(stderr, ", "); - fprintf(stderr, "%d", token_ids[i]); - } - fprintf(stderr, "]\n"); - - // Run the full phase 5 dump (text encoder + fusion encoder + DETR decoder + seg head) - const std::string phase5_out = cpp_out + "/phase5"; - ensure_dir(phase5_out); - if (!sam3_test_dump_phase5(*model, *state, token_ids, phase5_out, params.n_threads)) { - fprintf(stderr, " [FAIL] sam3_test_dump_phase5 failed\n"); - return 1; - } - fprintf(stderr, " Phase 5 tensors dumped to %s\n", phase5_out.c_str()); - - // ═══ Compare text encoder ═══ - fprintf(stderr, "\n --- Text Encoder ---\n"); - { - auto m = compare_flat("TextEnc", "text_features", - phase5_out + "/text_features", - ref_dir + "/text_features", 1e-4f); - m.stage = "TextEnc"; - report.push_back(m); - } - - // ═══ Compare fusion encoder layers ═══ - fprintf(stderr, "\n --- Fusion Encoder ---\n"); - { - // Compare fenc_output (the final encoder output) - // C++ dumps as [D, 5184, 1], Python dumps as [5184, 1, 256] - // Both are flat [5184*256] floats in memory - auto m = compare_flat("FEnc", "fenc_output", - phase5_out + "/fenc_output", - ref_dir + "/fenc_output", 5e-3f); - m.stage = "FEnc"; - report.push_back(m); - } - for (int i = 0; i < 6; ++i) { - char name[64]; - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - auto m = compare_flat("FEnc", name, - phase5_out + "/" + name, - ref_dir + "/" + name, 5e-3f); - m.stage = "FEnc"; - report.push_back(m); - } - - // ═══ Compare DETR decoder layers ═══ - fprintf(stderr, "\n --- DETR Decoder ---\n"); - for (int i = 0; i < 6; ++i) { - char name[64]; - snprintf(name, sizeof(name), "ddec_layer%d_out", i); - auto m = compare_flat("DDec", name, - phase5_out + "/" + name, - ref_dir + "/" + name, 0.1f); - m.stage = "DDec"; - report.push_back(m); - } - - // ═══ Compare scoring ═══ - fprintf(stderr, "\n --- Scoring ---\n"); - { - auto m = compare_flat("Score", "scoring_class_scores", - phase5_out + "/scoring_class_scores", - ref_dir + "/scoring_output", 0.5f); - m.stage = "Score"; - report.push_back(m); - } - - // ═══ Compare segmentation masks ═══ - fprintf(stderr, "\n --- Segmentation ---\n"); - { - auto m = compare_flat("SegHead", "seg_mask_logits", - phase5_out + "/seg_mask_logits", - ref_dir + "/seg_pred_masks", 1.0f); - m.stage = "SegHead"; - report.push_back(m); - } - - // ═══ Compare final model outputs ═══ - fprintf(stderr, "\n --- Final Outputs ---\n"); - { - // pred_boxes - auto m = compare_flat("Output", "pred_boxes", - phase5_out + "/ddec_pred_boxes", - ref_dir + "/pred_boxes", 0.1f); - m.stage = "Output"; - report.push_back(m); - } - { - // presence score - auto m = compare_flat("Output", "presence_logit", - phase5_out + "/ddec_presence_logit", - ref_dir + "/presence_logit", 0.5f); - m.stage = "Output"; - report.push_back(m); - } - - // ═══ Now run the actual sam3_segment_pcs and compare final results ═══ - fprintf(stderr, "\n═══ Running sam3_segment_pcs (end-to-end) ═══\n"); - - // Re-encode image (sam3_test_dump_phase5 may have invalidated state) - ok = sam3_encode_image_from_preprocessed(*state, *model, ref_img.data.data(), img_size); - if (!ok) { - fprintf(stderr, "Failed to re-encode image\n"); - return 1; - } - - sam3_pcs_params pcs; - pcs.text_prompt = "cat"; - pcs.score_threshold = 0.3f; - pcs.nms_threshold = 0.1f; - - sam3_result result = sam3_segment_pcs(*state, *model, pcs); - - fprintf(stderr, "\n C++ detections: %zu\n", result.detections.size()); - for (size_t i = 0; i < result.detections.size(); ++i) { - const auto & det = result.detections[i]; - fprintf(stderr, " Det %zu: score=%.4f box=[%.1f, %.1f, %.1f, %.1f]\n", - i, det.score, det.box.x0, det.box.y0, det.box.x1, det.box.y1); - } - - // Compare with Python results - auto py_scores = load_ref_f32(ref_dir + "/final_scores"); - auto py_boxes = load_ref_f32(ref_dir + "/final_boxes_xyxy"); - - if (!py_scores.data.empty() && !result.detections.empty()) { - fprintf(stderr, "\n Python detections: %d\n", py_scores.numel()); - for (int i = 0; i < py_scores.numel(); ++i) { - fprintf(stderr, " Py Det %d: score=%.4f box=[%.1f, %.1f, %.1f, %.1f]\n", - i, py_scores.data[i], - py_boxes.data[i*4+0], py_boxes.data[i*4+1], - py_boxes.data[i*4+2], py_boxes.data[i*4+3]); - } - - // Compare top detection - float score_diff = fabsf(result.detections[0].score - py_scores.data[0]); - float box_diff = fabsf(result.detections[0].box.x0 - py_boxes.data[0]) - + fabsf(result.detections[0].box.y0 - py_boxes.data[1]) - + fabsf(result.detections[0].box.x1 - py_boxes.data[2]) - + fabsf(result.detections[0].box.y1 - py_boxes.data[3]); - fprintf(stderr, "\n Score diff: %.4f\n", score_diff); - fprintf(stderr, " Box L1 diff: %.2f pixels\n", box_diff); - - metric_row m_score; - m_score.stage = "E2E"; - m_score.name = "score_match"; - m_score.max_abs = score_diff; - m_score.ok = score_diff < 0.05f; - m_score.tolerance = 0.05f; - report.push_back(m_score); - - metric_row m_box; - m_box.stage = "E2E"; - m_box.name = "box_match (L1 px)"; - m_box.max_abs = box_diff; - m_box.ok = box_diff < 50.0f; - m_box.tolerance = 50.0f; - report.push_back(m_box); - } - - // ═══ Compare masks ═══ - if (!result.detections.empty() && !result.detections[0].mask.data.empty()) { - auto py_mask = load_ref_f32(ref_dir + "/final_masks_binary"); - if (!py_mask.data.empty()) { - int mask_w = result.detections[0].mask.width; - int mask_h = result.detections[0].mask.height; - int n_pix = mask_w * mask_h; - int n_match = 0; - int n_total = std::min(n_pix, py_mask.numel()); - for (int i = 0; i < n_total; ++i) { - float cpp_val = result.detections[0].mask.data[i] > 0 ? 1.0f : 0.0f; - float py_val = py_mask.data[i]; - if (fabsf(cpp_val - py_val) < 0.5f) n_match++; - } - float iou_approx = (float)n_match / (float)n_total; - fprintf(stderr, "\n Mask agreement: %d/%d pixels (%.2f%%)\n", - n_match, n_total, 100.0f * iou_approx); - - metric_row m_mask; - m_mask.stage = "E2E"; - m_mask.name = "mask_agreement"; - m_mask.max_abs = 1.0f - iou_approx; - m_mask.ok = iou_approx > 0.95f; - m_mask.tolerance = 0.05f; - m_mask.cosine = iou_approx; - report.push_back(m_mask); - } - } - - // ═══ Print Full Report ═══ - fprintf(stderr, "\n"); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n"); - fprintf(stderr, " END-TO-END PCS PIPELINE COMPARISON: Python vs C++ (cat.jpg with text=\"cat\")\n"); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n\n"); - - int n_pass = 0, n_fail = 0, n_skip = 0; - fprintf(stderr, " %-8s %-8s %-30s %-20s %12s %12s %12s %12s %12s %12s\n", - "Status", "Stage", "Tensor", "Shape", "MAE", "Max", "RelErr", "Cosine", "P95", "P99"); - fprintf(stderr, " %-8s %-8s %-30s %-20s %12s %12s %12s %12s %12s %12s\n", - "------", "-----", "------", "-----", "---", "---", "------", "------", "---", "---"); - - for (const auto & m : report) { - if (m.shape_str.empty() && m.mae == 0.0f && m.max_abs == 0.0f && m.cosine == 0.0f) { - fprintf(stderr, " %-8s %-8s %-30s\n", "SKIP", m.stage.c_str(), m.name.c_str()); - n_skip++; - continue; - } - const char * status = m.ok ? "PASS" : "FAIL"; - fprintf(stderr, " %-8s %-8s %-30s %-20s %12.4e %12.4e %12.4e %12.8f %12.4e %12.4e\n", - status, m.stage.c_str(), m.name.c_str(), m.shape_str.c_str(), - m.mae, m.max_abs, m.mean_rel, m.cosine, m.p95, m.p99); - if (m.ok) n_pass++; - else n_fail++; - } - - fprintf(stderr, "\n══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n"); - fprintf(stderr, " TOTAL: %d PASS, %d FAIL, %d SKIP out of %d tensors\n", - n_pass, n_fail, n_skip, (int)report.size()); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n\n"); - - // Save mask for visual comparison - if (!result.detections.empty()) { - std::string mask_path = cpp_out + "/cat_mask.png"; - if (sam3_save_mask(result.detections[0].mask, mask_path)) { - fprintf(stderr, " Saved C++ mask: %s\n", mask_path.c_str()); - } - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/test_e2e_pvs.cpp b/tests/test_e2e_pvs.cpp deleted file mode 100644 index 848cffa..0000000 --- a/tests/test_e2e_pvs.cpp +++ /dev/null @@ -1,333 +0,0 @@ -/** - * End-to-end PVS pipeline test: loads Python-preprocessed image, - * runs through C++ ViT + neck + SAM prompt encoder + mask decoder, - * dumps all intermediate tensors for comparison against Python reference. - * - * Usage: - * ./test_e2e_pvs - * - * ref_dir should contain: - * - preprocessed.bin/.shape (from dump_phase3_reference.py) - * - phase6/cat_box/ (from dump_phase6_reference.py) - */ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -// Compare ggml [ne0, ne1, ne2] vs Python NHWC [1, H, W, E] — same flat layout -static compare_result compare_ggml_vs_nhwc(const ref_tensor_f32 & cpp, - const ref_tensor_f32 & ref) { - return compare_tensors(cpp.data.data(), ref.data.data(), - std::min(cpp.numel(), ref.numel())); -} - -// Compare ggml [C, W, H] vs Python NCHW [1, C, H, W] — need transposition -static compare_result compare_ggml_vs_nchw(const ref_tensor_f32 & cpp, - const ref_tensor_f32 & ref) { - if (cpp.shape.size() < 3 || ref.data.empty()) { - compare_result r; - r.max_diff = 999.0f; - return r; - } - int C = cpp.shape[0], W = cpp.shape[1], H = cpp.shape[2]; - - std::vector transposed(C * W * H); - for (int c = 0; c < C; ++c) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - transposed[c * H * W + h * W + w] = cpp.data[c + w * C + h * C * W]; - - return compare_tensors(transposed.data(), ref.data.data(), - std::min((int) transposed.size(), ref.numel())); -} - -struct metric_row { - std::string stage; - std::string name; - std::string py_shape_str; - float mae = 0.0f; - float max_abs = 0.0f; - float mean_rel = 0.0f; - float cosine = 0.0f; - float p95 = 0.0f; - float p99 = 0.0f; - bool ok = false; - float tolerance = 0.0f; -}; - -static metric_row compute_full_metrics(const float * a, const float * b, int n, float tol) { - metric_row m; - m.tolerance = tol; - if (n == 0) return m; - - double sum_diff = 0.0; - double sum_rel = 0.0; - double dot_ab = 0.0, dot_aa = 0.0, dot_bb = 0.0; - float max_d = 0.0f; - - std::vector diffs(n); - for (int i = 0; i < n; ++i) { - float d = fabsf(a[i] - b[i]); - diffs[i] = d; - sum_diff += d; - if (d > max_d) max_d = d; - float denom = fabsf(b[i]) + 1e-8f; - sum_rel += d / denom; - dot_ab += (double)a[i] * (double)b[i]; - dot_aa += (double)a[i] * (double)a[i]; - dot_bb += (double)b[i] * (double)b[i]; - } - - m.mae = (float)(sum_diff / n); - m.max_abs = max_d; - m.mean_rel = (float)(sum_rel / n); - double denom = sqrt(dot_aa) * sqrt(dot_bb); - m.cosine = denom > 0.0 ? (float)(dot_ab / denom) : 0.0f; - - std::sort(diffs.begin(), diffs.end()); - m.p95 = diffs[(int)(0.95 * n)]; - m.p99 = diffs[(int)(0.99 * n)]; - m.ok = max_d <= tol; - return m; -} - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - fprintf(stderr, "\nref_dir: directory with Python reference tensors\n"); - fprintf(stderr, " Should contain preprocessed.bin/.shape and phase6/cat_box/\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argv[2]; - const std::string cpp_out = ref_dir + "/cpp_out"; - const std::string cpp_p6_out = ref_dir + "/cpp_out_phase6"; - ensure_dir(cpp_out); - ensure_dir(cpp_p6_out); - - // ═══ Load model ═══ - fprintf(stderr, "\n═══ Loading model ═══\n"); - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state\n"); - return 1; - } - - // ═══ Stage 1: Load Python-preprocessed image ═══ - fprintf(stderr, "\n═══ Stage 1: Loading Python-preprocessed image ═══\n"); - auto ref_img = load_ref_f32(ref_dir + "/preprocessed"); - if (ref_img.data.empty()) { - fprintf(stderr, "Failed to load %s/preprocessed.bin\n", ref_dir.c_str()); - return 1; - } - fprintf(stderr, " Loaded preprocessed image: %d elements\n", ref_img.numel()); - - const int img_size = 1008; - bool ok = sam3_encode_image_from_preprocessed(*state, *model, ref_img.data.data(), img_size); - if (!ok) { - fprintf(stderr, "sam3_encode_image_from_preprocessed failed!\n"); - return 1; - } - fprintf(stderr, " Image encoded successfully\n"); - - // ═══ Collect all metrics ═══ - std::vector report; - - // Helper: dump state tensor, compare with ref in NHWC layout - auto check_nhwc = [&](const char * stage, const char * tensor_name, - const char * ref_name, float atol) { - sam3_dump_state_tensor(*state, tensor_name, cpp_out + "/" + tensor_name); - auto cpp = load_ref_f32(cpp_out + "/" + tensor_name); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - if (cpp.data.empty() || ref.data.empty()) { - fprintf(stderr, " [SKIP] %s\n", ref_name); - return; - } - int n = std::min(cpp.numel(), ref.numel()); - auto m = compute_full_metrics(cpp.data.data(), ref.data.data(), n, atol); - m.stage = stage; - m.name = ref_name; - // Build shape string from ref - std::string s; - for (size_t i = 0; i < ref.shape.size(); ++i) { - if (i > 0) s += ","; - s += std::to_string(ref.shape[i]); - } - m.py_shape_str = s; - report.push_back(m); - }; - - auto check_nchw = [&](const char * stage, const char * tensor_name, - const char * ref_name, float atol) { - sam3_dump_state_tensor(*state, tensor_name, cpp_out + "/" + tensor_name); - auto cpp = load_ref_f32(cpp_out + "/" + tensor_name); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - if (cpp.data.empty() || ref.data.empty()) { - fprintf(stderr, " [SKIP] %s\n", ref_name); - return; - } - // Transpose ggml→NCHW - if (cpp.shape.size() >= 3) { - int C = cpp.shape[0], W = cpp.shape[1], H = cpp.shape[2]; - std::vector transposed(C * W * H); - for (int c = 0; c < C; ++c) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - transposed[c * H * W + h * W + w] = cpp.data[c + w * C + h * C * W]; - int n = std::min((int) transposed.size(), ref.numel()); - auto m = compute_full_metrics(transposed.data(), ref.data.data(), n, atol); - m.stage = stage; - m.name = ref_name; - std::string s; - for (size_t i = 0; i < ref.shape.size(); ++i) { - if (i > 0) s += ","; - s += std::to_string(ref.shape[i]); - } - m.py_shape_str = s; - report.push_back(m); - } - }; - - // ═══ Stage 2: ViT backbone ═══ - fprintf(stderr, "\n═══ Stage 2: ViT Backbone ═══\n"); - check_nhwc("ViT", "dbg_patch_embed", "patch_embed", 1e-4f); - check_nhwc("ViT", "dbg_after_pos_embed", "after_pos_embed", 1e-4f); - check_nhwc("ViT", "dbg_after_ln_pre", "after_ln_pre", 1e-4f); - - for (int i = 0; i < 32; ++i) { - char cpp_name[64], ref_name[64]; - snprintf(cpp_name, sizeof(cpp_name), "dbg_block_%d_out", i); - snprintf(ref_name, sizeof(ref_name), "block_%d_out", i); - auto ref = load_ref_f32(ref_dir + "/" + ref_name); - if (ref.data.empty()) continue; - // f32 accumulation across 32 blocks: max abs error ~0.06 on values up to - // magnitude ~280 (relative error < 0.02%). Cosine similarity is 1.0. - float atol = 0.07f; - check_nhwc("ViT", cpp_name, ref_name, atol); - } - - // ═══ Stage 3: Neck ═══ - fprintf(stderr, "\n═══ Stage 3: Neck (SimpleFPN) ═══\n"); - check_nchw("Neck", "vit_output", "vit_output_bchw", 0.05f); - for (int i = 0; i < 4; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_det_%d", i); - check_nchw("Neck-Det", name, name, 5e-3f); - } - for (int i = 0; i < 4; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_trk_%d", i); - auto ref = load_ref_f32(ref_dir + "/" + name); - if (ref.data.empty()) continue; - check_nchw("Neck-Trk", name, name, 5e-3f); - } - - // ═══ Stage 4-5: SAM Prompt Encoder + Mask Decoder ═══ - // Use sam3_test_dump_phase6_from_ref_inputs which loads neck data from reference - // files instead of reusing state tensors (avoids graph allocator buffer issues). - // This still tests the SAM decoder numerics end-to-end since the neck outputs - // were verified to match in Stage 3 above. - fprintf(stderr, "\n═══ Stage 4-5: SAM Prompt Encoder + Mask Decoder ═══\n"); - sam3_pvs_params pvs_params; - pvs_params.box = {200.0f, 50.0f, 800.0f, 950.0f}; - pvs_params.use_box = true; - pvs_params.multimask = false; - - if (!sam3_test_dump_phase6_from_ref_inputs(*model, ref_dir, pvs_params, - cpp_p6_out, params.n_threads)) { - fprintf(stderr, " [FAIL] sam3_test_dump_phase6_from_ref_inputs failed\n"); - } else { - const std::string p6_ref = ref_dir + "/phase6/cat_box"; - struct { const char * name; const char * label; float tol; } p6_tensors[] = { - {"sam_pe_sparse", "PE sparse embeddings", 1e-4f}, - {"sam_pe_dense", "PE dense embeddings", 1e-6f}, - {"sam_pe_image_pe", "PE image PE", 1e-5f}, - {"sam_dec_image_feats", "Dec image features", 5e-3f}, - {"sam_dec_tokens_initial", "Dec initial tokens", 1e-4f}, - {"sam_dec_block0_queries", "Dec block0 queries", 1e-4f}, - {"sam_dec_block0_keys", "Dec block0 keys", 1e-4f}, - {"sam_dec_block1_queries", "Dec block1 queries", 2e-4f}, - {"sam_dec_block1_keys", "Dec block1 keys", 2e-4f}, - {"sam_dec_final_queries", "Dec final queries", 2e-4f}, - {"sam_dec_feat_s1_proj", "Dec feat_s1 projection", 5e-3f}, - {"sam_dec_feat_s0_proj", "Dec feat_s0 projection", 5e-3f}, - {"sam_dec_upscaled", "Dec upscaled embedding", 9e-4f}, - {"sam_dec_mask_tokens", "Dec mask tokens", 2e-4f}, - {"sam_dec_masks", "Dec mask logits", 8e-3f}, - {"sam_dec_iou", "Dec IoU predictions", 2e-4f}, - {"sam_dec_obj_score", "Dec object score", 2e-4f}, - {"sam_dec_sam_token", "Dec SAM token", 2e-4f}, - }; - - for (const auto & t : p6_tensors) { - auto cpp = load_ref_f32(cpp_p6_out + "/" + t.name); - auto ref = load_ref_f32(p6_ref + "/" + t.name); - if (cpp.data.empty() || ref.data.empty()) { - fprintf(stderr, " [SKIP] %s (missing cpp=%s ref=%s)\n", - t.name, - cpp.data.empty() ? "yes" : "no", - ref.data.empty() ? "yes" : "no"); - continue; - } - int n = std::min(cpp.numel(), ref.numel()); - auto m = compute_full_metrics(cpp.data.data(), ref.data.data(), n, t.tol); - m.stage = "SAM-Dec"; - m.name = t.label; - std::string s; - for (size_t i = 0; i < ref.shape.size(); ++i) { - if (i > 0) s += ","; - s += std::to_string(ref.shape[i]); - } - m.py_shape_str = s; - report.push_back(m); - } - } - - // ═══ Print Full Report ═══ - fprintf(stderr, "\n"); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n"); - fprintf(stderr, " END-TO-END PVS PIPELINE COMPARISON: Python vs C++ (cat.jpg with box [200,50,800,950])\n"); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n\n"); - - int n_pass = 0, n_fail = 0; - fprintf(stderr, " %-8s %-6s %-30s %-16s %12s %12s %12s %12s %12s %12s\n", - "Status", "Stage", "Tensor", "Shape", "MAE", "Max", "RelErr", "Cosine", "P95", "P99"); - fprintf(stderr, " %-8s %-6s %-30s %-16s %12s %12s %12s %12s %12s %12s\n", - "------", "-----", "------", "-----", "---", "---", "------", "------", "---", "---"); - - for (const auto & m : report) { - const char * status = m.ok ? "PASS" : "FAIL"; - fprintf(stderr, " %-8s %-6s %-30s %-16s %12.4e %12.4e %12.4e %12.8f %12.4e %12.4e\n", - status, m.stage.c_str(), m.name.c_str(), m.py_shape_str.c_str(), - m.mae, m.max_abs, m.mean_rel, m.cosine, m.p95, m.p99); - if (m.ok) n_pass++; - else n_fail++; - } - - fprintf(stderr, "\n══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n"); - fprintf(stderr, " TOTAL: %d PASS, %d FAIL out of %d tensors\n", n_pass, n_fail, (int) report.size()); - fprintf(stderr, "══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════\n\n"); - - state.reset(); - sam3_free_model(*model); - model.reset(); - - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/test_full_pipeline.py b/tests/test_full_pipeline.py deleted file mode 100644 index e9af497..0000000 --- a/tests/test_full_pipeline.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -"""Compare the full pipeline: C++ encode_image (with C++ preprocessing) -vs Python preprocessing → Python ViT. - -If the C++ bilinear resize matches PIL closely enough, the ViT outputs -should also be close. -""" -import numpy as np -import sys, os - -def load_tensor(path): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - data = np.fromfile(path + ".bin", dtype=np.float32) - return data, shape - -def compare(name, ref, cpp, ref_shape, cpp_shape): - """Compare flat arrays, handling NCHW vs ggml [C, W, H] transposition.""" - # If shapes suggest NCHW vs ggml, transpose - if len(cpp_shape) == 3 and len(ref_shape) == 4: - C, W, H = cpp_shape - # ggml [C, W, H] flat: c + w*C + h*C*W - # NCHW flat: c*H*W + h*W + w - cpp_transposed = np.zeros(C * W * H, dtype=np.float32) - cpp_3d = cpp.reshape(H, W, C) # ggml column-major - # Transpose to [C, H, W] for NCHW comparison - cpp_nchw = cpp_3d.transpose(2, 0, 1).flatten() - cpp = cpp_nchw - ref = ref[:len(cpp)] - - diff = np.abs(ref - cpp) - max_d = diff.max() - mean_d = diff.mean() - cos = np.dot(ref, cpp) / (np.linalg.norm(ref) * np.linalg.norm(cpp) + 1e-12) - print(f" {name:40s} max={max_d:.6e} mean={mean_d:.6e} cos={cos:.8f}") - - -if __name__ == "__main__": - ref_dir = sys.argv[1] if len(sys.argv) > 1 else "tests/ref_phase3" - cpp_dir = sys.argv[2] if len(sys.argv) > 2 else "tests/cpp_out_full" - - if not os.path.exists(cpp_dir): - print(f"C++ output dir '{cpp_dir}' not found") - print("Run: ./build/tests/test_debug_encoder --full") - sys.exit(1) - - print("═══ Full Pipeline Comparison (C++ preprocess + ViT vs Python preprocess + ViT) ═══\n") - - for name in ["vit_output", "neck_det_0", "neck_det_1", "neck_det_2", "neck_det_3"]: - ref_path = ref_dir + "/" + ("vit_output_bchw" if name == "vit_output" else name) - cpp_path = cpp_dir + "/" + name - if not os.path.exists(ref_path + ".bin") or not os.path.exists(cpp_path + ".bin"): - print(f" [SKIP] {name}") - continue - ref, ref_shape = load_tensor(ref_path) - cpp, cpp_shape = load_tensor(cpp_path) - compare(name, ref, cpp, ref_shape, cpp_shape) diff --git a/tests/test_geom_enc.cpp b/tests/test_geom_enc.cpp deleted file mode 100644 index 478d78e..0000000 --- a/tests/test_geom_enc.cpp +++ /dev/null @@ -1,142 +0,0 @@ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include - -struct geom_case { - std::string id; - sam3_pcs_params params; -}; - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string prephase_ref_dir = argv[2]; - const std::string model_path = argv[3]; - const std::string cpp_root = ref_dir + "/cpp_out_geom"; - - // Test cases matching the Python dump script - std::vector cases; - - // Case 1: dummy prompt (no exemplars, just CLS) - { - geom_case c; - c.id = "dummy_prompt"; - // No exemplars - cases.push_back(std::move(c)); - } - - // Case 2: single positive box (cx=0.5, cy=0.5, w=0.3, h=0.3) - // API uses XYXY format — convert from CxCyWH to XYXY - { - geom_case c; - c.id = "single_box"; - float cx = 0.5f, cy = 0.5f, w = 0.3f, h = 0.3f; - sam3_box box; - box.x0 = cx - w * 0.5f; // 0.35 - box.y0 = cy - h * 0.5f; // 0.35 - box.x1 = cx + w * 0.5f; // 0.65 - box.y1 = cy + h * 0.5f; // 0.65 - c.params.pos_exemplars.push_back(box); - cases.push_back(std::move(c)); - } - - // Case 3: two boxes (one positive, one negative) - // Box1: cx=0.3, cy=0.4, w=0.2, h=0.25 (positive) - // Box2: cx=0.7, cy=0.6, w=0.15, h=0.2 (negative) - { - geom_case c; - c.id = "two_boxes"; - { - float cx = 0.3f, cy = 0.4f, w = 0.2f, h = 0.25f; - sam3_box box = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f}; - c.params.pos_exemplars.push_back(box); - } - { - float cx = 0.7f, cy = 0.6f, w = 0.15f, h = 0.2f; - sam3_box box = {cx - w * 0.5f, cy - h * 0.5f, cx + w * 0.5f, cy + h * 0.5f}; - c.params.neg_exemplars.push_back(box); - } - cases.push_back(std::move(c)); - } - - sam3_params mparams; - mparams.model_path = model_path; - mparams.n_threads = 1; - mparams.use_gpu = false; - - auto model = sam3_load_model(mparams); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - bool overall_ok = true; - float atol = 1e-4f; - - for (const auto& tc : cases) { - fprintf(stderr, "\n=== Geometry Encoder: %s ===\n", tc.id.c_str()); - - const std::string case_ref_dir = ref_dir + "/" + tc.id; - const std::string case_cpp_dir = cpp_root + "/" + tc.id; - ensure_dir(cpp_root); - ensure_dir(case_cpp_dir); - - if (!sam3_test_dump_geom_enc(*model, prephase_ref_dir, tc.params, - case_cpp_dir, mparams.n_threads)) { - fprintf(stderr, " [FAIL] dump failed for %s\n", tc.id.c_str()); - overall_ok = false; - continue; - } - - // Compare tensors - struct tensor_check { - std::string name; - float tol; - }; - std::vector checks = { - // post_final_proj is an input tensor whose buffer gets reused by the graph - // allocator during computation — comparing it after graph compute gives stale - // data. Only geom_output (the final transformer output) is meaningful. - {"geom_output", 5e-4f}, - }; - - for (const auto& chk : checks) { - auto ref = load_ref_f32(case_ref_dir + "/" + chk.name); - auto got = load_ref_f32(case_cpp_dir + "/" + chk.name); - - if (ref.data.empty() || got.data.empty()) { - fprintf(stderr, " [FAIL] %s: missing tensor\n", chk.name.c_str()); - overall_ok = false; - continue; - } - if (ref.numel() != got.numel()) { - fprintf(stderr, " [FAIL] %s: numel mismatch ref=%d got=%d\n", - chk.name.c_str(), ref.numel(), got.numel()); - overall_ok = false; - continue; - } - - auto r = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), chk.tol); - bool ok = r.max_diff <= chk.tol; - fprintf(stderr, " [%s] %-30s max=%.3e mean=%.3e cos=%.8f tol=%.1e", - ok ? "PASS" : "FAIL", chk.name.c_str(), - r.max_diff, r.mean_diff, r.cosine_sim, chk.tol); - if (!ok) { - fprintf(stderr, " worst=%d got=%.6g ref=%.6g", - r.worst_index, r.worst_a, r.worst_b); - overall_ok = false; - } - fprintf(stderr, "\n"); - } - } - - fprintf(stderr, "\n%s\n", overall_ok ? "ALL PASS" : "SOME FAILED"); - return overall_ok ? 0 : 1; -} diff --git a/tests/test_jpeg_decode.cpp b/tests/test_jpeg_decode.cpp deleted file mode 100644 index b60d221..0000000 --- a/tests/test_jpeg_decode.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Compare stb_image JPEG decoding against PIL/libjpeg. -// Usage: ./test_jpeg_decode -#include "sam3.h" -#include "test_utils.h" -#include -#include -#include - -int main(int argc, char** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - auto img = sam3_load_image(argv[1]); - if (img.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - fprintf(stderr, "stb_image decoded: %dx%d, %d bytes\n", - img.width, img.height, (int)img.data.size()); - - // Load PIL reference - std::ifstream f(argv[2], std::ios::binary); - if (!f) { fprintf(stderr, "Failed to load PIL ref\n"); return 1; } - std::vector pil_data(img.data.size()); - f.read(reinterpret_cast(pil_data.data()), pil_data.size()); - fprintf(stderr, "PIL decoded: %d bytes\n", (int)pil_data.size()); - - // Compare - int n_diff = 0, max_diff = 0; - for (size_t i = 0; i < img.data.size(); ++i) { - int d = abs((int)img.data[i] - (int)pil_data[i]); - if (d > 0) n_diff++; - if (d > max_diff) max_diff = d; - } - fprintf(stderr, "\n═══ JPEG Decoder Comparison ═══\n"); - fprintf(stderr, " max pixel diff: %d\n", max_diff); - fprintf(stderr, " pixels differing: %d / %d (%.2f%%)\n", - n_diff, (int)img.data.size(), 100.0 * n_diff / img.data.size()); - - // Check specific pixels - int y=16, x=517, c=2; - int idx = (y * img.width + x) * 3 + c; - fprintf(stderr, "\n stb pixel at (%d,%d,%d): %d\n", y, x, c, img.data[idx]); - fprintf(stderr, " PIL pixel at (%d,%d,%d): %d\n", y, x, c, pil_data[idx]); - - return max_diff > 0 ? 1 : 0; -} diff --git a/tests/test_llama_pcs.cpp b/tests/test_llama_pcs.cpp deleted file mode 100644 index 1f1cb90..0000000 --- a/tests/test_llama_pcs.cpp +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Quick test: run PCS on the llama image with various text prompts. - * Uses sam3_encode_image (same as main_image example) — NOT preprocessed. - */ -#include "sam3.h" -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [prompt]\n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string image_path = argv[2]; - const std::string prompt = argc > 3 ? argv[3] : "llama"; - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - fprintf(stderr, "Loading model...\n"); - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - fprintf(stderr, "Loading image: %s\n", image_path.c_str()); - auto image = sam3_load_image(image_path); - if (image.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - fprintf(stderr, "Image: %dx%d\n", image.width, image.height); - - fprintf(stderr, "Encoding image...\n"); - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "Failed to encode image\n"); return 1; - } - - // Test multiple prompts - const char* prompts[] = {"llama", "guitar", "animal"}; - int n_prompts = 3; - if (argc > 4) { - prompts[0] = prompt.c_str(); - n_prompts = 1; - } - - for (int p = 0; p < n_prompts; ++p) { - fprintf(stderr, "\n═══ Testing prompt: '%s' ═══\n", prompts[p]); - - sam3_pcs_params pcs; - pcs.text_prompt = prompts[p]; - pcs.score_threshold = 0.3f; - pcs.nms_threshold = 0.1f; - - auto result = sam3_segment_pcs(*state, *model, pcs); - fprintf(stderr, "Result: %zu detections\n", result.detections.size()); - - for (size_t i = 0; i < result.detections.size() && i < 5; ++i) { - const auto& d = result.detections[i]; - fprintf(stderr, " det %zu: score=%.4f box=[%.1f,%.1f,%.1f,%.1f]\n", - i, d.score, d.box.x0, d.box.y0, d.box.x1, d.box.y1); - } - - // Save first mask - if (!result.detections.empty()) { - std::string mask_path = "/tmp/llama_mask_" + std::string(prompts[p]) + ".png"; - sam3_save_mask(result.detections[0].mask, mask_path); - fprintf(stderr, " Saved mask: %s\n", mask_path.c_str()); - } - } - - sam3_free_model(*model); - return 0; -} diff --git a/tests/test_load.cpp b/tests/test_load.cpp deleted file mode 100644 index f9d0b04..0000000 --- a/tests/test_load.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "sam3.h" -#include - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - sam3_params params; - params.model_path = argv[1]; - params.use_gpu = true; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - fprintf(stderr, "Model loaded successfully!\n"); - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state\n"); - return 1; - } - - fprintf(stderr, "State created successfully!\n"); - - // Explicit cleanup before ggml global shutdown - state.reset(); - sam3_free_model(*model); - model.reset(); - - fprintf(stderr, "Cleanup complete.\n"); - return 0; -} diff --git a/tests/test_mem_enc.cpp b/tests/test_mem_enc.cpp deleted file mode 100644 index 2d35ea3..0000000 --- a/tests/test_mem_enc.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Memory encoder comparison test. - * Encodes the llama image, runs PVS with a point, then dumps the - * memory encoder outputs for comparison against Python reference. - * - * Usage: test_mem_enc [x y] - */ -#include "sam3.h" -#include "test_utils.h" -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [x y]\n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = std::string(argv[2]) + "/video_ref"; - float px = argc > 4 ? atof(argv[3]) : 315.0f; - float py = argc > 4 ? atof(argv[4]) : 250.0f; - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - // Load llama image - auto image = sam3_load_image("../ggml/examples/sam/example.jpg"); - if (image.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - - // Encode image - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "Failed to encode image\n"); return 1; - } - - // Create tracker - sam3_video_params vp; - vp.hotstart_delay = 0; - auto tracker = sam3_create_tracker(*model, vp); - - // Add instance with point prompt → triggers PVS + memory encoding - sam3_pvs_params pvs; - pvs.pos_points.push_back({px, py}); - pvs.multimask = false; - - int inst_id = sam3_tracker_add_instance(*tracker, *state, *model, pvs); - fprintf(stderr, "Added instance %d\n", inst_id); - - if (inst_id < 0) { - fprintf(stderr, "Failed to add instance\n"); - return 1; - } - - // ── Compare memory encoder outputs against Python reference ────── - fprintf(stderr, "\n═══ Memory Encoder Comparison ═══\n"); - - // Note: We're comparing C++ (stb_image decoded) vs Python (PIL decoded). - // The images differ slightly, so we expect some divergence. - // We check that the outputs are in the same range and structure. - - // Check that Python reference exists - auto py_mem_feat = load_ref_f32(ref_dir + "/mem_enc_output_features"); - auto py_mem_pe = load_ref_f32(ref_dir + "/mem_enc_output_pe_0"); - auto py_obj_ptr = load_ref_f32(ref_dir + "/obj_ptr_output"); - - fprintf(stderr, "Python reference:\n"); - if (!py_mem_feat.data.empty()) - fprintf(stderr, " mem_enc_output: numel=%d\n", py_mem_feat.numel()); - if (!py_mem_pe.data.empty()) - fprintf(stderr, " mem_enc_pe: numel=%d\n", py_mem_pe.numel()); - if (!py_obj_ptr.data.empty()) - fprintf(stderr, " obj_ptr: numel=%d\n", py_obj_ptr.numel()); - - // The C++ memory encoder stored its output in the tracker's memory bank. - // We can't easily extract it from the tracker. But we can verify the - // pipeline ran correctly by checking the tracker state. - fprintf(stderr, "\nC++ tracker state:\n"); - fprintf(stderr, " frame_index: %d\n", sam3_tracker_frame_index(*tracker)); - - // Dump Python reference ranges for comparison - if (!py_mem_feat.data.empty()) { - float mn = py_mem_feat.data[0], mx = py_mem_feat.data[0]; - for (float v : py_mem_feat.data) { mn = std::min(mn, v); mx = std::max(mx, v); } - fprintf(stderr, " Python mem_enc_features range: [%.4f, %.4f]\n", mn, mx); - } - - if (!py_obj_ptr.data.empty()) { - float mn = py_obj_ptr.data[0], mx = py_obj_ptr.data[0]; - for (float v : py_obj_ptr.data) { mn = std::min(mn, v); mx = std::max(mx, v); } - fprintf(stderr, " Python obj_ptr range: [%.4f, %.4f]\n", mn, mx); - fprintf(stderr, " Python obj_ptr[0..4]: [%.4f, %.4f, %.4f, %.4f, %.4f]\n", - py_obj_ptr.data[0], py_obj_ptr.data[1], py_obj_ptr.data[2], - py_obj_ptr.data[3], py_obj_ptr.data[4]); - } - - fprintf(stderr, "\n═══ Result: Memory encoder reference data available ═══\n"); - fprintf(stderr, "For proper numerical comparison, use sam3_encode_image_from_preprocessed()\n"); - fprintf(stderr, "with the Python-exported preprocessed image to eliminate JPEG decoder differences.\n"); - - sam3_free_model(*model); - return 0; -} diff --git a/tests/test_metal_block0_stage_compare.cpp b/tests/test_metal_block0_stage_compare.cpp deleted file mode 100644 index a8a9c9a..0000000 --- a/tests/test_metal_block0_stage_compare.cpp +++ /dev/null @@ -1,398 +0,0 @@ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -struct dump_item { - const char * label; - const char * name; -}; - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static std::vector trim_shape(const int64_t ne[4]) { - std::vector shape = { - (int) ne[0], - (int) ne[1], - (int) ne[2], - (int) ne[3], - }; - while (!shape.empty() && shape.back() == 1) { - shape.pop_back(); - } - if (shape.empty()) { - shape.push_back(1); - } - return shape; -} - -static bool dump_named_tensors(const sam3_state & state, - const std::vector & items, - const std::string & output_dir) { - for (const auto & item : items) { - if (!sam3_dump_state_tensor(state, item.name, output_dir + "/" + item.name)) { - fprintf(stderr, "failed to dump %s\n", item.name); - return false; - } - } - return true; -} - -static bool run_selective_vit(sam3_state & state, - const sam3_model & model, - const std::vector & chw_data, - int img_size, - const std::vector & items, - const std::string & output_dir) { - std::vector names; - names.reserve(items.size()); - for (const auto & item : items) { - names.push_back(item.name); - } - - if (!sam3_encode_vit_from_preprocessed_selective(state, model, chw_data.data(), img_size, names)) { - return false; - } - return dump_named_tensors(state, items, output_dir); -} - -static ref_tensor_f32 run_stage_or_die(const sam3_model & model, - sam3_vit_block_stage stage, - const ref_tensor_f32 & input, - int n_threads) { - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < input.shape.size() && i < 4; ++i) { - in_ne[i] = input.shape[i]; - } - - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block_stage(model, 0, stage, input.data.data(), in_ne, out.data, out_ne, n_threads)) { - fprintf(stderr, "stage %d failed\n", (int) stage); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 run_linear_host_ref_or_die(const sam3_model & model, - sam3_vit_block_stage stage, - const ref_tensor_f32 & input, - bool use_double_accum) { - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < input.shape.size() && i < 4; ++i) { - in_ne[i] = input.shape[i]; - } - - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block_linear_host_ref(model, 0, stage, input.data.data(), in_ne, use_double_accum, out.data, out_ne)) { - fprintf(stderr, "host linear ref stage %d failed\n", (int) stage); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 run_block0_input_or_die(const sam3_model & model, - const std::vector & chw_data, - int img_size, - int n_threads) { - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block0_input(model, chw_data.data(), img_size, out.data, out_ne, n_threads)) { - fprintf(stderr, "block0 input prefix failed\n"); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 add_ref(const ref_tensor_f32 & a, const ref_tensor_f32 & b) { - if (a.shape != b.shape || a.data.size() != b.data.size()) { - fprintf(stderr, "add_ref: shape mismatch\n"); - std::exit(1); - } - ref_tensor_f32 out; - out.shape = a.shape; - out.data.resize(a.data.size()); - for (size_t i = 0; i < a.data.size(); ++i) { - out.data[i] = a.data[i] + b.data[i]; - } - return out; -} - -static void print_diff_row(const char * label, - const ref_tensor_f32 & a, - const ref_tensor_f32 & b, - float atol = 1e-4f) { - const compare_result r = compare_tensors(a.data.data(), b.data.data(), a.numel(), atol); - fprintf(stderr, "%-22s %-18s %12.6f %12.6f %10d\n", - label, - format_shape(a.shape).c_str(), - r.max_diff, - r.mean_diff, - r.n_bad); -} - -static void print_graph_row(const dump_item & item, - const std::string & cpu_dir, - const std::string & metal_dir) { - const ref_tensor_f32 cpu = load_ref_f32(cpu_dir + "/" + item.name); - const ref_tensor_f32 metal = load_ref_f32(metal_dir + "/" + item.name); - if (cpu.data.empty() || metal.data.empty()) { - fprintf(stderr, "%-22s %-18s %12s %12s %10s\n", - item.label, "-", "load-fail", "load-fail", "-"); - return; - } - print_diff_row(item.label, cpu, metal); -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [ref_dir]\n", argv[0]); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - - const std::vector graph_items = { - { "block_input", "dbg_after_ln_pre" }, - { "qkv_proj_output", "dbg_block_0_qkv_proj"}, - { "attn_proj_output", "dbg_block_0_attn_proj"}, - { "resid1_output", "dbg_block_0_resid1" }, - { "mlp_output", "dbg_block_0_mlp" }, - { "block_output", "dbg_block_0_out" }, - }; - - const std::string cpu_dir = "/tmp/sam3_block0_stage_cpu"; - const std::string metal_dir = "/tmp/sam3_block0_stage_metal"; - ensure_dir(cpu_dir); - ensure_dir(metal_dir); - - const ref_tensor_f32 preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty() || preprocessed.shape.size() != 4) { - fprintf(stderr, "failed to load %s/preprocessed\n", ref_dir.c_str()); - return 1; - } - const int img_size = preprocessed.shape[2]; - const int n_threads = 8; - - sam3_params cpu_params; - cpu_params.model_path = model_path; - cpu_params.use_gpu = false; - cpu_params.n_threads = n_threads; - - sam3_params metal_params = cpu_params; - metal_params.use_gpu = true; - - auto cpu_model = sam3_load_model(cpu_params); - auto metal_model = sam3_load_model(metal_params); - if (!cpu_model || !metal_model) { - fprintf(stderr, "failed to load CPU or Metal model\n"); - return 1; - } - - auto cpu_state = sam3_create_state(*cpu_model, cpu_params); - auto metal_state = sam3_create_state(*metal_model, metal_params); - if (!cpu_state || !metal_state) { - fprintf(stderr, "failed to create CPU or Metal state\n"); - return 1; - } - - if (!run_selective_vit(*cpu_state, *cpu_model, preprocessed.data, img_size, graph_items, cpu_dir)) { - fprintf(stderr, "CPU selective block-0 run failed\n"); - return 1; - } - if (!run_selective_vit(*metal_state, *metal_model, preprocessed.data, img_size, graph_items, metal_dir)) { - fprintf(stderr, "Metal selective block-0 run failed\n"); - return 1; - } - - fprintf(stderr, "\n=== Block-0 graph outputs: CPU vs Metal ===\n"); - fprintf(stderr, "%-22s %-18s %12s %12s %10s\n", - "checkpoint", "shape", "max_abs_diff", "mean_abs", "n_bad"); - for (const auto & item : graph_items) { - print_graph_row(item, cpu_dir, metal_dir); - } - - const ref_tensor_f32 cpu_block_input = run_block0_input_or_die(*cpu_model, preprocessed.data, img_size, n_threads); - const ref_tensor_f32 metal_block_input = run_block0_input_or_die(*metal_model, preprocessed.data, img_size, n_threads); - - sam3_tensor_info qkv_info = {}; - sam3_tensor_info proj_info = {}; - sam3_tensor_info mlp1_info = {}; - sam3_tensor_info rope_info = {}; - sam3_get_model_tensor_info(*cpu_model, "vit.blocks.0.attn.qkv.weight", qkv_info); - sam3_get_model_tensor_info(*cpu_model, "vit.blocks.0.attn.proj.weight", proj_info); - sam3_get_model_tensor_info(*cpu_model, "vit.blocks.0.mlp.lin1.weight", mlp1_info); - sam3_get_model_tensor_info(*cpu_model, "vit.blocks.0.attn.freqs_cis", rope_info); - - fprintf(stderr, "\n=== Exact block-0 case ===\n"); - fprintf(stderr, "block input shape: %s\n", format_shape(cpu_block_input.shape).c_str()); - fprintf(stderr, "window attention: full_spatial=72x72 window=24 heads=%d head_dim=%d\n", 16, 64); - fprintf(stderr, "qkv weight: shape=[%lld,%lld] type=%d\n", - (long long) qkv_info.ne[0], (long long) qkv_info.ne[1], qkv_info.type); - fprintf(stderr, "proj weight: shape=[%lld,%lld] type=%d\n", - (long long) proj_info.ne[0], (long long) proj_info.ne[1], proj_info.type); - fprintf(stderr, "mlp fc1: shape=[%lld,%lld] type=%d\n", - (long long) mlp1_info.ne[0], (long long) mlp1_info.ne[1], mlp1_info.type); - fprintf(stderr, "freqs_cis: shape=[%lld,%lld,%lld] type=%d\n", - (long long) rope_info.ne[0], (long long) rope_info.ne[1], (long long) rope_info.ne[2], rope_info.type); - print_diff_row("block0_input_exact", cpu_block_input, metal_block_input); - - const ref_tensor_f32 cpu_norm1_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_NORM1, cpu_block_input, n_threads); - const ref_tensor_f32 metal_norm1_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_NORM1, metal_block_input, n_threads); - const ref_tensor_f32 cpu_win_part_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_WINDOW_PART, cpu_norm1_self, n_threads); - const ref_tensor_f32 metal_win_part_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_WINDOW_PART, metal_norm1_self, n_threads); - const ref_tensor_f32 cpu_qkv_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, cpu_win_part_self, n_threads); - const ref_tensor_f32 metal_qkv_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, metal_win_part_self, n_threads); - const ref_tensor_f32 cpu_attn_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_ATTN_CORE, cpu_qkv_self, n_threads); - const ref_tensor_f32 metal_attn_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_ATTN_CORE, metal_qkv_self, n_threads); - const ref_tensor_f32 cpu_attn_proj_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, cpu_attn_self, n_threads); - const ref_tensor_f32 metal_attn_proj_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, metal_attn_self, n_threads); - const ref_tensor_f32 cpu_unpart_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, cpu_attn_proj_self, n_threads); - const ref_tensor_f32 metal_unpart_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, metal_attn_proj_self, n_threads); - const ref_tensor_f32 cpu_resid1_self = add_ref(cpu_block_input, cpu_unpart_self); - const ref_tensor_f32 metal_resid1_self = add_ref(metal_block_input, metal_unpart_self); - const ref_tensor_f32 cpu_norm2_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_NORM2, cpu_resid1_self, n_threads); - const ref_tensor_f32 metal_norm2_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_NORM2, metal_resid1_self, n_threads); - const ref_tensor_f32 cpu_mlp_self = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP, cpu_norm2_self, n_threads); - const ref_tensor_f32 metal_mlp_self = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_MLP, metal_norm2_self, n_threads); - const ref_tensor_f32 cpu_block_self = add_ref(cpu_resid1_self, cpu_mlp_self); - const ref_tensor_f32 metal_block_self = add_ref(metal_resid1_self, metal_mlp_self); - - const ref_tensor_f32 cpu_norm1_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_NORM1, cpu_block_input, n_threads); - const ref_tensor_f32 metal_norm1_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_NORM1, cpu_block_input, n_threads); - const ref_tensor_f32 cpu_win_part_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_WINDOW_PART, cpu_norm1_shared, n_threads); - const ref_tensor_f32 metal_win_part_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_WINDOW_PART, cpu_norm1_shared, n_threads); - const ref_tensor_f32 cpu_qkv_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, cpu_win_part_shared, n_threads); - const ref_tensor_f32 metal_qkv_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, cpu_win_part_shared, n_threads); - const ref_tensor_f32 cpu_attn_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_ATTN_CORE, cpu_qkv_shared, n_threads); - const ref_tensor_f32 metal_attn_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_ATTN_CORE, cpu_qkv_shared, n_threads); - const ref_tensor_f32 cpu_attn_proj_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, cpu_attn_shared, n_threads); - const ref_tensor_f32 metal_attn_proj_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, cpu_attn_shared, n_threads); - const ref_tensor_f32 cpu_unpart_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, cpu_attn_proj_shared, n_threads); - const ref_tensor_f32 metal_unpart_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, cpu_attn_proj_shared, n_threads); - const ref_tensor_f32 cpu_resid1_shared = add_ref(cpu_block_input, cpu_unpart_shared); - const ref_tensor_f32 metal_resid1_shared = add_ref(cpu_block_input, metal_unpart_shared); - const ref_tensor_f32 cpu_norm2_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_NORM2, cpu_resid1_shared, n_threads); - const ref_tensor_f32 metal_norm2_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_NORM2, cpu_resid1_shared, n_threads); - const ref_tensor_f32 cpu_mlp_fc1_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP_FC1, cpu_norm2_shared, n_threads); - const ref_tensor_f32 metal_mlp_fc1_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_MLP_FC1, cpu_norm2_shared, n_threads); - const ref_tensor_f32 cpu_mlp_gelu_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP_GELU, cpu_mlp_fc1_shared, n_threads); - const ref_tensor_f32 metal_mlp_gelu_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_MLP_GELU, cpu_mlp_fc1_shared, n_threads); - const ref_tensor_f32 cpu_mlp_fc2_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP_FC2, cpu_mlp_gelu_shared, n_threads); - const ref_tensor_f32 metal_mlp_fc2_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_MLP_FC2, cpu_mlp_gelu_shared, n_threads); - const ref_tensor_f32 cpu_mlp_shared = run_stage_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP, cpu_norm2_shared, n_threads); - const ref_tensor_f32 metal_mlp_shared = run_stage_or_die(*metal_model, SAM3_VIT_BLOCK_STAGE_MLP, cpu_norm2_shared, n_threads); - const ref_tensor_f32 cpu_block_shared = add_ref(cpu_resid1_shared, cpu_mlp_shared); - const ref_tensor_f32 metal_block_shared = add_ref(cpu_resid1_shared, metal_mlp_shared); - - const std::map cpu_graph = { - {"qkv_proj_output", load_ref_f32(cpu_dir + "/dbg_block_0_qkv_proj")}, - {"attn_proj_output", load_ref_f32(cpu_dir + "/dbg_block_0_attn_proj")}, - {"resid1_output", load_ref_f32(cpu_dir + "/dbg_block_0_resid1")}, - {"mlp_output", load_ref_f32(cpu_dir + "/dbg_block_0_mlp")}, - {"block_output", load_ref_f32(cpu_dir + "/dbg_block_0_out")}, - }; - const std::map metal_graph = { - {"qkv_proj_output", load_ref_f32(metal_dir + "/dbg_block_0_qkv_proj")}, - {"attn_proj_output", load_ref_f32(metal_dir + "/dbg_block_0_attn_proj")}, - {"resid1_output", load_ref_f32(metal_dir + "/dbg_block_0_resid1")}, - {"mlp_output", load_ref_f32(metal_dir + "/dbg_block_0_mlp")}, - {"block_output", load_ref_f32(metal_dir + "/dbg_block_0_out")}, - }; - - fprintf(stderr, "\n=== Exact isolated stages on shared canonical CPU inputs ===\n"); - fprintf(stderr, "%-22s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("norm1_output", cpu_norm1_shared, metal_norm1_shared); - print_diff_row("win_part_output", cpu_win_part_shared, metal_win_part_shared); - print_diff_row("qkv_proj_output", cpu_qkv_shared, metal_qkv_shared); - print_diff_row("attn_core_output", cpu_attn_shared, metal_attn_shared); - print_diff_row("attn_proj_output", cpu_attn_proj_shared, metal_attn_proj_shared); - print_diff_row("win_unpart_output", cpu_unpart_shared, metal_unpart_shared); - print_diff_row("resid1_output", cpu_resid1_shared, metal_resid1_shared); - print_diff_row("norm2_output", cpu_norm2_shared, metal_norm2_shared); - print_diff_row("mlp_output", cpu_mlp_shared, metal_mlp_shared); - print_diff_row("block_output", cpu_block_shared, metal_block_shared); - - const ref_tensor_f32 host_qkv_f64 = run_linear_host_ref_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, cpu_win_part_shared, true); - const ref_tensor_f32 host_attn_proj_f64 = run_linear_host_ref_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, cpu_attn_shared, true); - const ref_tensor_f32 host_mlp_fc1_f64 = run_linear_host_ref_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP_FC1, cpu_norm2_shared, true); - const ref_tensor_f32 host_mlp_fc2_f64 = run_linear_host_ref_or_die(*cpu_model, SAM3_VIT_BLOCK_STAGE_MLP_FC2, cpu_mlp_gelu_shared, true); - - fprintf(stderr, "\n=== Block-0 host linear refs (double accumulation) ===\n"); - fprintf(stderr, "%-22s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("qkv_cpu_vs_host", cpu_qkv_shared, host_qkv_f64); - print_diff_row("qkv_mtl_vs_host", metal_qkv_shared, host_qkv_f64); - print_diff_row("attn_proj_cpu_vs_host", cpu_attn_proj_shared, host_attn_proj_f64); - print_diff_row("attn_proj_mtl_vs_host", metal_attn_proj_shared, host_attn_proj_f64); - print_diff_row("mlp_fc1_cpu_vs_host", cpu_mlp_fc1_shared, host_mlp_fc1_f64); - print_diff_row("mlp_fc1_mtl_vs_host", metal_mlp_fc1_shared, host_mlp_fc1_f64); - print_diff_row("mlp_fc2_cpu_vs_host", cpu_mlp_fc2_shared, host_mlp_fc2_f64); - print_diff_row("mlp_fc2_mtl_vs_host", metal_mlp_fc2_shared, host_mlp_fc2_f64); - - fprintf(stderr, "\n=== Block-0 MLP split on shared canonical CPU inputs ===\n"); - fprintf(stderr, "%-22s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("mlp_fc1_output", cpu_mlp_fc1_shared, metal_mlp_fc1_shared); - print_diff_row("mlp_gelu_output", cpu_mlp_gelu_shared, metal_mlp_gelu_shared); - print_diff_row("mlp_fc2_output", cpu_mlp_fc2_shared, metal_mlp_fc2_shared); - - fprintf(stderr, "\n=== Isolated stage self-consistency vs graph outputs ===\n"); - fprintf(stderr, "%-22s %-18s %-24s %-24s\n", - "stage", "shape", "cpu_iso_vs_cpu_graph", "metal_iso_vs_metal_graph"); - - auto print_self_row = [&](const char * label, - const ref_tensor_f32 & cpu_iso, - const ref_tensor_f32 & metal_iso, - const char * graph_key) { - const auto cpu_it = cpu_graph.find(graph_key); - const auto metal_it = metal_graph.find(graph_key); - if (cpu_it == cpu_graph.end() || metal_it == metal_graph.end()) { - fprintf(stderr, "%-22s %-18s %-24s %-24s\n", - label, "-", "missing", "missing"); - return; - } - - const compare_result cpu_r = compare_tensors(cpu_iso.data.data(), cpu_it->second.data.data(), cpu_iso.numel(), 1e-4f); - const compare_result metal_r = compare_tensors(metal_iso.data.data(), metal_it->second.data.data(), metal_iso.numel(), 1e-4f); - - char cpu_buf[64]; - char metal_buf[64]; - snprintf(cpu_buf, sizeof(cpu_buf), "max=%.6f mean=%.6f", cpu_r.max_diff, cpu_r.mean_diff); - snprintf(metal_buf, sizeof(metal_buf), "max=%.6f mean=%.6f", metal_r.max_diff, metal_r.mean_diff); - fprintf(stderr, "%-22s %-18s %-24s %-24s\n", - label, - format_shape(cpu_iso.shape).c_str(), - cpu_buf, - metal_buf); - }; - - print_self_row("qkv_proj_output", cpu_qkv_self, metal_qkv_self, "qkv_proj_output"); - print_self_row("attn_proj_output", cpu_attn_proj_self, metal_attn_proj_self, "attn_proj_output"); - print_self_row("resid1_output", cpu_resid1_self, metal_resid1_self, "resid1_output"); - print_self_row("mlp_output", cpu_mlp_self, metal_mlp_self, "mlp_output"); - print_self_row("block_output", cpu_block_self, metal_block_self, "block_output"); - - return 0; -} diff --git a/tests/test_metal_block_stage_compare.cpp b/tests/test_metal_block_stage_compare.cpp deleted file mode 100644 index 28c486e..0000000 --- a/tests/test_metal_block_stage_compare.cpp +++ /dev/null @@ -1,535 +0,0 @@ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -struct dump_item { - const char * label; - std::string name; -}; - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static std::vector trim_shape(const int64_t ne[4]) { - std::vector shape = { - (int) ne[0], - (int) ne[1], - (int) ne[2], - (int) ne[3], - }; - while (!shape.empty() && shape.back() == 1) { - shape.pop_back(); - } - if (shape.empty()) { - shape.push_back(1); - } - return shape; -} - -static std::string make_block_name(int block_idx, const char * suffix) { - char buf[64]; - snprintf(buf, sizeof(buf), "dbg_block_%d_%s", block_idx, suffix); - return buf; -} - -static bool dump_named_tensors(const sam3_state & state, - const std::vector & items, - const std::string & output_dir) { - for (const auto & item : items) { - if (!sam3_dump_state_tensor(state, item.name, output_dir + "/" + item.name)) { - fprintf(stderr, "failed to dump %s\n", item.name.c_str()); - return false; - } - } - return true; -} - -static bool run_selective_vit(sam3_state & state, - const sam3_model & model, - const std::vector & chw_data, - int img_size, - int debug_block_idx, - const std::vector & items, - const std::string & output_dir) { - std::vector names; - names.reserve(items.size()); - for (const auto & item : items) { - names.push_back(item.name); - } - - if (!sam3_encode_vit_from_preprocessed_selective(state, model, chw_data.data(), img_size, names)) { - return false; - } - return dump_named_tensors(state, items, output_dir); -} - -static ref_tensor_f32 run_stage_or_die(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const ref_tensor_f32 & input, - int n_threads) { - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < input.shape.size() && i < 4; ++i) { - in_ne[i] = input.shape[i]; - } - - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block_stage(model, block_idx, stage, input.data.data(), in_ne, out.data, out_ne, n_threads)) { - fprintf(stderr, "stage %d for block %d failed\n", (int) stage, block_idx); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 run_linear_host_ref_or_die(const sam3_model & model, - int block_idx, - sam3_vit_block_stage stage, - const ref_tensor_f32 & input, - bool use_double_accum) { - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < input.shape.size() && i < 4; ++i) { - in_ne[i] = input.shape[i]; - } - - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block_linear_host_ref( - model, block_idx, stage, input.data.data(), in_ne, use_double_accum, out.data, out_ne)) { - fprintf(stderr, "host linear ref stage %d for block %d failed\n", (int) stage, block_idx); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 add_ref(const ref_tensor_f32 & a, const ref_tensor_f32 & b) { - if (a.shape != b.shape || a.data.size() != b.data.size()) { - fprintf(stderr, "add_ref: shape mismatch\n"); - std::exit(1); - } - ref_tensor_f32 out; - out.shape = a.shape; - out.data.resize(a.data.size()); - for (size_t i = 0; i < a.data.size(); ++i) { - out.data[i] = a.data[i] + b.data[i]; - } - return out; -} - -static ref_tensor_f32 slice_first_columns(const ref_tensor_f32 & t, int n_cols) { - if (t.shape.empty()) { - fprintf(stderr, "slice_first_columns: empty tensor\n"); - std::exit(1); - } - - const int ne0 = t.shape[0]; - const int total_cols = (int) t.data.size() / ne0; - if (total_cols <= 0 || ne0 * total_cols != (int) t.data.size()) { - fprintf(stderr, "slice_first_columns: invalid shape/data size\n"); - std::exit(1); - } - - const int cols = n_cols < total_cols ? n_cols : total_cols; - - ref_tensor_f32 out; - out.shape = {ne0, cols, 1, 1}; - out.data.resize((size_t) ne0 * cols); - std::copy_n(t.data.begin(), (size_t) ne0 * cols, out.data.begin()); - return out; -} - -static ref_tensor_f32 slice_column_range(const ref_tensor_f32 & t, int col0, int n_cols) { - if (t.shape.empty()) { - fprintf(stderr, "slice_column_range: empty tensor\n"); - std::exit(1); - } - - const int ne0 = t.shape[0]; - const int total_cols = (int) t.data.size() / ne0; - if (total_cols <= 0 || ne0 * total_cols != (int) t.data.size()) { - fprintf(stderr, "slice_column_range: invalid shape/data size\n"); - std::exit(1); - } - - if (col0 < 0) { - col0 = 0; - } - if (col0 >= total_cols) { - col0 = total_cols - 1; - } - const int cols = (col0 + n_cols <= total_cols) ? n_cols : (total_cols - col0); - - ref_tensor_f32 out; - out.shape = {ne0, cols, 1, 1}; - out.data.resize((size_t) ne0 * cols); - const float * src = t.data.data() + (size_t) ne0 * col0; - std::copy_n(src, (size_t) ne0 * cols, out.data.begin()); - return out; -} - -static compare_result print_diff_row_ex(const char * label, - const ref_tensor_f32 & a, - const ref_tensor_f32 & b, - float atol = 1e-4f) { - const compare_result r = compare_tensors(a.data.data(), b.data.data(), a.numel(), atol); - fprintf(stderr, "%-26s %-18s %12.6f %12.6f %10d\n", - label, - format_shape(a.shape).c_str(), - r.max_diff, - r.mean_diff, - r.n_bad); - return r; -} - -static void print_diff_row(const char * label, - const ref_tensor_f32 & a, - const ref_tensor_f32 & b, - float atol = 1e-4f) { - (void) print_diff_row_ex(label, a, b, atol); -} - -static void print_graph_row(const dump_item & item, - const std::string & cpu_dir, - const std::string & metal_dir) { - const ref_tensor_f32 cpu = load_ref_f32(cpu_dir + "/" + item.name); - const ref_tensor_f32 metal = load_ref_f32(metal_dir + "/" + item.name); - if (cpu.data.empty() || metal.data.empty()) { - fprintf(stderr, "%-26s %-18s %12s %12s %10s\n", - item.label, "-", "load-fail", "load-fail", "-"); - return; - } - print_diff_row(item.label, cpu, metal); -} - -static bool is_global_block(const sam3_model & model, int block_idx, sam3_tensor_info & rope_info) { - const std::string rope_name = "vit.blocks." + std::to_string(block_idx) + ".attn.freqs_cis"; - if (!sam3_get_model_tensor_info(model, rope_name, rope_info)) { - fprintf(stderr, "failed to load %s info\n", rope_name.c_str()); - std::exit(1); - } - return rope_info.ne[2] == 5184; -} - -struct stage_bundle { - ref_tensor_f32 norm1; - ref_tensor_f32 attn_input; - ref_tensor_f32 qkv; - ref_tensor_f32 attn; - ref_tensor_f32 attn_proj; - ref_tensor_f32 post_attn; - ref_tensor_f32 resid1; - ref_tensor_f32 norm2; - ref_tensor_f32 mlp_fc1; - ref_tensor_f32 mlp_gelu; - ref_tensor_f32 mlp_fc2; - ref_tensor_f32 mlp; - ref_tensor_f32 block; -}; - -static stage_bundle run_block_bundle(const sam3_model & model, - int block_idx, - bool is_global, - const ref_tensor_f32 & block_input, - int n_threads) { - stage_bundle out; - - out.norm1 = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_NORM1, block_input, n_threads); - out.attn_input = is_global - ? out.norm1 - : run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_WINDOW_PART, out.norm1, n_threads); - out.qkv = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, out.attn_input, n_threads); - out.attn = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_ATTN_CORE, out.qkv, n_threads); - out.attn_proj = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, out.attn, n_threads); - out.post_attn = is_global - ? out.attn_proj - : run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_WINDOW_UNPART, out.attn_proj, n_threads); - out.resid1 = add_ref(block_input, out.post_attn); - out.norm2 = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_NORM2, out.resid1, n_threads); - out.mlp_fc1 = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC1, out.norm2, n_threads); - out.mlp_gelu = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_GELU, out.mlp_fc1, n_threads); - out.mlp_fc2 = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC2, out.mlp_gelu, n_threads); - out.mlp = run_stage_or_die(model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP, out.norm2, n_threads); - out.block = add_ref(out.resid1, out.mlp); - - return out; -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [ref_dir] [block_idx]\n", argv[0]); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); - fprintf(stderr, "Default block_idx: 16\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - const int block_idx = argc >= 4 ? std::atoi(argv[3]) : 16; - - const std::string input_name = block_idx == 0 ? "dbg_after_ln_pre" : make_block_name(block_idx - 1, "out"); - const std::vector graph_items = { - { "block_input", input_name }, - { "qkv_proj_output", make_block_name(block_idx, "qkv_proj") }, - { "attn_proj_output", make_block_name(block_idx, "attn_proj") }, - { "resid1_output", make_block_name(block_idx, "resid1") }, - { "mlp_output", make_block_name(block_idx, "mlp") }, - { "block_output", make_block_name(block_idx, "out") }, - }; - - const std::string cpu_dir = "/tmp/sam3_block_stage_cpu"; - const std::string metal_dir = "/tmp/sam3_block_stage_metal"; - ensure_dir(cpu_dir); - ensure_dir(metal_dir); - - const ref_tensor_f32 preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty() || preprocessed.shape.size() != 4) { - fprintf(stderr, "failed to load %s/preprocessed\n", ref_dir.c_str()); - return 1; - } - const int img_size = preprocessed.shape[2]; - const int n_threads = 8; - - sam3_params cpu_params; - cpu_params.model_path = model_path; - cpu_params.use_gpu = false; - cpu_params.n_threads = n_threads; - - sam3_params metal_params = cpu_params; - metal_params.use_gpu = true; - - auto cpu_model = sam3_load_model(cpu_params); - auto metal_model = sam3_load_model(metal_params); - if (!cpu_model || !metal_model) { - fprintf(stderr, "failed to load CPU or Metal model\n"); - return 1; - } - - auto cpu_state = sam3_create_state(*cpu_model, cpu_params); - auto metal_state = sam3_create_state(*metal_model, metal_params); - if (!cpu_state || !metal_state) { - fprintf(stderr, "failed to create CPU or Metal state\n"); - return 1; - } - - if (!run_selective_vit(*cpu_state, *cpu_model, preprocessed.data, img_size, block_idx, graph_items, cpu_dir)) { - fprintf(stderr, "CPU selective block run failed\n"); - return 1; - } - if (!run_selective_vit(*metal_state, *metal_model, preprocessed.data, img_size, block_idx, graph_items, metal_dir)) { - fprintf(stderr, "Metal selective block run failed\n"); - return 1; - } - - fprintf(stderr, "\n=== Block-%d graph outputs: CPU vs Metal ===\n", block_idx); - fprintf(stderr, "%-26s %-18s %12s %12s %10s\n", - "checkpoint", "shape", "max_abs_diff", "mean_abs", "n_bad"); - for (const auto & item : graph_items) { - print_graph_row(item, cpu_dir, metal_dir); - } - - const ref_tensor_f32 cpu_block_input = load_ref_f32(cpu_dir + "/" + input_name); - const ref_tensor_f32 metal_block_input = load_ref_f32(metal_dir + "/" + input_name); - if (cpu_block_input.data.empty() || metal_block_input.data.empty()) { - fprintf(stderr, "failed to load canonical block input %s\n", input_name.c_str()); - return 1; - } - - sam3_tensor_info rope_info = {}; - const bool global = is_global_block(*cpu_model, block_idx, rope_info); - - sam3_tensor_info qkv_info = {}; - sam3_tensor_info proj_info = {}; - sam3_tensor_info mlp1_info = {}; - sam3_get_model_tensor_info(*cpu_model, "vit.blocks." + std::to_string(block_idx) + ".attn.qkv.weight", qkv_info); - sam3_get_model_tensor_info(*cpu_model, "vit.blocks." + std::to_string(block_idx) + ".attn.proj.weight", proj_info); - sam3_get_model_tensor_info(*cpu_model, "vit.blocks." + std::to_string(block_idx) + ".mlp.lin1.weight", mlp1_info); - - fprintf(stderr, "\n=== Exact block-%d case ===\n", block_idx); - fprintf(stderr, "block input shape: %s\n", format_shape(cpu_block_input.shape).c_str()); - fprintf(stderr, "attention type: %s\n", global ? "global" : "windowed"); - fprintf(stderr, "qkv weight: shape=[%lld,%lld] type=%d\n", - (long long) qkv_info.ne[0], (long long) qkv_info.ne[1], qkv_info.type); - fprintf(stderr, "proj weight: shape=[%lld,%lld] type=%d\n", - (long long) proj_info.ne[0], (long long) proj_info.ne[1], proj_info.type); - fprintf(stderr, "mlp fc1 weight: shape=[%lld,%lld] type=%d\n", - (long long) mlp1_info.ne[0], (long long) mlp1_info.ne[1], mlp1_info.type); - fprintf(stderr, "freqs_cis: shape=[%lld,%lld,%lld] type=%d\n", - (long long) rope_info.ne[0], (long long) rope_info.ne[1], (long long) rope_info.ne[2], rope_info.type); - print_diff_row("block_input_exact", cpu_block_input, metal_block_input); - - const stage_bundle cpu_shared = run_block_bundle(*cpu_model, block_idx, global, cpu_block_input, n_threads); - const stage_bundle metal_shared = run_block_bundle(*metal_model, block_idx, global, cpu_block_input, n_threads); - const stage_bundle cpu_self = run_block_bundle(*cpu_model, block_idx, global, cpu_block_input, n_threads); - const stage_bundle metal_self = run_block_bundle(*metal_model, block_idx, global, metal_block_input, n_threads); - - fprintf(stderr, "\n=== Exact isolated stages on shared canonical CPU inputs ===\n"); - fprintf(stderr, "%-26s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("norm1_output", cpu_shared.norm1, metal_shared.norm1); - print_diff_row(global ? "attn_input_output" : "window_part_output", cpu_shared.attn_input, metal_shared.attn_input); - const compare_result qkv_diff = print_diff_row_ex("qkv_proj_output", cpu_shared.qkv, metal_shared.qkv); - const compare_result attn_diff = print_diff_row_ex("attn_core_output", cpu_shared.attn, metal_shared.attn); - const compare_result attn_proj_diff = print_diff_row_ex("attn_proj_output", cpu_shared.attn_proj, metal_shared.attn_proj); - print_diff_row(global ? "post_attn_output" : "window_unpart_output", cpu_shared.post_attn, metal_shared.post_attn); - print_diff_row("resid1_output", cpu_shared.resid1, metal_shared.resid1); - print_diff_row("norm2_output", cpu_shared.norm2, metal_shared.norm2); - print_diff_row("mlp_fc1_output", cpu_shared.mlp_fc1, metal_shared.mlp_fc1); - print_diff_row("mlp_gelu_output", cpu_shared.mlp_gelu, metal_shared.mlp_gelu); - const compare_result mlp_fc2_diff = print_diff_row_ex("mlp_fc2_output", cpu_shared.mlp_fc2, metal_shared.mlp_fc2); - print_diff_row("mlp_output", cpu_shared.mlp, metal_shared.mlp); - print_diff_row("block_output", cpu_shared.block, metal_shared.block); - - auto print_worst_loc = [](const char * label, const ref_tensor_f32 & t, const compare_result & diff) { - const int ne0 = t.shape.empty() ? 0 : t.shape[0]; - const int worst_col = ne0 > 0 ? diff.worst_index / ne0 : -1; - const int worst_row = ne0 > 0 ? diff.worst_index % ne0 : -1; - fprintf(stderr, "%s worst: col=%d row=%d cpu=%.6f metal=%.6f max_abs=%.6f\n", - label, worst_col, worst_row, diff.worst_a, diff.worst_b, diff.max_diff); - }; - - fprintf(stderr, "\n=== Worst-element locations on shared canonical CPU inputs ===\n"); - print_worst_loc("qkv_proj_output", cpu_shared.qkv, qkv_diff); - print_worst_loc("attn_core_output", cpu_shared.attn, attn_diff); - print_worst_loc("attn_proj_output", cpu_shared.attn_proj, attn_proj_diff); - print_worst_loc("mlp_fc2_output", cpu_shared.mlp_fc2, mlp_fc2_diff); - - const int linear_slice_cols = 32; - const ref_tensor_f32 qkv_in_slice = slice_first_columns(cpu_shared.attn_input, linear_slice_cols); - const ref_tensor_f32 qkv_cpu_slice = slice_first_columns(cpu_shared.qkv, linear_slice_cols); - const ref_tensor_f32 qkv_mtl_slice = slice_first_columns(metal_shared.qkv, linear_slice_cols); - const ref_tensor_f32 attn_proj_in_slice = slice_first_columns(cpu_shared.attn, linear_slice_cols); - const ref_tensor_f32 attn_proj_cpu_slice = slice_first_columns(cpu_shared.attn_proj, linear_slice_cols); - const ref_tensor_f32 attn_proj_mtl_slice = slice_first_columns(metal_shared.attn_proj, linear_slice_cols); - const ref_tensor_f32 mlp_fc1_in_slice = slice_first_columns(cpu_shared.norm2, linear_slice_cols); - const ref_tensor_f32 mlp_fc1_cpu_slice = slice_first_columns(cpu_shared.mlp_fc1, linear_slice_cols); - const ref_tensor_f32 mlp_fc1_mtl_slice = slice_first_columns(metal_shared.mlp_fc1, linear_slice_cols); - const ref_tensor_f32 mlp_fc2_in_slice = slice_first_columns(cpu_shared.mlp_gelu, linear_slice_cols); - const ref_tensor_f32 mlp_fc2_cpu_slice = slice_first_columns(cpu_shared.mlp_fc2, linear_slice_cols); - const ref_tensor_f32 mlp_fc2_mtl_slice = slice_first_columns(metal_shared.mlp_fc2, linear_slice_cols); - - const ref_tensor_f32 host_qkv_f32 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, qkv_in_slice, false); - const ref_tensor_f32 host_qkv_f64 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, qkv_in_slice, true); - const ref_tensor_f32 host_attn_proj_f32 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, attn_proj_in_slice, false); - const ref_tensor_f32 host_attn_proj_f64 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, attn_proj_in_slice, true); - const ref_tensor_f32 host_mlp_fc1_f32 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC1, mlp_fc1_in_slice, false); - const ref_tensor_f32 host_mlp_fc1_f64 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC1, mlp_fc1_in_slice, true); - const ref_tensor_f32 host_mlp_fc2_f32 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC2, mlp_fc2_in_slice, false); - const ref_tensor_f32 host_mlp_fc2_f64 = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC2, mlp_fc2_in_slice, true); - - fprintf(stderr, "\n=== Exact block-%d linear host refs (first %d tokens) ===\n", block_idx, linear_slice_cols); - fprintf(stderr, "%-26s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("qkv_cpu_vs_host_f32", qkv_cpu_slice, host_qkv_f32); - print_diff_row("qkv_mtl_vs_host_f32", qkv_mtl_slice, host_qkv_f32); - print_diff_row("qkv_cpu_vs_host_f64", qkv_cpu_slice, host_qkv_f64); - print_diff_row("qkv_mtl_vs_host_f64", qkv_mtl_slice, host_qkv_f64); - print_diff_row("attn_proj_cpu_vs_f32", attn_proj_cpu_slice, host_attn_proj_f32); - print_diff_row("attn_proj_mtl_vs_f32", attn_proj_mtl_slice, host_attn_proj_f32); - print_diff_row("attn_proj_cpu_vs_f64", attn_proj_cpu_slice, host_attn_proj_f64); - print_diff_row("attn_proj_mtl_vs_f64", attn_proj_mtl_slice, host_attn_proj_f64); - print_diff_row("mlp_fc1_cpu_vs_f32", mlp_fc1_cpu_slice, host_mlp_fc1_f32); - print_diff_row("mlp_fc1_mtl_vs_f32", mlp_fc1_mtl_slice, host_mlp_fc1_f32); - print_diff_row("mlp_fc1_cpu_vs_f64", mlp_fc1_cpu_slice, host_mlp_fc1_f64); - print_diff_row("mlp_fc1_mtl_vs_f64", mlp_fc1_mtl_slice, host_mlp_fc1_f64); - print_diff_row("mlp_fc2_cpu_vs_f32", mlp_fc2_cpu_slice, host_mlp_fc2_f32); - print_diff_row("mlp_fc2_mtl_vs_f32", mlp_fc2_mtl_slice, host_mlp_fc2_f32); - print_diff_row("mlp_fc2_cpu_vs_f64", mlp_fc2_cpu_slice, host_mlp_fc2_f64); - print_diff_row("mlp_fc2_mtl_vs_f64", mlp_fc2_mtl_slice, host_mlp_fc2_f64); - - const int qkv_worst_col = qkv_diff.worst_index / cpu_shared.qkv.shape[0]; - const int attn_proj_worst_col = attn_proj_diff.worst_index / cpu_shared.attn_proj.shape[0]; - const int mlp_fc2_worst_col = mlp_fc2_diff.worst_index / cpu_shared.mlp_fc2.shape[0]; - - const ref_tensor_f32 qkv_in_worst = slice_column_range(cpu_shared.attn_input, qkv_worst_col, 1); - const ref_tensor_f32 qkv_cpu_worst = slice_column_range(cpu_shared.qkv, qkv_worst_col, 1); - const ref_tensor_f32 qkv_mtl_worst = slice_column_range(metal_shared.qkv, qkv_worst_col, 1); - const ref_tensor_f32 qkv_host_worst = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_QKV_PROJ, qkv_in_worst, false); - - const ref_tensor_f32 attn_proj_in_worst = slice_column_range(cpu_shared.attn, attn_proj_worst_col, 1); - const ref_tensor_f32 attn_proj_cpu_worst = slice_column_range(cpu_shared.attn_proj, attn_proj_worst_col, 1); - const ref_tensor_f32 attn_proj_mtl_worst = slice_column_range(metal_shared.attn_proj, attn_proj_worst_col, 1); - const ref_tensor_f32 attn_proj_host_worst = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_ATTN_PROJ, attn_proj_in_worst, false); - - const ref_tensor_f32 mlp_fc2_in_worst = slice_column_range(cpu_shared.mlp_gelu, mlp_fc2_worst_col, 1); - const ref_tensor_f32 mlp_fc2_cpu_worst = slice_column_range(cpu_shared.mlp_fc2, mlp_fc2_worst_col, 1); - const ref_tensor_f32 mlp_fc2_mtl_worst = slice_column_range(metal_shared.mlp_fc2, mlp_fc2_worst_col, 1); - const ref_tensor_f32 mlp_fc2_host_worst = run_linear_host_ref_or_die(*cpu_model, block_idx, SAM3_VIT_BLOCK_STAGE_MLP_FC2, mlp_fc2_in_worst, false); - - fprintf(stderr, "\n=== Worst-column host refs (single real token) ===\n"); - fprintf(stderr, "%-26s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - print_diff_row("qkv_cpu_vs_host_worst", qkv_cpu_worst, qkv_host_worst); - print_diff_row("qkv_mtl_vs_host_worst", qkv_mtl_worst, qkv_host_worst); - print_diff_row("attn_proj_cpu_vs_worst", attn_proj_cpu_worst, attn_proj_host_worst); - print_diff_row("attn_proj_mtl_vs_worst", attn_proj_mtl_worst, attn_proj_host_worst); - print_diff_row("mlp_fc2_cpu_vs_worst", mlp_fc2_cpu_worst, mlp_fc2_host_worst); - print_diff_row("mlp_fc2_mtl_vs_worst", mlp_fc2_mtl_worst, mlp_fc2_host_worst); - - const std::map cpu_graph = { - {"qkv_proj_output", load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "qkv_proj"))}, - {"attn_proj_output", load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "attn_proj"))}, - {"resid1_output", load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "resid1"))}, - {"mlp_output", load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "mlp"))}, - {"block_output", load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "out"))}, - }; - const std::map metal_graph = { - {"qkv_proj_output", load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "qkv_proj"))}, - {"attn_proj_output", load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "attn_proj"))}, - {"resid1_output", load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "resid1"))}, - {"mlp_output", load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "mlp"))}, - {"block_output", load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "out"))}, - }; - - fprintf(stderr, "\n=== Isolated self-consistency vs graph outputs ===\n"); - fprintf(stderr, "%-26s %-18s %-24s %-24s\n", - "stage", "shape", "cpu_iso_vs_cpu_graph", "metal_iso_vs_metal_graph"); - - auto print_self_row = [&](const char * label, - const ref_tensor_f32 & cpu_iso, - const ref_tensor_f32 & metal_iso, - const char * graph_key) { - const auto cpu_it = cpu_graph.find(graph_key); - const auto metal_it = metal_graph.find(graph_key); - if (cpu_it == cpu_graph.end() || metal_it == metal_graph.end()) { - fprintf(stderr, "%-26s %-18s %-24s %-24s\n", label, "-", "missing", "missing"); - return; - } - - const compare_result cpu_r = compare_tensors(cpu_iso.data.data(), cpu_it->second.data.data(), cpu_iso.numel(), 1e-4f); - const compare_result metal_r = compare_tensors(metal_iso.data.data(), metal_it->second.data.data(), metal_iso.numel(), 1e-4f); - - char cpu_buf[64]; - char metal_buf[64]; - snprintf(cpu_buf, sizeof(cpu_buf), "max=%.6f mean=%.6f", cpu_r.max_diff, cpu_r.mean_diff); - snprintf(metal_buf, sizeof(metal_buf), "max=%.6f mean=%.6f", metal_r.max_diff, metal_r.mean_diff); - fprintf(stderr, "%-26s %-18s %-24s %-24s\n", - label, - format_shape(cpu_iso.shape).c_str(), - cpu_buf, - metal_buf); - }; - - print_self_row("qkv_proj_output", cpu_self.qkv, metal_self.qkv, "qkv_proj_output"); - print_self_row("attn_proj_output", cpu_self.attn_proj, metal_self.attn_proj, "attn_proj_output"); - print_self_row("resid1_output", cpu_self.resid1, metal_self.resid1, "resid1_output"); - print_self_row("mlp_output", cpu_self.mlp, metal_self.mlp, "mlp_output"); - print_self_row("block_output", cpu_self.block, metal_self.block, "block_output"); - - return 0; -} diff --git a/tests/test_metal_conv_dw.cpp b/tests/test_metal_conv_dw.cpp deleted file mode 100644 index 86ab02a..0000000 --- a/tests/test_metal_conv_dw.cpp +++ /dev/null @@ -1,214 +0,0 @@ -// Test Metal kernel for ggml_conv_2d_dw_direct (GGML_OP_CONV_2D_DW). -// Runs the same graph on CPU and Metal, compares results element-by-element. - -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include - -static std::vector run_conv_2d_dw(ggml_backend_t backend, - const std::vector& kernel_data, - const std::vector& input_data, - int KW, int KH, int C, - int W, int H, int N, - int s0, int s1, int p0, int p1, - int d0, int d1, - int* out_w, int* out_h) { - size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); - struct ggml_init_params params = {ctx_size, nullptr, true}; - struct ggml_context* ctx = ggml_init(params); - - // Kernel: [KW, KH, 1, C] - auto* knl = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, KW, KH, 1, C); - ggml_set_name(knl, "kernel"); - ggml_set_input(knl); - - // Input: [W, H, C, N] - auto* inp = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, W, H, C, N); - ggml_set_name(inp, "input"); - ggml_set_input(inp); - - // Depthwise conv - auto* out = ggml_conv_2d_dw_direct(ctx, knl, inp, s0, s1, p0, p1, d0, d1); - ggml_set_name(out, "output"); - ggml_set_output(out); - - auto* graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, out); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "FAIL: gallocr_alloc_graph failed\n"); - exit(1); - } - - ggml_backend_tensor_set(knl, kernel_data.data(), 0, kernel_data.size() * sizeof(float)); - ggml_backend_tensor_set(inp, input_data.data(), 0, input_data.size() * sizeof(float)); - - ggml_backend_graph_compute(backend, graph); - - *out_w = (int)out->ne[0]; - *out_h = (int)out->ne[1]; - int64_t total = ggml_nelements(out); - - std::vector output(total); - ggml_backend_tensor_get(out, output.data(), 0, total * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return output; -} - -static bool compare_vectors(const std::vector& a, const std::vector& b, - const char* label, float tol = 1e-5f) { - if (a.size() != b.size()) { - fprintf(stderr, " FAIL [%s]: size mismatch: %zu vs %zu\n", label, a.size(), b.size()); - return false; - } - float max_diff = 0.0f; - int first_fail = -1; - for (size_t i = 0; i < a.size(); i++) { - float diff = std::fabs(a[i] - b[i]); - if (diff > max_diff) max_diff = diff; - if (diff > tol && first_fail < 0) first_fail = (int)i; - } - if (first_fail >= 0) { - fprintf(stderr, " FAIL [%s]: max_diff=%.8f at index %d (cpu=%.8f metal=%.8f)\n", - label, max_diff, first_fail, a[first_fail], b[first_fail]); - return false; - } - fprintf(stderr, " PASS [%s]: max_diff=%.2e (%zu elements)\n", label, max_diff, a.size()); - return true; -} - -int main() { -#ifndef GGML_USE_METAL - fprintf(stderr, "Metal not available, skipping test\n"); - return 0; -#else - ggml_backend_t cpu_backend = ggml_backend_cpu_init(); - ggml_backend_t metal_backend = ggml_backend_metal_init(); - if (!metal_backend) { - fprintf(stderr, "FAIL: could not init Metal backend\n"); - return 1; - } - - bool all_pass = true; - - // ── Test 1: sam3 exact dimensions (K=7, C=256, 72x72, s=1, p=3, d=1) ── - { - fprintf(stderr, "\n=== Test 1: sam3 dims (K=7, C=256, 72x72, s=1, p=3) ===\n"); - const int KW = 7, KH = 7, C = 256, W = 72, H = 72, N = 1; - const int s0 = 1, s1 = 1, p0 = 3, p1 = 3, d0 = 1, d1 = 1; - - std::vector kernel_data(KW * KH * 1 * C); - std::vector input_data(W * H * C * N); - for (size_t i = 0; i < kernel_data.size(); i++) - kernel_data[i] = 0.01f * ((float)(i % 97) - 48.0f); - for (size_t i = 0; i < input_data.size(); i++) - input_data[i] = 0.001f * ((float)(i % 9973) - 4986.0f); - - int ow_cpu, oh_cpu, ow_metal, oh_metal; - auto cpu_out = run_conv_2d_dw(cpu_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_cpu, &oh_cpu); - auto metal_out = run_conv_2d_dw(metal_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_metal, &oh_metal); - - fprintf(stderr, " output shape: [%d, %d, %d, %d]\n", ow_cpu, oh_cpu, C, N); - all_pass &= compare_vectors(cpu_out, metal_out, "sam3_conv_dw", 1e-4f); - } - - // ── Test 2: small case (K=3, C=4, 8x8, s=1, p=1) ────────────────── - { - fprintf(stderr, "\n=== Test 2: small (K=3, C=4, 8x8, s=1, p=1) ===\n"); - const int KW = 3, KH = 3, C = 4, W = 8, H = 8, N = 1; - const int s0 = 1, s1 = 1, p0 = 1, p1 = 1, d0 = 1, d1 = 1; - - std::vector kernel_data(KW * KH * 1 * C); - std::vector input_data(W * H * C * N); - for (size_t i = 0; i < kernel_data.size(); i++) - kernel_data[i] = (float)(i + 1) * 0.1f; - for (size_t i = 0; i < input_data.size(); i++) - input_data[i] = (float)(i + 1) * 0.01f; - - int ow_cpu, oh_cpu, ow_metal, oh_metal; - auto cpu_out = run_conv_2d_dw(cpu_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_cpu, &oh_cpu); - auto metal_out = run_conv_2d_dw(metal_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_metal, &oh_metal); - - fprintf(stderr, " output shape: [%d, %d, %d, %d]\n", ow_cpu, oh_cpu, C, N); - all_pass &= compare_vectors(cpu_out, metal_out, "small_conv_dw", 1e-5f); - } - - // ── Test 3: stride=2, dilation=2 ──────────────────────────────────── - { - fprintf(stderr, "\n=== Test 3: stride=2, dilation=2 (K=3, C=16, 32x32) ===\n"); - const int KW = 3, KH = 3, C = 16, W = 32, H = 32, N = 1; - const int s0 = 2, s1 = 2, p0 = 2, p1 = 2, d0 = 2, d1 = 2; - - std::vector kernel_data(KW * KH * 1 * C); - std::vector input_data(W * H * C * N); - for (size_t i = 0; i < kernel_data.size(); i++) - kernel_data[i] = sinf((float)i * 0.1f); - for (size_t i = 0; i < input_data.size(); i++) - input_data[i] = cosf((float)i * 0.01f); - - int ow_cpu, oh_cpu, ow_metal, oh_metal; - auto cpu_out = run_conv_2d_dw(cpu_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_cpu, &oh_cpu); - auto metal_out = run_conv_2d_dw(metal_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_metal, &oh_metal); - - fprintf(stderr, " output shape: [%d, %d, %d, %d]\n", ow_cpu, oh_cpu, C, N); - all_pass &= compare_vectors(cpu_out, metal_out, "strided_dilated", 1e-5f); - } - - // ── Test 4: no padding ────────────────────────────────────────────── - { - fprintf(stderr, "\n=== Test 4: no padding (K=5, C=8, 16x16) ===\n"); - const int KW = 5, KH = 5, C = 8, W = 16, H = 16, N = 2; - const int s0 = 1, s1 = 1, p0 = 0, p1 = 0, d0 = 1, d1 = 1; - - std::vector kernel_data(KW * KH * 1 * C); - std::vector input_data(W * H * C * N); - for (size_t i = 0; i < kernel_data.size(); i++) - kernel_data[i] = 0.02f * ((float)(i % 51) - 25.0f); - for (size_t i = 0; i < input_data.size(); i++) - input_data[i] = 0.005f * ((float)(i % 199) - 99.0f); - - int ow_cpu, oh_cpu, ow_metal, oh_metal; - auto cpu_out = run_conv_2d_dw(cpu_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_cpu, &oh_cpu); - auto metal_out = run_conv_2d_dw(metal_backend, kernel_data, input_data, - KW, KH, C, W, H, N, s0, s1, p0, p1, d0, d1, - &ow_metal, &oh_metal); - - fprintf(stderr, " output shape: [%d, %d, %d, %d]\n", ow_cpu, oh_cpu, C, N); - all_pass &= compare_vectors(cpu_out, metal_out, "no_pad_batch2", 1e-5f); - } - - ggml_backend_free(cpu_backend); - ggml_backend_free(metal_backend); - - fprintf(stderr, "\n%s\n", all_pass ? "ALL TESTS PASSED" : "SOME TESTS FAILED"); - return all_pass ? 0 : 1; -#endif -} diff --git a/tests/test_metal_conv_transpose_2d.cpp b/tests/test_metal_conv_transpose_2d.cpp deleted file mode 100644 index 31b9838..0000000 --- a/tests/test_metal_conv_transpose_2d.cpp +++ /dev/null @@ -1,338 +0,0 @@ -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include "test_utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -struct options { - std::string case_name = "all"; - bool dump_mismatch = false; - int n_threads = 8; - int max_wall_ms = 30000; -}; - -struct case_desc { - const char * name; - int64_t in_w; - int64_t in_h; - int64_t in_c; - int64_t out_c; - int64_t kw; - int64_t kh; - int stride; - uint32_t seed; - float tol; -}; - -struct run_result { - std::vector output; - int64_t out_w = 0; - int64_t out_h = 0; - int64_t out_c = 0; -}; - -static const case_desc CASES[] = { - { "cpu_ref_stride1", 3, 2, 2, 3, 2, 2, 1, 0x1234u, 1e-5f }, - { "cpu_ref_stride2", 3, 2, 2, 3, 2, 2, 2, 0x2345u, 1e-5f }, - { "cpu_ref_stride3", 3, 2, 2, 3, 2, 2, 3, 0x3456u, 1e-5f }, - { "small_rect", 5, 4, 3, 4, 3, 2, 1, 0x4567u, 1e-5f }, - { "medium_stride2", 8, 7, 5, 6, 3, 3, 2, 0x5678u, 1e-4f }, - { "sam_first", 72, 72, 1024, 512, 2, 2, 2, 0x6789u, 2e-3f }, - { "sam_second", 144, 144, 512, 256, 2, 2, 2, 0x789au, 2e-3f }, -}; - -static void usage(const char * argv0) { - fprintf(stderr, - "Usage: %s [--case NAME|all] [--dump-mismatch] [--n-threads N] [--max-wall-ms N] [--list]\n", - argv0); -} - -static bool parse_args(int argc, char ** argv, options & opts) { - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "--case") == 0) { - if (i + 1 >= argc) return false; - opts.case_name = argv[++i]; - } else if (strcmp(argv[i], "--dump-mismatch") == 0) { - opts.dump_mismatch = true; - } else if (strcmp(argv[i], "--n-threads") == 0) { - if (i + 1 >= argc) return false; - opts.n_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "--max-wall-ms") == 0) { - if (i + 1 >= argc) return false; - opts.max_wall_ms = atoi(argv[++i]); - } else if (strcmp(argv[i], "--list") == 0) { - for (const case_desc & tc : CASES) { - fprintf(stderr, "%s\n", tc.name); - } - exit(0); - } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { - return false; - } else { - fprintf(stderr, "Unknown argument: %s\n", argv[i]); - return false; - } - } - return true; -} - -static float gen_value(uint64_t idx, uint32_t seed, float scale) { - uint32_t x = (uint32_t) idx ^ seed; - x = x * 1664525u + 1013904223u; - x ^= x >> 16; - const int32_t centered = (int32_t) (x % 511u) - 255; - return centered * scale; -} - -static void fill_input_data(const case_desc & tc, std::vector & input) { - for (size_t i = 0; i < input.size(); ++i) { - input[i] = gen_value(i, tc.seed ^ 0x13579bdu, 1.0f / 256.0f); - } -} - -static void fill_weight_data(const case_desc & tc, std::vector & weight_f16, std::vector & weight_f32) { - for (size_t i = 0; i < weight_f16.size(); ++i) { - const float v = gen_value(i, tc.seed ^ 0x2468aceu, 1.0f / 128.0f); - weight_f32[i] = v; - weight_f16[i] = ggml_fp32_to_fp16(v); - } -} - -static ggml_backend_t create_backend(const std::string & name, int n_threads) { - if (name == "cpu") { - ggml_backend_t backend = ggml_backend_cpu_init(); - if (backend) { - ggml_backend_cpu_set_n_threads(backend, n_threads); - } - return backend; - } - -#ifdef GGML_USE_METAL - if (name == "metal") { - return ggml_backend_metal_init(); - } -#endif - - return nullptr; -} - -static run_result run_case(const case_desc & tc, - const std::string & backend_name, - int n_threads, - const std::vector & weight_data, - const std::vector & input_data) { - ggml_backend_t backend = create_backend(backend_name, n_threads); - if (!backend) { - fprintf(stderr, "Failed to create backend '%s'\n", backend_name.c_str()); - exit(2); - } - - const size_t ctx_size = ggml_tensor_overhead() * 8 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - - ggml_context * ctx = ggml_init(params); - if (!ctx) { - fprintf(stderr, "Failed to create ggml context\n"); - ggml_backend_free(backend); - exit(3); - } - - ggml_tensor * weight = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, tc.kw, tc.kh, tc.out_c, tc.in_c); - ggml_tensor * input = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, tc.in_w, tc.in_h, tc.in_c, 1); - ggml_set_name(weight, "weight"); - ggml_set_name(input, "input"); - ggml_set_input(weight); - ggml_set_input(input); - - ggml_tensor * output = ggml_conv_transpose_2d_p0(ctx, weight, input, tc.stride); - ggml_set_name(output, "output"); - ggml_set_output(output); - - ggml_cgraph * graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, output); - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "Failed to allocate graph for case %s (%s)\n", tc.name, backend_name.c_str()); - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - exit(4); - } - - ggml_backend_tensor_set(weight, weight_data.data(), 0, weight_data.size() * sizeof(uint16_t)); - ggml_backend_tensor_set(input, input_data.data(), 0, input_data.size() * sizeof(float)); - - const ggml_status status = ggml_backend_graph_compute(backend, graph); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "Graph compute failed for case %s on %s: %s\n", - tc.name, backend_name.c_str(), ggml_status_to_string(status)); - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - exit(5); - } - - run_result res; - res.out_w = output->ne[0]; - res.out_h = output->ne[1]; - res.out_c = output->ne[2]; - res.output.resize((size_t) ggml_nelements(output)); - ggml_backend_tensor_get(output, res.output.data(), 0, res.output.size() * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - return res; -} - -static void dump_reference_mismatch(const case_desc & tc, - const std::vector & input_data, - const std::vector & weight_data, - const run_result & cpu, - const run_result & metal, - int worst_index) { - if (worst_index < 0) { - return; - } - - const int64_t ow = worst_index % cpu.out_w; - const int64_t rem0 = worst_index / cpu.out_w; - const int64_t oh = rem0 % cpu.out_h; - const int64_t oc = rem0 / cpu.out_h; - - fprintf(stderr, - " mismatch case=%s coord=(ow=%lld, oh=%lld, oc=%lld) cpu=%.8f metal=%.8f\n", - tc.name, - (long long) ow, - (long long) oh, - (long long) oc, - cpu.output[worst_index], - metal.output[worst_index]); - - int shown = 0; - double recomputed = 0.0; - for (int64_t kh = 0; kh < tc.kh; ++kh) { - const int64_t in_y_nom = oh - kh; - if (in_y_nom < 0 || in_y_nom % tc.stride != 0) { - continue; - } - const int64_t in_y = in_y_nom / tc.stride; - if (in_y >= tc.in_h) { - continue; - } - - for (int64_t kw = 0; kw < tc.kw; ++kw) { - const int64_t in_x_nom = ow - kw; - if (in_x_nom < 0 || in_x_nom % tc.stride != 0) { - continue; - } - const int64_t in_x = in_x_nom / tc.stride; - if (in_x >= tc.in_w) { - continue; - } - - for (int64_t ic = 0; ic < tc.in_c; ++ic) { - const size_t input_idx = (size_t) ic * tc.in_h * tc.in_w + (size_t) in_y * tc.in_w + (size_t) in_x; - const size_t weight_idx = ((size_t) ic * tc.out_c * tc.kh * tc.kw) + - ((size_t) oc * tc.kh * tc.kw) + - (size_t) kh * tc.kw + - (size_t) kw; - const float contrib = input_data[input_idx] * weight_data[weight_idx]; - recomputed += contrib; - if (shown < 16) { - fprintf(stderr, - " term[%2d] in=(x=%lld,y=%lld,c=%lld)=%.8f weight=(kw=%lld,kh=%lld,oc=%lld,ic=%lld)=%.8f contrib=%.8f\n", - shown, - (long long) in_x, - (long long) in_y, - (long long) ic, - input_data[input_idx], - (long long) kw, - (long long) kh, - (long long) oc, - (long long) ic, - weight_data[weight_idx], - contrib); - } - ++shown; - } - } - } - - fprintf(stderr, " recomputed_reference=%.8f from %d contributing terms\n", recomputed, shown); -} - -int main(int argc, char ** argv) { - options opts; - if (!parse_args(argc, argv, opts)) { - usage(argv[0]); - return 1; - } - -#ifndef GGML_USE_METAL - fprintf(stderr, "Metal not available, skipping test\n"); - return 0; -#else - bool all_pass = true; - - fprintf(stderr, "| case | output | max abs diff | mean abs diff | tolerance | status |\n"); - fprintf(stderr, "| --- | --- | ---: | ---: | ---: | --- |\n"); - - for (const case_desc & tc : CASES) { - if (opts.case_name != "all" && opts.case_name != tc.name) { - continue; - } - - std::vector weight_f16((size_t) tc.kw * tc.kh * tc.out_c * tc.in_c); - std::vector weight_f32(weight_f16.size()); - std::vector input_data((size_t) tc.in_w * tc.in_h * tc.in_c); - - fill_weight_data(tc, weight_f16, weight_f32); - fill_input_data(tc, input_data); - - run_result cpu = run_case(tc, "cpu", opts.n_threads, weight_f16, input_data); - run_result metal = run_case(tc, "metal", opts.n_threads, weight_f16, input_data); - - compare_result diff = compare_tensors(cpu.output.data(), metal.output.data(), (int) cpu.output.size(), tc.tol); - const bool pass = diff.max_diff <= tc.tol; - fprintf(stderr, - "| %s | [%lld,%lld,%lld] | %.8f | %.8f | %.1e | %s |\n", - tc.name, - (long long) cpu.out_w, - (long long) cpu.out_h, - (long long) cpu.out_c, - diff.max_diff, - diff.mean_diff, - tc.tol, - pass ? "PASS" : "FAIL"); - - if (!pass) { - all_pass = false; - if (opts.dump_mismatch || (tc.in_w <= 8 && tc.in_h <= 8 && tc.in_c <= 8)) { - dump_reference_mismatch(tc, input_data, weight_f32, cpu, metal, diff.worst_index); - } - } - } - - fprintf(stderr, "\n%s\n", all_pass ? "ALL TESTS PASSED" : "SOME TESTS FAILED"); - return all_pass ? 0 : 1; -#endif -} diff --git a/tests/test_metal_conv_transpose_2d_stress.cpp b/tests/test_metal_conv_transpose_2d_stress.cpp deleted file mode 100644 index 76f1207..0000000 --- a/tests/test_metal_conv_transpose_2d_stress.cpp +++ /dev/null @@ -1,219 +0,0 @@ -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include -#include - -struct options { - std::string backend = "metal"; - std::string variant = "first"; - int warmup = 1; - int repeats = 3; - int n_threads = 8; - int max_wall_ms = 0; -}; - -struct variant_desc { - const char * name; - int64_t in_w; - int64_t in_h; - int64_t in_c; - int64_t out_c; -}; - -static const variant_desc VARIANT_FIRST = { "first", 72, 72, 1024, 512 }; -static const variant_desc VARIANT_SECOND = { "second", 144, 144, 512, 256 }; - -static void usage(const char * argv0) { - fprintf(stderr, - "Usage: %s [--backend metal|cpu] [--variant first|second] [--warmup N] [--repeats N] [--max-wall-ms N]\n", - argv0); -} - -static bool parse_args(int argc, char ** argv, options & opts) { - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "--backend") == 0) { - if (i + 1 >= argc) return false; - opts.backend = argv[++i]; - } else if (strcmp(argv[i], "--variant") == 0) { - if (i + 1 >= argc) return false; - opts.variant = argv[++i]; - } else if (strcmp(argv[i], "--warmup") == 0) { - if (i + 1 >= argc) return false; - opts.warmup = atoi(argv[++i]); - } else if (strcmp(argv[i], "--repeats") == 0) { - if (i + 1 >= argc) return false; - opts.repeats = atoi(argv[++i]); - } else if (strcmp(argv[i], "--n-threads") == 0) { - if (i + 1 >= argc) return false; - opts.n_threads = atoi(argv[++i]); - } else if (strcmp(argv[i], "--max-wall-ms") == 0) { - if (i + 1 >= argc) return false; - opts.max_wall_ms = atoi(argv[++i]); - } else if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0) { - return false; - } else { - fprintf(stderr, "Unknown argument: %s\n", argv[i]); - return false; - } - } - - if (opts.backend != "metal" && opts.backend != "cpu") { - return false; - } - - if (opts.variant != "first" && opts.variant != "second") { - return false; - } - - return true; -} - -static const variant_desc & get_variant(const std::string & name) { - return name == "second" ? VARIANT_SECOND : VARIANT_FIRST; -} - -static ggml_backend_t create_backend(const options & opts) { -#ifdef GGML_USE_METAL - if (opts.backend == "metal") { - return ggml_backend_metal_init(); - } -#endif - return ggml_backend_cpu_init(); -} - -int main(int argc, char ** argv) { - options opts; - if (!parse_args(argc, argv, opts)) { - usage(argv[0]); - return 1; - } - - ggml_backend_t backend = create_backend(opts); - if (!backend) { - fprintf(stderr, "Failed to create backend '%s'\n", opts.backend.c_str()); - return 2; - } - - if (ggml_backend_is_cpu(backend)) { - ggml_backend_cpu_set_n_threads(backend, opts.n_threads); - } - - const variant_desc & v = get_variant(opts.variant); - const int64_t kw = 2; - const int64_t kh = 2; - const int stride = 2; - - const size_t ctx_size = ggml_tensor_overhead() * 8 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context * ctx = ggml_init(params); - if (!ctx) { - fprintf(stderr, "Failed to create ggml context\n"); - ggml_backend_free(backend); - return 3; - } - - ggml_tensor * weight = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kw, kh, v.out_c, v.in_c); - ggml_tensor * input = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, v.in_w, v.in_h, v.in_c, 1); - ggml_set_name(weight, "weight"); - ggml_set_name(input, "input"); - ggml_set_input(weight); - ggml_set_input(input); - - ggml_tensor * output = ggml_conv_transpose_2d_p0(ctx, weight, input, stride); - ggml_set_name(output, "output"); - ggml_set_output(output); - - ggml_cgraph * graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, output); - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "Failed to allocate graph\n"); - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - return 4; - } - - std::vector wdata((size_t) ggml_nelements(weight)); - for (size_t i = 0; i < wdata.size(); ++i) { - const int32_t centered = (int32_t) (i % 17) - 8; - wdata[i] = ggml_fp32_to_fp16((float) centered * 0.015625f); - } - std::vector xdata((size_t) ggml_nelements(input)); - for (size_t i = 0; i < xdata.size(); ++i) { - const int32_t centered = (int32_t) (i % 23) - 11; - xdata[i] = (float) centered * 0.0078125f; - } - - ggml_backend_tensor_set(weight, wdata.data(), 0, wdata.size() * sizeof(uint16_t)); - ggml_backend_tensor_set(input, xdata.data(), 0, xdata.size() * sizeof(float)); - - for (int i = 0; i < opts.warmup; ++i) { - const ggml_status status = ggml_backend_graph_compute(backend, graph); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "Warmup failed: %s\n", ggml_status_to_string(status)); - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - return 5; - } - } - - auto t0 = std::chrono::high_resolution_clock::now(); - for (int i = 0; i < opts.repeats; ++i) { - const ggml_status status = ggml_backend_graph_compute(backend, graph); - if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "Run %d failed: %s\n", i + 1, ggml_status_to_string(status)); - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - return 6; - } - } - auto t1 = std::chrono::high_resolution_clock::now(); - - std::vector out((size_t) std::min(ggml_nelements(output), 64)); - ggml_backend_tensor_get(output, out.data(), 0, out.size() * sizeof(float)); - double checksum = 0.0; - for (float vout : out) { - checksum += vout; - } - - const double avg_ms = std::chrono::duration(t1 - t0).count() / opts.repeats; - - fprintf(stderr, - "RESULT backend=%s variant=%s in=[%lld,%lld,%lld] out_c=%lld out=[%lld,%lld,%lld] avg_ms=%.3f checksum=%.6f repeats=%d\n", - opts.backend.c_str(), - v.name, - (long long) v.in_w, - (long long) v.in_h, - (long long) v.in_c, - (long long) v.out_c, - (long long) output->ne[0], - (long long) output->ne[1], - (long long) output->ne[2], - avg_ms, - checksum, - opts.repeats); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - ggml_backend_free(backend); - return 0; -} diff --git a/tests/test_metal_encoder_checkpoints.cpp b/tests/test_metal_encoder_checkpoints.cpp deleted file mode 100644 index cbbd0be..0000000 --- a/tests/test_metal_encoder_checkpoints.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include - -struct checkpoint_stat { - std::string name; - std::string shape; - compare_result diff; -}; - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static bool dump_checkpoints(const sam3_state & state, - const std::vector & checkpoints, - const std::string & output_dir) { - for (const auto & name : checkpoints) { - if (!sam3_dump_state_tensor(state, name, output_dir + "/" + name)) { - fprintf(stderr, "failed to dump checkpoint '%s'\n", name.c_str()); - return false; - } - } - - return true; -} - -static bool run_backend(const std::string & model_path, - const std::vector & chw_data, - int img_size, - bool use_gpu, - int n_threads, - const std::vector & checkpoints, - const std::string & dump_dir, - double & elapsed_ms) { - sam3_params params; - params.model_path = model_path; - params.use_gpu = use_gpu; - params.n_threads = n_threads; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "failed to load model for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "failed to create state for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto t0 = std::chrono::high_resolution_clock::now(); - const bool ok = sam3_encode_image_from_preprocessed(*state, *model, chw_data.data(), img_size); - auto t1 = std::chrono::high_resolution_clock::now(); - elapsed_ms = std::chrono::duration(t1 - t0).count(); - if (!ok) { - fprintf(stderr, "%s encode failed\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - if (!dump_checkpoints(*state, checkpoints, dump_dir)) { - return false; - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - return true; -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [ref_dir]\n", argv[0]); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - - auto preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty()) { - fprintf(stderr, "failed to load %s/preprocessed.bin\n", ref_dir.c_str()); - return 1; - } - if (preprocessed.shape.size() != 4) { - fprintf(stderr, "unexpected preprocessed shape rank: %zu\n", preprocessed.shape.size()); - return 1; - } - - const int img_size = preprocessed.shape[2]; - std::vector checkpoints = { - "dbg_patch_embed", - "dbg_after_pos_embed", - "dbg_ln_pre_norm", - "dbg_ln_pre_scale", - "dbg_after_ln_pre", - "dbg_block_15_norm1", - "dbg_block_15_attn_out", - "dbg_block_15_attn_proj", - "dbg_block_15_resid1", - "dbg_block_15_norm2", - "dbg_block_15_mlp", - }; - for (int i = 0; i < 31; ++i) { - checkpoints.emplace_back("dbg_block_" + std::to_string(i) + "_out"); - } - checkpoints.emplace_back("vit_output"); - checkpoints.emplace_back("neck_trk_0"); - checkpoints.emplace_back("neck_trk_1"); - checkpoints.emplace_back("neck_trk_2"); - - const std::string cpu_dir = "/tmp/sam3_cpu_checkpoints"; - const std::string metal_dir = "/tmp/sam3_metal_checkpoints"; - ensure_dir(cpu_dir); - ensure_dir(metal_dir); - - double cpu_ms = 0.0; - double metal_ms = 0.0; - - fprintf(stderr, "\n=== CPU Run ===\n"); - if (!run_backend(model_path, preprocessed.data, img_size, false, 8, checkpoints, cpu_dir, cpu_ms)) { - return 1; - } - fprintf(stderr, "CPU encoder wall time: %.1f ms\n", cpu_ms); - - fprintf(stderr, "\n=== Metal Run ===\n"); - if (!run_backend(model_path, preprocessed.data, img_size, true, 8, checkpoints, metal_dir, metal_ms)) { - return 1; - } - fprintf(stderr, "Metal encoder wall time: %.1f ms\n", metal_ms); - - fprintf(stderr, "\n=== CPU vs Metal Checkpoints ===\n"); - fprintf(stderr, "%-22s %-18s %14s %14s %12s\n", - "checkpoint", "shape", "max_abs_diff", "mean_abs_diff", "n_bad"); - - std::vector stats; - stats.reserve(checkpoints.size()); - - for (const auto & name : checkpoints) { - auto cpu = load_ref_f32(cpu_dir + "/" + name); - auto metal = load_ref_f32(metal_dir + "/" + name); - if (cpu.data.empty() || metal.data.empty()) { - fprintf(stderr, "%-22s %-18s %14s %14s %12s\n", - name.c_str(), "-", "load-fail", "load-fail", "-"); - continue; - } - if (cpu.shape != metal.shape) { - fprintf(stderr, "%-22s %-18s %14s %14s %12s\n", - name.c_str(), "shape-mismatch", "shape-mismatch", "shape-mismatch", "-"); - continue; - } - - checkpoint_stat stat; - stat.name = name; - stat.shape = format_shape(cpu.shape); - stat.diff = compare_tensors(cpu.data.data(), metal.data.data(), cpu.numel(), 1e-4f); - stats.push_back(stat); - - fprintf(stderr, "%-22s %-18s %14.6f %14.6f %12d\n", - stat.name.c_str(), - stat.shape.c_str(), - stat.diff.max_diff, - stat.diff.mean_diff, - stat.diff.n_bad); - } - - const float checkpoint_tol = 1e-2f; - const checkpoint_stat * first_mismatch = nullptr; - for (const auto & stat : stats) { - if (stat.diff.max_diff > checkpoint_tol) { - first_mismatch = &stat; - break; - } - } - - if (first_mismatch) { - fprintf(stderr, - "\nFIRST CLEAR MISMATCH: %s shape=%s max_abs_diff=%.6f mean_abs_diff=%.6f\n", - first_mismatch->name.c_str(), - first_mismatch->shape.c_str(), - first_mismatch->diff.max_diff, - first_mismatch->diff.mean_diff); - return 1; - } - - fprintf(stderr, "\nAll checkpoints stayed within %.3g max abs diff.\n", checkpoint_tol); - return 0; -} diff --git a/tests/test_metal_flash_attn_ext_block15_compare.cpp b/tests/test_metal_flash_attn_ext_block15_compare.cpp deleted file mode 100644 index 2e90fae..0000000 --- a/tests/test_metal_flash_attn_ext_block15_compare.cpp +++ /dev/null @@ -1,715 +0,0 @@ -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include -#include -#include - -struct tensor_case { - ref_tensor_f32 qkv_proj; - ref_tensor_f32 freqs_cis; - ref_tensor_f32 q_heads; - ref_tensor_f32 k_heads; - ref_tensor_f32 q_rope; - ref_tensor_f32 k_rope; - ref_tensor_f32 q_flash; - ref_tensor_f32 k_flash; - ref_tensor_f32 v_base; - ref_tensor_f32 v_flash; - ref_tensor_f32 attn_out; -}; - -struct prep_run_result { - std::map tensors; -}; - -struct backend_run_result { - double elapsed_ms = 0.0; - std::map infos; -}; - -struct isolated_run_result { - double elapsed_ms = 0.0; - std::vector output; - std::vector shape; -}; - -static void usage(const char * argv0) { - fprintf(stderr, "Usage: %s [ref_dir]\n", argv0); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); -} - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static const char * ggml_type_name_public(int type) { - switch ((ggml_type) type) { - case GGML_TYPE_F32: return "F32"; - case GGML_TYPE_F16: return "F16"; - case GGML_TYPE_BF16: return "BF16"; - case GGML_TYPE_I32: return "I32"; - default: return "other"; - } -} - -static bool dump_named_tensors(const sam3_state & state, - const std::vector & names, - const std::string & output_dir, - std::map & infos) { - for (const auto & name : names) { - sam3_tensor_info info; - if (!sam3_get_state_tensor_info(state, name, info)) { - fprintf(stderr, "missing tensor info for %s\n", name.c_str()); - return false; - } - infos[name] = info; - - if (!sam3_dump_state_tensor(state, name, output_dir + "/" + name)) { - fprintf(stderr, "failed to dump %s\n", name.c_str()); - return false; - } - } - return true; -} - -static bool run_encoder_backend(const std::string & model_path, - const std::vector & chw_data, - int img_size, - bool use_gpu, - int n_threads, - const std::vector & names, - const std::string & dump_dir, - backend_run_result & result) { - sam3_params params; - params.model_path = model_path; - params.use_gpu = use_gpu; - params.n_threads = n_threads; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "failed to load model for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "failed to create state for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto t0 = std::chrono::high_resolution_clock::now(); - const bool ok = sam3_encode_image_from_preprocessed(*state, *model, chw_data.data(), img_size); - auto t1 = std::chrono::high_resolution_clock::now(); - result.elapsed_ms = std::chrono::duration(t1 - t0).count(); - if (!ok) { - fprintf(stderr, "%s encode failed\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - if (!dump_named_tensors(*state, names, dump_dir, result.infos)) { - return false; - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - return true; -} - -static ggml_backend_t create_backend(bool use_gpu, int n_threads) { -#ifdef GGML_USE_METAL - if (use_gpu) { - return ggml_backend_metal_init(); - } -#else - (void) use_gpu; -#endif - ggml_backend_t backend = ggml_backend_cpu_init(); - if (backend) { - ggml_backend_cpu_set_n_threads(backend, n_threads); - } - return backend; -} - -static std::vector normalize_shape_4d(const std::vector & shape) { - std::vector out = shape; - while (out.size() < 4) { - out.push_back(1); - } - return out; -} - -static ggml_tensor * new_tensor_4d_from_shape(ggml_context * ctx, - ggml_type type, - const std::vector & shape) { - const std::vector s = normalize_shape_4d(shape); - if (s.size() != 4) { - return nullptr; - } - return ggml_new_tensor_4d(ctx, type, s[0], s[1], s[2], s[3]); -} - -static isolated_run_result run_isolated_attention(ggml_backend_t backend, - const tensor_case & tc, - bool force_contiguous_v) { - isolated_run_result result; - - const size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context * ctx = ggml_init(params); - if (!ctx) { - return result; - } - - ggml_tensor * q = new_tensor_4d_from_shape(ctx, GGML_TYPE_F32, tc.q_flash.shape); - ggml_tensor * k = new_tensor_4d_from_shape(ctx, GGML_TYPE_F32, tc.k_flash.shape); - ggml_tensor * v_src = nullptr; - ggml_tensor * v = nullptr; - - ggml_set_input(q); - ggml_set_input(k); - if (!q || !k) { - ggml_free(ctx); - return result; - } - - if (force_contiguous_v) { - v = new_tensor_4d_from_shape(ctx, GGML_TYPE_F32, tc.v_flash.shape); - ggml_set_input(v); - } else { - v_src = new_tensor_4d_from_shape(ctx, GGML_TYPE_F32, tc.v_base.shape); - ggml_set_input(v_src); - v = ggml_permute(ctx, v_src, 0, 2, 1, 3); - } - if (!v) { - ggml_free(ctx); - return result; - } - - const float scale = 1.0f / std::sqrt((float) tc.q_flash.shape[0]); - ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, nullptr, scale, 0.0f, 0.0f); - ggml_set_output(out); - - ggml_cgraph * graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, out); - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(q, tc.q_flash.data.data(), 0, tc.q_flash.data.size() * sizeof(float)); - ggml_backend_tensor_set(k, tc.k_flash.data.data(), 0, tc.k_flash.data.size() * sizeof(float)); - if (force_contiguous_v) { - ggml_backend_tensor_set(v, tc.v_flash.data.data(), 0, tc.v_flash.data.size() * sizeof(float)); - } else { - ggml_backend_tensor_set(v_src, tc.v_base.data.data(), 0, tc.v_base.data.size() * sizeof(float)); - } - - auto t0 = std::chrono::high_resolution_clock::now(); - const ggml_status status = ggml_backend_graph_compute(backend, graph); - auto t1 = std::chrono::high_resolution_clock::now(); - result.elapsed_ms = std::chrono::duration(t1 - t0).count(); - if (status == GGML_STATUS_SUCCESS) { - result.shape = { - (int) out->ne[0], - (int) out->ne[1], - (int) out->ne[2], - (int) out->ne[3], - }; - result.output.resize((size_t) ggml_nelements(out)); - ggml_backend_tensor_get(out, result.output.data(), 0, result.output.size() * sizeof(float)); - } else { - fprintf(stderr, "isolated attention failed: %s\n", ggml_status_to_string(status)); - } - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; -} - -static ref_tensor_f32 tensor_to_ref(ggml_tensor * t) { - ref_tensor_f32 out; - out.shape = { - (int) t->ne[0], - (int) t->ne[1], - (int) t->ne[2], - (int) t->ne[3], - }; - while (!out.shape.empty() && out.shape.back() == 1) { - out.shape.pop_back(); - } - if (out.shape.empty()) { - out.shape.push_back(1); - } - - out.data.resize((size_t) ggml_nelements(t)); - ggml_backend_tensor_get(t, out.data.data(), 0, out.data.size() * sizeof(float)); - return out; -} - -static prep_run_result run_isolated_qkv_prep(ggml_backend_t backend, const tensor_case & tc) { - prep_run_result result; - - const int E = 1024; - const int NH = 16; - const int HD = 64; - const int W = 72; - const int H = 72; - const int N = W * H; - - const size_t ctx_size = ggml_tensor_overhead() * 32 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context * ctx = ggml_init(params); - if (!ctx) { - return result; - } - - ggml_tensor * qkv = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 3 * E, W, H, 1); - ggml_set_input(qkv); - ggml_set_name(qkv, "qkv_proj"); - - ggml_tensor * cur = ggml_reshape_4d(ctx, qkv, E, 3, N, 1); - cur = ggml_cont(ctx, ggml_permute(ctx, cur, 0, 3, 1, 2)); - - ggml_tensor * q_split = ggml_view_3d(ctx, cur, E, N, 1, cur->nb[1], cur->nb[2], 0); - ggml_tensor * k_split = ggml_view_3d(ctx, cur, E, N, 1, cur->nb[1], cur->nb[2], 1 * cur->nb[3]); - ggml_tensor * v_split = ggml_view_3d(ctx, cur, E, N, 1, cur->nb[1], cur->nb[2], 2 * cur->nb[3]); - ggml_set_name(q_split, "q_split"); - ggml_set_name(k_split, "k_split"); - ggml_set_name(v_split, "v_split"); - - ggml_tensor * q_heads_base = ggml_reshape_4d(ctx, q_split, HD, NH, N, 1); - ggml_tensor * k_heads_base = ggml_reshape_4d(ctx, k_split, HD, NH, N, 1); - ggml_tensor * v_heads_base = ggml_reshape_4d(ctx, v_split, HD, NH, N, 1); - ggml_set_name(q_heads_base, "q_heads_base"); - ggml_set_name(k_heads_base, "k_heads_base"); - ggml_set_name(v_heads_base, "v_heads_base"); - - ggml_tensor * q_heads = ggml_cont(ctx, ggml_permute(ctx, q_heads_base, 0, 2, 1, 3)); - ggml_tensor * k_heads = ggml_cont(ctx, ggml_permute(ctx, k_heads_base, 0, 2, 1, 3)); - ggml_tensor * v_flash = ggml_permute(ctx, v_heads_base, 0, 2, 1, 3); - ggml_set_name(q_heads, "q_heads"); - ggml_set_name(k_heads, "k_heads"); - ggml_set_name(v_flash, "v_flash"); - - std::vector outs = { - q_split, - k_split, - v_split, - q_heads_base, - k_heads_base, - v_heads_base, - q_heads, - k_heads, - v_flash, - }; - ggml_cgraph * graph = ggml_new_graph(ctx); - for (ggml_tensor * out : outs) { - ggml_set_output(out); - ggml_build_forward_expand(graph, out); - } - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(qkv, tc.qkv_proj.data.data(), 0, tc.qkv_proj.data.size() * sizeof(float)); - const ggml_status status = ggml_backend_graph_compute(backend, graph); - if (status == GGML_STATUS_SUCCESS) { - for (ggml_tensor * out : outs) { - result.tensors[out->name] = tensor_to_ref(out); - } - } else { - fprintf(stderr, "isolated qkv prep failed: %s\n", ggml_status_to_string(status)); - } - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; -} - -static isolated_run_result run_isolated_rope(ggml_backend_t backend, - const ref_tensor_f32 & x_in, - const ref_tensor_f32 & freqs_in) { - isolated_run_result result; - - const std::vector x_shape = normalize_shape_4d(x_in.shape); - const std::vector freqs_shape = normalize_shape_4d(freqs_in.shape); - if (x_shape[0] != 64 || freqs_shape[0] != 2) { - return result; - } - - const int64_t head_dim = x_shape[0]; - const int64_t N = x_shape[1]; - const int64_t nheads_B = x_shape[2] * x_shape[3]; - const int64_t half = head_dim / 2; - - const size_t ctx_size = ggml_tensor_overhead() * 32 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context * ctx = ggml_init(params); - if (!ctx) { - return result; - } - - ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, x_shape[0], x_shape[1], x_shape[2] * x_shape[3]); - ggml_tensor * freqs = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, freqs_shape[0], freqs_shape[1], freqs_shape[2] * freqs_shape[3]); - ggml_set_input(x); - ggml_set_input(freqs); - - ggml_tensor * x_pairs = ggml_reshape_4d(ctx, x, 2, half, N, nheads_B); - ggml_tensor * fc = ggml_reshape_4d(ctx, freqs, 2, half, N, 1); - - ggml_tensor * cos_f = ggml_view_4d(ctx, fc, 1, half, N, 1, fc->nb[1], fc->nb[2], fc->nb[3], 0); - ggml_tensor * sin_f = ggml_view_4d(ctx, fc, 1, half, N, 1, fc->nb[1], fc->nb[2], fc->nb[3], fc->nb[0]); - ggml_tensor * x_re = ggml_view_4d(ctx, x_pairs, 1, half, N, nheads_B, x_pairs->nb[1], x_pairs->nb[2], x_pairs->nb[3], 0); - ggml_tensor * x_im = ggml_view_4d(ctx, x_pairs, 1, half, N, nheads_B, x_pairs->nb[1], x_pairs->nb[2], x_pairs->nb[3], x_pairs->nb[0]); - - ggml_tensor * out_re = ggml_sub(ctx, ggml_mul(ctx, x_re, cos_f), ggml_mul(ctx, x_im, sin_f)); - ggml_tensor * out_im = ggml_add(ctx, ggml_mul(ctx, x_re, sin_f), ggml_mul(ctx, x_im, cos_f)); - ggml_tensor * out = ggml_concat(ctx, out_re, out_im, 0); - out = ggml_reshape_3d(ctx, ggml_cont(ctx, out), head_dim, N, nheads_B); - ggml_set_output(out); - - ggml_cgraph * graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, out); - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(x, x_in.data.data(), 0, x_in.data.size() * sizeof(float)); - ggml_backend_tensor_set(freqs, freqs_in.data.data(), 0, freqs_in.data.size() * sizeof(float)); - const ggml_status status = ggml_backend_graph_compute(backend, graph); - if (status == GGML_STATUS_SUCCESS) { - result.shape = { (int) out->ne[0], (int) out->ne[1], (int) out->ne[2] }; - result.output.resize((size_t) ggml_nelements(out)); - ggml_backend_tensor_get(out, result.output.data(), 0, result.output.size() * sizeof(float)); - } else { - fprintf(stderr, "isolated rope failed: %s\n", ggml_status_to_string(status)); - } - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; -} - -static bool load_tensor_case(const std::string & dir, tensor_case & tc) { - tc.qkv_proj = load_ref_f32(dir + "/dbg_block_15_qkv_proj"); - tc.q_heads = load_ref_f32(dir + "/dbg_block_15_q_heads"); - tc.k_heads = load_ref_f32(dir + "/dbg_block_15_k_heads"); - tc.q_rope = load_ref_f32(dir + "/dbg_block_15_q_rope"); - tc.k_rope = load_ref_f32(dir + "/dbg_block_15_k_rope"); - tc.q_flash = load_ref_f32(dir + "/dbg_block_15_q_flash"); - tc.k_flash = load_ref_f32(dir + "/dbg_block_15_k_flash"); - tc.v_base = load_ref_f32(dir + "/dbg_block_15_v_heads_base"); - tc.v_flash = load_ref_f32(dir + "/dbg_block_15_v_flash"); - tc.attn_out = load_ref_f32(dir + "/dbg_block_15_attn_out"); - - return !tc.qkv_proj.data.empty() && - !tc.q_heads.data.empty() && - !tc.k_heads.data.empty() && - !tc.q_rope.data.empty() && - !tc.k_rope.data.empty() && - !tc.q_flash.data.empty() && - !tc.k_flash.data.empty() && - !tc.v_base.data.empty() && - !tc.v_flash.data.empty() && - !tc.attn_out.data.empty(); -} - -static void print_tensor_info(const char * label, - const std::string & name, - const sam3_tensor_info & info) { - fprintf(stderr, - "%s %-24s shape=[%lld,%lld,%lld,%lld] strides=[%llu,%llu,%llu,%llu] type=%s contiguous=%d op=%d\n", - label, - name.c_str(), - (long long) info.ne[0], - (long long) info.ne[1], - (long long) info.ne[2], - (long long) info.ne[3], - (unsigned long long) info.nb[0], - (unsigned long long) info.nb[1], - (unsigned long long) info.nb[2], - (unsigned long long) info.nb[3], - ggml_type_name_public(info.type), - info.is_contiguous ? 1 : 0, - info.op); -} - -int main(int argc, char ** argv) { - if (argc < 2) { - usage(argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - - auto preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty() || preprocessed.shape.size() != 4) { - fprintf(stderr, "failed to load %s/preprocessed\n", ref_dir.c_str()); - return 1; - } - - const int img_size = preprocessed.shape[2]; - const std::vector names = { - "dbg_block_15_norm1", - "dbg_block_15_qkv_proj", - "dbg_block_15_q_split", - "dbg_block_15_k_split", - "dbg_block_15_v_split", - "dbg_block_15_q_heads_base", - "dbg_block_15_k_heads_base", - "dbg_block_15_v_heads_base", - "dbg_block_15_q_heads", - "dbg_block_15_k_heads", - "dbg_block_15_v_flash", - "dbg_block_15_q_rope", - "dbg_block_15_k_rope", - "dbg_block_15_q_flash", - "dbg_block_15_k_flash", - "dbg_block_15_attn_out", - }; - - const std::string cpu_dir = "/tmp/sam3_block15_flash_cpu"; - const std::string metal_dir = "/tmp/sam3_block15_flash_metal"; - ensure_dir(cpu_dir); - ensure_dir(metal_dir); - - backend_run_result cpu_run; - backend_run_result metal_run; - - fprintf(stderr, "\n=== Extract CPU Block-15 Attention Tensors ===\n"); - if (!run_encoder_backend(model_path, preprocessed.data, img_size, false, 8, names, cpu_dir, cpu_run)) { - return 1; - } - fprintf(stderr, "CPU extract wall time: %.1f ms\n", cpu_run.elapsed_ms); - - fprintf(stderr, "\n=== Extract Metal Block-15 Attention Tensors ===\n"); - if (!run_encoder_backend(model_path, preprocessed.data, img_size, true, 8, names, metal_dir, metal_run)) { - return 1; - } - fprintf(stderr, "Metal extract wall time: %.1f ms\n", metal_run.elapsed_ms); - - fprintf(stderr, "\n=== Exact Attention Case ===\n"); - fprintf(stderr, "block=15 variant=global_self_attention heads=16 head_dim=64 tokens=72*72=5184 batch=1 mask=none\n"); - print_tensor_info("CPU ", "dbg_block_15_q_flash", cpu_run.infos["dbg_block_15_q_flash"]); - print_tensor_info("CPU ", "dbg_block_15_k_flash", cpu_run.infos["dbg_block_15_k_flash"]); - print_tensor_info("CPU ", "dbg_block_15_v_heads_base", cpu_run.infos["dbg_block_15_v_heads_base"]); - print_tensor_info("CPU ", "dbg_block_15_v_flash", cpu_run.infos["dbg_block_15_v_flash"]); - print_tensor_info("Metal", "dbg_block_15_q_flash", metal_run.infos["dbg_block_15_q_flash"]); - print_tensor_info("Metal", "dbg_block_15_k_flash", metal_run.infos["dbg_block_15_k_flash"]); - print_tensor_info("Metal", "dbg_block_15_v_heads_base", metal_run.infos["dbg_block_15_v_heads_base"]); - print_tensor_info("Metal", "dbg_block_15_v_flash", metal_run.infos["dbg_block_15_v_flash"]); - - fprintf(stderr, "\n=== CPU vs Metal Attention Prep Checkpoints ===\n"); - fprintf(stderr, "%-24s %-18s %14s %14s %12s\n", - "checkpoint", "shape", "max_abs_diff", "mean_abs_diff", "n_bad"); - for (const auto & name : names) { - auto cpu = load_ref_f32(cpu_dir + "/" + name); - auto metal = load_ref_f32(metal_dir + "/" + name); - if (cpu.data.empty() || metal.data.empty() || cpu.shape != metal.shape) { - fprintf(stderr, "%-24s %-18s %14s %14s %12s\n", - name.c_str(), "-", "load-fail", "load-fail", "-"); - continue; - } - compare_result diff = compare_tensors(cpu.data.data(), metal.data.data(), cpu.numel(), 1e-4f); - fprintf(stderr, "%-24s %-18s %14.6f %14.6f %12d\n", - name.c_str(), format_shape(cpu.shape).c_str(), diff.max_diff, diff.mean_diff, diff.n_bad); - } - - tensor_case tc; - if (!load_tensor_case(cpu_dir, tc)) { - fprintf(stderr, "failed to load isolated attention tensors from %s\n", cpu_dir.c_str()); - return 1; - } - - { - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 8; - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "failed to reload model for freqs_cis dump\n"); - return 1; - } - const std::string freqs_path = "/tmp/sam3_block15_flash_cpu/vit.blocks.15.attn.freqs_cis"; - if (!sam3_dump_model_tensor(*model, "vit.blocks.15.attn.freqs_cis", freqs_path)) { - fprintf(stderr, "failed to dump vit.blocks.15.attn.freqs_cis\n"); - sam3_free_model(*model); - model.reset(); - return 1; - } - tc.freqs_cis = load_ref_f32(freqs_path); - sam3_free_model(*model); - model.reset(); - if (tc.freqs_cis.data.empty()) { - fprintf(stderr, "failed to load dumped freqs_cis\n"); - return 1; - } - } - - ggml_backend_t cpu_backend = create_backend(false, 8); - ggml_backend_t metal_backend = create_backend(true, 8); - if (!cpu_backend || !metal_backend) { - fprintf(stderr, "failed to initialize CPU or Metal backend\n"); - if (cpu_backend) ggml_backend_free(cpu_backend); - if (metal_backend) ggml_backend_free(metal_backend); - return 1; - } - - fprintf(stderr, "\n=== Isolated QKV Preparation From Exact qkv_proj ===\n"); - prep_run_result cpu_prep = run_isolated_qkv_prep(cpu_backend, tc); - prep_run_result metal_prep = run_isolated_qkv_prep(metal_backend, tc); - const char * prep_names[] = { - "q_split", - "k_split", - "v_split", - "q_heads_base", - "k_heads_base", - "v_heads_base", - "q_heads", - "k_heads", - "v_flash", - }; - fprintf(stderr, "%-16s %-18s %14s %14s %12s %14s\n", - "stage", "shape", "cpu_vs_dump", "metal_vs_cpu", "n_bad", "mean_diff"); - for (const char * stage : prep_names) { - const auto cpu_it = cpu_prep.tensors.find(stage); - const auto metal_it = metal_prep.tensors.find(stage); - if (cpu_it == cpu_prep.tensors.end() || metal_it == metal_prep.tensors.end()) { - fprintf(stderr, "%-16s %-18s %14s %14s %12s %14s\n", - stage, "-", "missing", "missing", "-", "-"); - continue; - } - - const std::string dump_name = std::string("dbg_block_15_") + stage; - auto cpu_dump = load_ref_f32(cpu_dir + "/" + dump_name); - compare_result cpu_vs_dump_stage = compare_tensors(cpu_it->second.data.data(), cpu_dump.data.data(), cpu_it->second.numel(), 1e-4f); - compare_result metal_vs_cpu_stage = compare_tensors(metal_it->second.data.data(), cpu_it->second.data.data(), cpu_it->second.numel(), 1e-4f); - fprintf(stderr, "%-16s %-18s %14.6f %14.6f %12d %14.6f\n", - stage, - format_shape(cpu_it->second.shape).c_str(), - cpu_vs_dump_stage.max_diff, - metal_vs_cpu_stage.max_diff, - metal_vs_cpu_stage.n_bad, - metal_vs_cpu_stage.mean_diff); - } - - fprintf(stderr, "\n=== Isolated RoPE On Exact q_heads / k_heads ===\n"); - isolated_run_result cpu_q_rope = run_isolated_rope(cpu_backend, tc.q_heads, tc.freqs_cis); - isolated_run_result metal_q_rope = run_isolated_rope(metal_backend, tc.q_heads, tc.freqs_cis); - isolated_run_result cpu_k_rope = run_isolated_rope(cpu_backend, tc.k_heads, tc.freqs_cis); - isolated_run_result metal_k_rope = run_isolated_rope(metal_backend, tc.k_heads, tc.freqs_cis); - compare_result q_rope_cpu_vs_dump = compare_tensors(cpu_q_rope.output.data(), tc.q_rope.data.data(), (int) tc.q_rope.data.size(), 1e-4f); - compare_result q_rope_metal_vs_cpu = compare_tensors(metal_q_rope.output.data(), cpu_q_rope.output.data(), (int) cpu_q_rope.output.size(), 1e-4f); - compare_result k_rope_cpu_vs_dump = compare_tensors(cpu_k_rope.output.data(), tc.k_rope.data.data(), (int) tc.k_rope.data.size(), 1e-4f); - compare_result k_rope_metal_vs_cpu = compare_tensors(metal_k_rope.output.data(), cpu_k_rope.output.data(), (int) cpu_k_rope.output.size(), 1e-4f); - fprintf(stderr, - "q_rope cpu_vs_dump(max=%.6f mean=%.6f) metal_vs_cpu(max=%.6f mean=%.6f n_bad=%d)\n", - q_rope_cpu_vs_dump.max_diff, - q_rope_cpu_vs_dump.mean_diff, - q_rope_metal_vs_cpu.max_diff, - q_rope_metal_vs_cpu.mean_diff, - q_rope_metal_vs_cpu.n_bad); - fprintf(stderr, - "k_rope cpu_vs_dump(max=%.6f mean=%.6f) metal_vs_cpu(max=%.6f mean=%.6f n_bad=%d)\n", - k_rope_cpu_vs_dump.max_diff, - k_rope_cpu_vs_dump.mean_diff, - k_rope_metal_vs_cpu.max_diff, - k_rope_metal_vs_cpu.mean_diff, - k_rope_metal_vs_cpu.n_bad); - - fprintf(stderr, "\n=== Isolated FLASH_ATTN_EXT (Exact V View) ===\n"); - isolated_run_result cpu_exact = run_isolated_attention(cpu_backend, tc, false); - isolated_run_result metal_exact = run_isolated_attention(metal_backend, tc, false); - if (cpu_exact.output.empty() || metal_exact.output.empty()) { - fprintf(stderr, "isolated exact-view attention did not produce outputs\n"); - ggml_backend_free(cpu_backend); - ggml_backend_free(metal_backend); - return 1; - } - compare_result cpu_vs_dump = compare_tensors(cpu_exact.output.data(), tc.attn_out.data.data(), (int) tc.attn_out.data.size(), 1e-4f); - compare_result metal_vs_cpu = compare_tensors(metal_exact.output.data(), cpu_exact.output.data(), (int) cpu_exact.output.size(), 1e-4f); - fprintf(stderr, - "exact_view shape=%s cpu_ms=%.3f metal_ms=%.3f cpu_vs_dump(max=%.6f mean=%.6f) metal_vs_cpu(max=%.6f mean=%.6f n_bad=%d)\n", - format_shape(cpu_exact.shape).c_str(), - cpu_exact.elapsed_ms, - metal_exact.elapsed_ms, - cpu_vs_dump.max_diff, - cpu_vs_dump.mean_diff, - metal_vs_cpu.max_diff, - metal_vs_cpu.mean_diff, - metal_vs_cpu.n_bad); - - fprintf(stderr, "\n=== Isolated FLASH_ATTN_EXT (Forced Contiguous V) ===\n"); - isolated_run_result cpu_cont = run_isolated_attention(cpu_backend, tc, true); - isolated_run_result metal_cont = run_isolated_attention(metal_backend, tc, true); - if (cpu_cont.output.empty() || metal_cont.output.empty()) { - fprintf(stderr, "isolated contiguous-V attention did not produce outputs\n"); - ggml_backend_free(cpu_backend); - ggml_backend_free(metal_backend); - return 1; - } - compare_result cont_cpu_vs_exact = compare_tensors(cpu_cont.output.data(), cpu_exact.output.data(), (int) cpu_exact.output.size(), 1e-4f); - compare_result cont_metal_vs_cpu = compare_tensors(metal_cont.output.data(), cpu_cont.output.data(), (int) cpu_cont.output.size(), 1e-4f); - fprintf(stderr, - "contiguous_v shape=%s cpu_ms=%.3f metal_ms=%.3f cpu_cont_vs_exact(max=%.6f mean=%.6f) metal_vs_cpu(max=%.6f mean=%.6f n_bad=%d)\n", - format_shape(cpu_cont.shape).c_str(), - cpu_cont.elapsed_ms, - metal_cont.elapsed_ms, - cont_cpu_vs_exact.max_diff, - cont_cpu_vs_exact.mean_diff, - cont_metal_vs_cpu.max_diff, - cont_metal_vs_cpu.mean_diff, - cont_metal_vs_cpu.n_bad); - - ggml_backend_free(cpu_backend); - ggml_backend_free(metal_backend); - return 0; -} diff --git a/tests/test_metal_overhead.cpp b/tests/test_metal_overhead.cpp deleted file mode 100644 index 8ea99c2..0000000 --- a/tests/test_metal_overhead.cpp +++ /dev/null @@ -1,377 +0,0 @@ -// Measure Metal dispatch overhead and its impact on SAM3 ViT performance. -// Proves WHY Metal is slower than CPU for the full 2524-node ViT graph. - -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include - -static double run_graph(ggml_backend_t backend, ggml_context* ctx, ggml_cgraph* graph, - const std::vector& inputs, int warmup, int repeats) { - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "FAIL: alloc\n"); - ggml_gallocr_free(galloc); - return -1; - } - for (auto* t : inputs) { - int64_t nel = ggml_nelements(t); - if (t->type == GGML_TYPE_F32) { - std::vector d(nel, 0.1f); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(float)); - } else if (t->type == GGML_TYPE_F16) { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = ggml_fp32_to_fp16(0.1f); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(uint16_t)); - } - } - if (ggml_backend_is_cpu(backend)) - ggml_backend_cpu_set_n_threads(backend, 8); - - for (int i = 0; i < warmup; i++) - ggml_backend_graph_compute(backend, graph); - auto t0 = std::chrono::high_resolution_clock::now(); - for (int i = 0; i < repeats; i++) - ggml_backend_graph_compute(backend, graph); - auto t1 = std::chrono::high_resolution_clock::now(); - ggml_gallocr_free(galloc); - return std::chrono::duration(t1 - t0).count() / repeats; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// TEST 1: Pure dispatch overhead — chain of N trivial add ops -// ═══════════════════════════════════════════════════════════════════════════════ -static void test_dispatch_overhead(ggml_backend_t cpu, ggml_backend_t metal) { - fprintf(stderr, "\n=== TEST 1: Dispatch overhead (chain of N add ops on [1024,72,72] tensors) ===\n\n"); - fprintf(stderr, " %6s %10s %10s %12s\n", "N_ops", "CPU (ms)", "Metal (ms)", "Overhead/op"); - - int counts[] = {1, 10, 50, 100, 250, 500, 1000, 2500}; - double metal_1op = 0; - - for (int N : counts) { - int max_tensors = N + 20; - int max_graph = N + 20; - - auto build_and_run = [&](ggml_backend_t backend) -> double { - size_t ctx_size = ggml_tensor_overhead() * max_tensors + ggml_graph_overhead_custom(max_graph, false); - ggml_init_params params = {ctx_size, nullptr, true}; - auto* ctx = ggml_init(params); - - auto* a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1024, 72, 72); ggml_set_input(a); - auto* b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1024, 72, 72); ggml_set_input(b); - auto* x = ggml_add(ctx, a, b); - for (int i = 1; i < N; i++) - x = ggml_add(ctx, x, b); - ggml_set_output(x); - - auto* graph = ggml_new_graph_custom(ctx, max_graph, false); - ggml_build_forward_expand(graph, x); - - std::vector inputs = {a, b}; - double ms = run_graph(backend, ctx, graph, inputs, 1, 3); - ggml_free(ctx); - return ms; - }; - - double cpu_ms = build_and_run(cpu); - double metal_ms = build_and_run(metal); - - if (N == 1) metal_1op = metal_ms; - double overhead_per_op = (N > 1) ? (metal_ms - metal_1op) / (N - 1) : 0; - - fprintf(stderr, " %6d %8.2f ms %8.2f ms %10.3f ms\n", - N, cpu_ms, metal_ms, overhead_per_op); - } -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// TEST 2: Single ViT block as ONE graph vs sum of individual ops -// ═══════════════════════════════════════════════════════════════════════════════ -static void test_vit_block_graph(ggml_backend_t cpu, ggml_backend_t metal) { - fprintf(stderr, "\n=== TEST 2: One ViT block as single graph (windowed, all ops together) ===\n\n"); - - const int E = 1024, HD = 64, NH = 16, MLP = 4736; - const int W = 24, H = 24, NP = 9; // windowed - const ggml_type WT = GGML_TYPE_F16; - - auto run_block = [&](ggml_backend_t backend) -> std::pair> { - size_t ctx_size = ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(128, false); - ggml_init_params params = {ctx_size, nullptr, true}; - auto* ctx = ggml_init(params); - std::vector inputs; - - auto inp = [&](ggml_type type, int64_t d0, int64_t d1=1, int64_t d2=1, int64_t d3=1) { - auto* t = ggml_new_tensor_4d(ctx, type, d0, d1, d2, d3); - ggml_set_input(t); - inputs.push_back(t); - return t; - }; - - // Input + residual - auto* x = inp(GGML_TYPE_F32, E, W, H, NP); - - // --- Norm1 --- - auto* norm1_w = inp(GGML_TYPE_F32, E); - auto* norm1_b = inp(GGML_TYPE_F32, E); - auto* xn = ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, 1e-6f), norm1_w), norm1_b); - - // --- QKV matmul --- - auto* qkv_w = inp(WT, E, 3*E); - auto* qkv_b = inp(GGML_TYPE_F32, 3*E); - auto* qkv = ggml_add(ctx, ggml_mul_mat(ctx, qkv_w, xn), qkv_b); - - // --- Reshape + split Q/K/V --- - int N = W * H; - qkv = ggml_reshape_4d(ctx, qkv, E, 3, N, NP); - qkv = ggml_cont(ctx, ggml_permute(ctx, qkv, 0, 3, 1, 2)); // expensive cont #1 - auto* Q = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], 0); - auto* K = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], qkv->nb[3]); - auto* V = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], 2*qkv->nb[3]); - - // --- Multi-head reshape + permute + cont (×3) --- - Q = ggml_reshape_4d(ctx, Q, HD, NH, N, NP); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); // expensive cont #2 - Q = ggml_reshape_4d(ctx, Q, HD, N, NH, NP); - - K = ggml_reshape_4d(ctx, K, HD, NH, N, NP); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); // expensive cont #3 - K = ggml_reshape_4d(ctx, K, HD, N, NH, NP); - - V = ggml_reshape_4d(ctx, V, HD, NH, N, NP); - V = ggml_permute(ctx, V, 0, 2, 1, 3); // non-contiguous view — flash_attn uses strides - - // --- Flash attention --- - auto* attn = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, 1.0f/8.0f, 0, 0); - auto* attn_r = ggml_reshape_4d(ctx, attn, E, W, H, NP); - - // --- Output proj --- - auto* proj_w = inp(WT, E, E); - auto* proj_b = inp(GGML_TYPE_F32, E); - auto* proj = ggml_add(ctx, ggml_mul_mat(ctx, proj_w, attn_r), proj_b); - - // --- Residual --- - auto* res1 = ggml_add(ctx, x, proj); - - // --- Norm2 --- - auto* norm2_w = inp(GGML_TYPE_F32, E); - auto* norm2_b = inp(GGML_TYPE_F32, E); - auto* xn2 = ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, res1, 1e-6f), norm2_w), norm2_b); - - // --- MLP --- - auto* fc1_w = inp(WT, E, MLP); - auto* fc1_b = inp(GGML_TYPE_F32, MLP); - auto* fc2_w = inp(WT, MLP, E); - auto* fc2_b = inp(GGML_TYPE_F32, E); - - auto* h = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, xn2), fc1_b); - h = ggml_gelu_erf(ctx, h); - h = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, h), fc2_b); - - // --- Residual --- - auto* out = ggml_add(ctx, res1, h); - ggml_set_output(out); - - auto* graph = ggml_new_graph_custom(ctx, 128, false); - ggml_build_forward_expand(graph, out); - - int n_nodes = ggml_graph_n_nodes(graph); - - // Allocate and fill inputs - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - ggml_gallocr_reserve(galloc, graph); - ggml_gallocr_alloc_graph(galloc, graph); - - // Fill inputs with deterministic data - for (auto* t : inputs) { - int64_t nel = ggml_nelements(t); - if (t->type == GGML_TYPE_F32) { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = 0.01f * ((j % 97) - 48); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(float)); - } else { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = ggml_fp32_to_fp16(0.01f * ((j % 97) - 48)); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(uint16_t)); - } - } - - if (ggml_backend_is_cpu(backend)) - ggml_backend_cpu_set_n_threads(backend, 8); - - // Warmup + timed run - ggml_backend_graph_compute(backend, graph); - auto t0 = std::chrono::high_resolution_clock::now(); - for (int i = 0; i < 5; i++) - ggml_backend_graph_compute(backend, graph); - auto t1 = std::chrono::high_resolution_clock::now(); - double ms = std::chrono::duration(t1 - t0).count() / 5; - - // Read output - int64_t nel = ggml_nelements(out); - std::vector output(nel); - ggml_backend_tensor_get(out, output.data(), 0, nel * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - fprintf(stderr, " %-20s %7.2f ms (%d nodes)\n", - ggml_backend_is_cpu(backend) ? "CPU" : "Metal", ms, n_nodes); - return std::make_pair(ms, output); - }; - - auto cpu_result = run_block(cpu); - auto metal_result = run_block(metal); - double cpu_block = cpu_result.first; - double metal_block = metal_result.first; - const auto& cpu_out = cpu_result.second; - const auto& metal_out = metal_result.second; - - // Compare outputs - float max_diff = 0; - int max_idx = 0; - for (size_t i = 0; i < cpu_out.size(); i++) { - float d = std::fabs(cpu_out[i] - metal_out[i]); - if (d > max_diff) { max_diff = d; max_idx = (int)i; } - } - fprintf(stderr, " Output comparison: max_diff=%.6f at [%d] (cpu=%.6f metal=%.6f) — %s\n", - max_diff, max_idx, cpu_out[max_idx], metal_out[max_idx], - max_diff < 0.01f ? "PASS" : "FAIL"); - - fprintf(stderr, " Speedup: %.2fx\n", cpu_block / metal_block); - fprintf(stderr, "\n Extrapolated to 32 layers:\n"); - fprintf(stderr, " CPU: %.0f ms\n", cpu_block * 32); - fprintf(stderr, " Metal: %.0f ms\n", metal_block * 32); - fprintf(stderr, " Speedup: %.2fx\n", (cpu_block * 32) / (metal_block * 32)); -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// TEST 3: Scale from 1 to 32 ViT blocks in ONE graph -// ═══════════════════════════════════════════════════════════════════════════════ -static void test_scaling(ggml_backend_t cpu, ggml_backend_t metal) { - fprintf(stderr, "\n=== TEST 3: Scaling — N chained matmul+add blocks in one graph ===\n"); - fprintf(stderr, " (Each block: F16 matmul [1024,1024]x[1024,24,24,9] + add + norm + gelu)\n\n"); - fprintf(stderr, " %6s %6s %10s %10s %8s\n", "Blocks", "Nodes", "CPU (ms)", "Metal (ms)", "Speedup"); - - const int E = 1024; - const ggml_type WT = GGML_TYPE_F16; - int block_counts[] = {1, 4, 8, 16, 32}; - - for (int nblocks : block_counts) { - int max_nodes = nblocks * 20 + 20; - size_t ctx_size = ggml_tensor_overhead() * (max_nodes + 50) + - ggml_graph_overhead_custom(max_nodes + 50, false); - ggml_init_params params = {ctx_size, nullptr, true}; - - auto run_on = [&](ggml_backend_t backend) -> std::pair { - auto* ctx = ggml_init(params); - std::vector inputs; - - auto inp = [&](ggml_type type, int64_t d0, int64_t d1=1, int64_t d2=1, int64_t d3=1) { - auto* t = ggml_new_tensor_4d(ctx, type, d0, d1, d2, d3); - ggml_set_input(t); - inputs.push_back(t); - return t; - }; - - auto* x = inp(GGML_TYPE_F32, E, 24, 24, 9); - - for (int i = 0; i < nblocks; i++) { - auto* w = inp(WT, E, E); - auto* b = inp(GGML_TYPE_F32, E); - auto* nw = inp(GGML_TYPE_F32, E); - auto* nb_ = inp(GGML_TYPE_F32, E); - x = ggml_add(ctx, ggml_mul_mat(ctx, w, x), b); - x = ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, 1e-6f), nw), nb_); - x = ggml_gelu_erf(ctx, x); - } - ggml_set_output(x); - - auto* graph = ggml_new_graph_custom(ctx, max_nodes + 50, false); - ggml_build_forward_expand(graph, x); - int n_nodes = ggml_graph_n_nodes(graph); - - double ms = run_graph(backend, ctx, graph, inputs, 2, 5); - ggml_free(ctx); - return {ms, n_nodes}; - }; - - double cpu_ms; int cpu_nodes; - { auto r = run_on(cpu); cpu_ms = r.first; cpu_nodes = r.second; } - double metal_ms; int metal_nodes; - { auto r = run_on(metal); metal_ms = r.first; metal_nodes = r.second; } - fprintf(stderr, " %6d %6d %8.2f ms %8.2f ms %7.2fx\n", - nblocks, cpu_nodes, cpu_ms, metal_ms, cpu_ms / metal_ms); - } -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// TEST 4: cont(permute) cost in isolation vs inside a larger graph -// ═══════════════════════════════════════════════════════════════════════════════ -static void test_cont_in_graph(ggml_backend_t cpu, ggml_backend_t metal) { - fprintf(stderr, "\n=== TEST 4: N cont(permute) ops in ONE graph ===\n"); - fprintf(stderr, " (Measures whether cont overhead compounds in large graphs)\n\n"); - fprintf(stderr, " %6s %10s %10s %10s\n", "N_cont", "CPU (ms)", "Metal (ms)", "Metal/op"); - - int counts[] = {1, 4, 16, 32, 64, 96}; - - for (int N : counts) { - int max_nodes = N * 3 + 10; - size_t ctx_size = ggml_tensor_overhead() * (max_nodes + 20) + - ggml_graph_overhead_custom(max_nodes + 20, false); - ggml_init_params params = {ctx_size, nullptr, true}; - - auto run_on = [&](ggml_backend_t backend) -> double { - auto* ctx = ggml_init(params); - std::vector inputs; - - // Chain: permute → cont → permute → cont → ... - auto* x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1024, 576, 144); - ggml_set_input(x); - inputs.push_back(x); - - ggml_tensor* last = x; - for (int i = 0; i < N; i++) { - last = ggml_cont(ctx, ggml_permute(ctx, last, 0, 2, 1, 3)); - } - ggml_set_output(last); - - auto* graph = ggml_new_graph_custom(ctx, max_nodes + 20, false); - ggml_build_forward_expand(graph, last); - double ms = run_graph(backend, ctx, graph, inputs, 2, 3); - ggml_free(ctx); - return ms; - }; - - double cpu_ms = run_on(cpu); - double metal_ms = run_on(metal); - fprintf(stderr, " %6d %8.2f ms %8.2f ms %8.2f ms\n", - N, cpu_ms, metal_ms, metal_ms / N); - } -} - -int main() { -#ifndef GGML_USE_METAL - fprintf(stderr, "Metal not available\n"); return 1; -#else - auto* cpu = ggml_backend_cpu_init(); - auto* metal = ggml_backend_metal_init(); - if (!metal) { fprintf(stderr, "No Metal\n"); return 1; } - - test_dispatch_overhead(cpu, metal); - test_vit_block_graph(cpu, metal); - test_scaling(cpu, metal); - test_cont_in_graph(cpu, metal); - - ggml_backend_free(cpu); - ggml_backend_free(metal); - return 0; -#endif -} diff --git a/tests/test_metal_phase6.cpp b/tests/test_metal_phase6.cpp deleted file mode 100644 index 0ea1c37..0000000 --- a/tests/test_metal_phase6.cpp +++ /dev/null @@ -1,195 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include - -struct phase6_case { - std::string id; - sam3_pvs_params params; -}; - -struct tensor_case { - const char * name; - const char * label; - float tol; -}; - -static std::vector parse_points(const std::string & field) { - std::vector pts; - if (field.empty()) { - return pts; - } - - size_t start = 0; - while (start < field.size()) { - size_t end = field.find('|', start); - if (end == std::string::npos) { - end = field.size(); - } - - const std::string part = field.substr(start, end - start); - const size_t colon = part.find(':'); - if (colon != std::string::npos) { - pts.push_back({std::stof(part.substr(0, colon)), std::stof(part.substr(colon + 1))}); - } - - start = end + 1; - } - - return pts; -} - -static bool parse_box(const std::string & field, sam3_box & box) { - if (field.empty()) { - return false; - } - - float vals[4]; - size_t start = 0; - for (int i = 0; i < 4; ++i) { - size_t end = field.find(':', start); - if (end == std::string::npos) { - end = field.size(); - } - vals[i] = std::stof(field.substr(start, end - start)); - start = end + 1; - } - - box = {vals[0], vals[1], vals[2], vals[3]}; - return true; -} - -static std::vector load_cases(const std::string & path) { - std::vector cases; - std::ifstream f(path); - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - - std::vector fields; - size_t start = 0; - while (start <= line.size()) { - size_t end = line.find('\t', start); - if (end == std::string::npos) { - end = line.size(); - } - fields.push_back(line.substr(start, end - start)); - start = end + 1; - if (end == line.size()) { - break; - } - } - while (fields.size() < 5) { - fields.emplace_back(); - } - - phase6_case tc; - tc.id = fields[0]; - tc.params.multimask = fields[1] == "1"; - tc.params.pos_points = parse_points(fields[2]); - tc.params.neg_points = parse_points(fields[3]); - tc.params.use_box = parse_box(fields[4], tc.params.box); - cases.push_back(std::move(tc)); - } - - return cases; -} - -static const std::vector & tensor_cases() { - static const std::vector cases = { - {"sam_dec_final_queries", "Decoder final queries", 6e-3f}, - {"sam_dec_masks", "Decoder mask logits", 2e-2f}, - {"sam_dec_iou", "Decoder IoU predictions", 2e-3f}, - {"sam_dec_obj_score", "Decoder object score logit", 3e-3f}, - {"sam_dec_sam_token", "Decoder SAM token", 6e-3f}, - }; - return cases; -} - -int main(int argc, char ** argv) { - if (argc < 5) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string prephase_ref_dir = argv[2]; - const std::string model_path = argv[3]; - const std::string cases_path = argv[4]; - const std::string cpp_root = ref_dir + "/cpp_out_phase6_metal"; - - auto cases = load_cases(cases_path); - if (cases.empty()) { - fprintf(stderr, "No phase 6 cases loaded from %s\n", cases_path.c_str()); - return 1; - } - - if (!ensure_dir(cpp_root)) { - fprintf(stderr, "Failed to create %s\n", cpp_root.c_str()); - return 1; - } - - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = true; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - bool overall_ok = true; - for (const auto & tc : cases) { - fprintf(stderr, "\n=== Metal Phase 6 Case %s ===\n", tc.id.c_str()); - - const std::string case_ref_dir = ref_dir + "/" + tc.id; - const std::string case_cpp_dir = cpp_root + "/" + tc.id; - if (!ensure_dir(case_cpp_dir)) { - fprintf(stderr, "Failed to create %s\n", case_cpp_dir.c_str()); - overall_ok = false; - continue; - } - - if (!sam3_test_dump_phase6_from_ref_inputs(*model, prephase_ref_dir, tc.params, - case_cpp_dir, params.n_threads)) { - fprintf(stderr, " [FAIL] could not run Metal phase 6 dump for %s\n", tc.id.c_str()); - overall_ok = false; - continue; - } - - for (const auto & tensor : tensor_cases()) { - auto ref = load_ref_f32(case_ref_dir + "/" + tensor.name); - auto got = load_ref_f32(case_cpp_dir + "/" + tensor.name); - if (ref.data.empty() || got.data.empty() || ref.numel() != got.numel()) { - fprintf(stderr, " [FAIL] %s missing or shape mismatch\n", tensor.name); - overall_ok = false; - continue; - } - - auto cmp = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), tensor.tol); - const bool ok = cmp.max_diff <= tensor.tol; - fprintf(stderr, - " %-22s max=%.6f mean=%.6f cos=%.8f tol=%.6f %s\n", - tensor.name, - cmp.max_diff, - cmp.mean_diff, - cmp.cosine_sim, - tensor.tol, - ok ? "PASS" : "FAIL"); - if (!ok) { - overall_ok = false; - } - } - } - - sam3_free_model(*model); - return overall_ok ? 0 : 1; -} diff --git a/tests/test_metal_profile.cpp b/tests/test_metal_profile.cpp deleted file mode 100644 index 730f403..0000000 --- a/tests/test_metal_profile.cpp +++ /dev/null @@ -1,215 +0,0 @@ -// Profile individual ggml ops at SAM3 ViT shapes on CPU vs Metal. -// Each op gets its own tiny graph — runs in seconds, not minutes. - -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include -#include - -using build_fn = std::function>(ggml_context*)>; - -static double run_bench(ggml_backend_t backend, build_fn build, - int warmup = 2, int repeats = 5) { - size_t ctx_size = ggml_tensor_overhead() * 64 + ggml_graph_overhead(); - struct ggml_init_params params = {ctx_size, nullptr, true}; - auto* ctx = ggml_init(params); - - auto result = build(ctx); - auto* out = result.first; - auto& inputs = result.second; - ggml_set_output(out); - - auto* graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, out); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "FAIL: alloc\n"); - ggml_gallocr_free(galloc); ggml_free(ctx); - return -1; - } - - for (auto* t : inputs) { - int64_t nel = ggml_nelements(t); - if (t->type == GGML_TYPE_F32) { - std::vector d(nel, 0.1f); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(float)); - } else if (t->type == GGML_TYPE_F16) { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = ggml_fp32_to_fp16(0.1f); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(uint16_t)); - } - } - - if (ggml_backend_is_cpu(backend)) - ggml_backend_cpu_set_n_threads(backend, 8); - - for (int i = 0; i < warmup; i++) - ggml_backend_graph_compute(backend, graph); - - auto t0 = std::chrono::high_resolution_clock::now(); - for (int i = 0; i < repeats; i++) - ggml_backend_graph_compute(backend, graph); - auto t1 = std::chrono::high_resolution_clock::now(); - - double ms = std::chrono::duration(t1 - t0).count() / repeats; - ggml_gallocr_free(galloc); - ggml_free(ctx); - return ms; -} - -struct op_bench { const char* name; int count; double cpu_ms; double metal_ms; }; - -// Helper macros for input tensor creation -#define INP_F32(...) [&]{ auto* t = ggml_new_tensor(__VA_ARGS__); ggml_set_name(t,"i"); ggml_set_input(t); return t; }() -#define INP(ctx, type, ...) [&]{ int64_t ne[] = {__VA_ARGS__}; auto* t = ggml_new_tensor(ctx, type, sizeof(ne)/sizeof(ne[0]), ne); ggml_set_input(t); return t; }() - -int main() { -#ifndef GGML_USE_METAL - fprintf(stderr, "Metal not available\n"); return 1; -#else - auto* cpu = ggml_backend_cpu_init(); - auto* metal = ggml_backend_metal_init(); - if (!metal) { fprintf(stderr, "No Metal\n"); return 1; } - - const int E = 1024, MLP = 4736, HD = 64, NH = 16; - const int W_win = 24, H_win = 24, NP = 9; - const int W_glb = 72, H_glb = 72; - const int N_win = W_win * H_win; - const int N_glb = W_glb * H_glb; - const ggml_type WT = GGML_TYPE_F16; - - std::vector results; - - auto bench = [&](const char* name, int count, build_fn build) { - double cpu_ms = run_bench(cpu, build); - double metal_ms = run_bench(metal, build); - results.push_back({name, count, cpu_ms, metal_ms}); - fprintf(stderr, " %-45s CPU:%7.2f ms Metal:%7.2f ms (x%d → CPU %5.0f / Metal %5.0f)\n", - name, cpu_ms, metal_ms, count, cpu_ms*count, metal_ms*count); - }; - - fprintf(stderr, "\n=== SAM3 ViT Op Profiling (F16 weights, F32 activations, CPU 8 threads) ===\n\n"); - - // ── Matmuls (windowed: 28 layers × 4 = 112) ──────────────────────── - bench("matmul QKV win [1024,3072]x[1024,24,24,9]", 28, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, E, 3*E); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - bench("matmul proj win [1024,1024]x[1024,24,24,9]", 28, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, E, E); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - bench("matmul fc1 win [1024,4736]x[1024,24,24,9]", 28, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, E, MLP); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - bench("matmul fc2 win [4736,1024]x[4736,24,24,9]", 28, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, MLP, E); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, MLP, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - - // ── Matmuls (global: 4 layers × 4 = 16) ──────────────────────────── - bench("matmul QKV glb [1024,3072]x[1024,72,72,1]", 4, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, E, 3*E); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_glb, H_glb, 1); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - bench("matmul fc1 glb [1024,4736]x[1024,72,72,1]", 4, [&](ggml_context* c) { - auto* w = ggml_new_tensor_2d(c, WT, E, MLP); ggml_set_input(w); - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_glb, H_glb, 1); ggml_set_input(x); - return std::make_pair(ggml_mul_mat(c, w, x), std::vector{w, x}); - }); - - // ── Flash attention ───────────────────────────────────────────────── - bench("flash_attn win HD=64 N=576 NH=16 B=9", 28, [&](ggml_context* c) { - auto* Q = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_win, NH, NP); ggml_set_input(Q); - auto* K = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_win, NH, NP); ggml_set_input(K); - auto* V = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_win, NH, NP); ggml_set_input(V); - return std::make_pair(ggml_flash_attn_ext(c, Q, K, V, nullptr, 1.0f/8.0f, 0, 0), - std::vector{Q, K, V}); - }); - bench("flash_attn glb HD=64 N=5184 NH=16 B=1", 4, [&](ggml_context* c) { - auto* Q = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_glb, NH, 1); ggml_set_input(Q); - auto* K = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_glb, NH, 1); ggml_set_input(K); - auto* V = ggml_new_tensor_4d(c, GGML_TYPE_F32, HD, N_glb, NH, 1); ggml_set_input(V); - return std::make_pair(ggml_flash_attn_ext(c, Q, K, V, nullptr, 1.0f/8.0f, 0, 0), - std::vector{Q, K, V}); - }); - - // ── Layer norm (norm + mul + add) ─────────────────────────────────── - bench("layer_norm [1024, 72, 72]", 64, [&](ggml_context* c) { - auto* x = ggml_new_tensor_3d(c, GGML_TYPE_F32, E, W_glb, H_glb); ggml_set_input(x); - auto* w = ggml_new_tensor_1d(c, GGML_TYPE_F32, E); ggml_set_input(w); - auto* b = ggml_new_tensor_1d(c, GGML_TYPE_F32, E); ggml_set_input(b); - auto* n = ggml_add(c, ggml_mul(c, ggml_norm(c, x, 1e-6f), w), b); - return std::make_pair(n, std::vector{x, w, b}); - }); - - // ── GELU ──────────────────────────────────────────────────────────── - bench("gelu_erf [4736, 24, 24, 9]", 28, [&](ggml_context* c) { - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, MLP, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_gelu_erf(c, x), std::vector{x}); - }); - - // ── Add ───────────────────────────────────────────────────────────── - bench("add [1024, 72, 72]", 128, [&](ggml_context* c) { - auto* a = ggml_new_tensor_3d(c, GGML_TYPE_F32, E, W_glb, H_glb); ggml_set_input(a); - auto* b = ggml_new_tensor_3d(c, GGML_TYPE_F32, E, W_glb, H_glb); ggml_set_input(b); - return std::make_pair(ggml_add(c, a, b), std::vector{a, b}); - }); - - // ── Win part / unpart ─────────────────────────────────────────────── - bench("win_part [1024,72,72] -> [1024,24,24,9]", 28, [&](ggml_context* c) { - auto* x = ggml_new_tensor_3d(c, GGML_TYPE_F32, E, W_glb, H_glb); ggml_set_input(x); - return std::make_pair(ggml_win_part(c, x, 24), std::vector{x}); - }); - bench("win_unpart [1024,24,24,9] -> [1024,72,72]", 28, [&](ggml_context* c) { - auto* x = ggml_new_tensor_4d(c, GGML_TYPE_F32, E, W_win, H_win, NP); ggml_set_input(x); - return std::make_pair(ggml_win_unpart(c, x, W_glb, H_glb, 24), std::vector{x}); - }); - - // ── Cont (permute + make contiguous — happens ~96x in ViT) ────────── - bench("cont(permute) [1024, 576, 144]", 96, [&](ggml_context* c) { - auto* x = ggml_new_tensor_3d(c, GGML_TYPE_F32, E, N_win, NH*NP); ggml_set_input(x); - return std::make_pair(ggml_cont(c, ggml_permute(c, x, 0, 2, 1, 3)), - std::vector{x}); - }); - - // ── Summary ───────────────────────────────────────────────────────── - fprintf(stderr, "\n=== Estimated Total ViT Cost ===\n\n"); - fprintf(stderr, " %-45s %10s %10s %8s\n", "Op", "CPU total", "Metal total", "Speedup"); - fprintf(stderr, " %-45s %10s %10s %8s\n", - "---------------------------------------------", "---------", "-----------", "-------"); - - double total_cpu = 0, total_metal = 0; - for (auto& r : results) { - double ct = r.cpu_ms * r.count; - double mt = r.metal_ms * r.count; - total_cpu += ct; - total_metal += mt; - fprintf(stderr, " %-45s %7.0f ms %9.0f ms %7.2fx\n", r.name, ct, mt, ct/mt); - } - fprintf(stderr, " %-45s %7.0f ms %9.0f ms %7.2fx\n", - "TOTAL (estimated ViT)", total_cpu, total_metal, total_cpu/total_metal); - - ggml_backend_free(cpu); - ggml_backend_free(metal); - return 0; -#endif -} diff --git a/tests/test_metal_python_ref_compare.cpp b/tests/test_metal_python_ref_compare.cpp deleted file mode 100644 index fa5cf29..0000000 --- a/tests/test_metal_python_ref_compare.cpp +++ /dev/null @@ -1,503 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include -#include -#include -#include - -struct metric_row { - std::string tensor; - std::string shape; - float max_abs = 0.0f; - float mean_abs = 0.0f; - float mean_rel = 0.0f; - float cosine = 0.0f; - bool measured = false; - std::string note; -}; - -struct backend_run { - std::string backend; - double encode_ms = 0.0; - double final_ms = 0.0; - std::vector encoder_rows; - std::vector final_rows; -}; - -static std::string shape_to_string(const std::vector & shape) { - std::string s; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) s += "x"; - s += std::to_string(shape[i]); - } - return s; -} - -static metric_row compute_metrics(const std::string & tensor, - const float * got, - const float * ref, - int n, - const std::vector & ref_shape) { - metric_row row; - row.tensor = tensor; - row.shape = shape_to_string(ref_shape); - if (n <= 0) { - row.note = "empty"; - return row; - } - - double sum_abs = 0.0; - double sum_rel = 0.0; - double dot_ab = 0.0; - double dot_aa = 0.0; - double dot_bb = 0.0; - float max_abs = 0.0f; - - for (int i = 0; i < n; ++i) { - const float a = got[i]; - const float b = ref[i]; - const float d = fabsf(a - b); - sum_abs += d; - sum_rel += d / (fabsf(b) + 1e-8f); - if (d > max_abs) { - max_abs = d; - } - dot_ab += (double) a * (double) b; - dot_aa += (double) a * (double) a; - dot_bb += (double) b * (double) b; - } - - row.max_abs = max_abs; - row.mean_abs = (float) (sum_abs / n); - row.mean_rel = (float) (sum_rel / n); - const double denom = sqrt(dot_aa) * sqrt(dot_bb); - row.cosine = denom > 0.0 ? (float) (dot_ab / denom) : 0.0f; - row.measured = true; - return row; -} - -static std::vector ggml_dump_to_nhwc_flat(const ref_tensor_f32 & cpp) { - std::vector out; - if (cpp.shape.size() < 3) { - return out; - } - - const int E = cpp.shape[0]; - const int W = cpp.shape[1]; - const int H = cpp.shape[2]; - out.resize((size_t) E * W * H); - for (int h = 0; h < H; ++h) { - for (int w = 0; w < W; ++w) { - for (int e = 0; e < E; ++e) { - out[(size_t) h * W * E + (size_t) w * E + e] = - cpp.data[(size_t) e + (size_t) w * E + (size_t) h * E * W]; - } - } - } - return out; -} - -static std::vector ggml_dump_to_nchw_flat(const ref_tensor_f32 & cpp) { - std::vector out; - if (cpp.shape.size() < 3) { - return out; - } - - const int C = cpp.shape[0]; - const int W = cpp.shape[1]; - const int H = cpp.shape[2]; - out.resize((size_t) C * W * H); - for (int c = 0; c < C; ++c) { - for (int h = 0; h < H; ++h) { - for (int w = 0; w < W; ++w) { - out[(size_t) c * H * W + (size_t) h * W + w] = - cpp.data[(size_t) c + (size_t) w * C + (size_t) h * C * W]; - } - } - } - return out; -} - -static bool parse_phase6_case(const std::string & path, - sam3_pvs_params & params, - std::string & case_id) { - FILE * f = fopen(path.c_str(), "r"); - if (!f) { - return false; - } - - char line[1024]; - if (!fgets(line, sizeof(line), f)) { - fclose(f); - return false; - } - fclose(f); - - std::string row(line); - while (!row.empty() && (row.back() == '\n' || row.back() == '\r')) { - row.pop_back(); - } - - std::vector fields; - size_t start = 0; - while (start <= row.size()) { - size_t end = row.find('\t', start); - if (end == std::string::npos) { - end = row.size(); - } - fields.emplace_back(row.substr(start, end - start)); - start = end + 1; - if (end == row.size()) { - break; - } - } - while (fields.size() < 5) { - fields.emplace_back(); - } - - case_id = fields[0]; - params.multimask = fields[1] == "1"; - params.use_box = false; - - if (!fields[4].empty()) { - float vals[4] = {0, 0, 0, 0}; - size_t start = 0; - for (int i = 0; i < 4; ++i) { - size_t end = fields[4].find(':', start); - if (end == std::string::npos) end = fields[4].size(); - vals[i] = std::stof(fields[4].substr(start, end - start)); - start = end + 1; - } - params.box = {vals[0], vals[1], vals[2], vals[3]}; - params.use_box = true; - } - - auto parse_points = [](const std::string & field, std::vector & dst) { - size_t start = 0; - while (start < field.size()) { - size_t end = field.find('|', start); - if (end == std::string::npos) end = field.size(); - const std::string part = field.substr(start, end - start); - size_t sep = part.find(':'); - if (sep != std::string::npos) { - sam3_point p; - p.x = std::stof(part.substr(0, sep)); - p.y = std::stof(part.substr(sep + 1)); - dst.push_back(p); - } - start = end + 1; - } - }; - - parse_points(fields[2], params.pos_points); - parse_points(fields[3], params.neg_points); - return true; -} - -static metric_row compare_state_nhwc(const sam3_state & state, - const std::string & tensor_name, - const std::string & ref_path, - const std::string & dump_dir) { - metric_row row; - auto ref = load_ref_f32(ref_path); - row.tensor = tensor_name; - row.shape = shape_to_string(ref.shape); - - if (ref.data.empty()) { - row.note = "missing python ref"; - return row; - } - if (!sam3_dump_state_tensor(state, tensor_name, dump_dir + "/" + tensor_name)) { - row.note = "missing state tensor"; - return row; - } - auto cpp = load_ref_f32(dump_dir + "/" + tensor_name); - if (cpp.data.empty()) { - row.note = "missing dumped tensor"; - return row; - } - - auto transposed = ggml_dump_to_nhwc_flat(cpp); - if ((int) transposed.size() != ref.numel()) { - row.note = "numel mismatch"; - return row; - } - return compute_metrics(tensor_name, transposed.data(), ref.data.data(), ref.numel(), ref.shape); -} - -static metric_row compare_state_nchw(const sam3_state & state, - const std::string & tensor_name, - const std::string & ref_path, - const std::string & dump_dir) { - metric_row row; - auto ref = load_ref_f32(ref_path); - row.tensor = tensor_name; - row.shape = shape_to_string(ref.shape); - - if (ref.data.empty()) { - row.note = "missing python ref"; - return row; - } - if (!sam3_dump_state_tensor(state, tensor_name, dump_dir + "/" + tensor_name)) { - row.note = "missing state tensor"; - return row; - } - auto cpp = load_ref_f32(dump_dir + "/" + tensor_name); - if (cpp.data.empty()) { - row.note = "missing dumped tensor"; - return row; - } - - auto transposed = ggml_dump_to_nchw_flat(cpp); - if ((int) transposed.size() != ref.numel()) { - row.note = "numel mismatch"; - return row; - } - return compute_metrics(tensor_name, transposed.data(), ref.data.data(), ref.numel(), ref.shape); -} - -static bool save_ref_f32(const std::string & path, - const std::vector & data, - const std::vector & shape) { - FILE * shape_f = fopen((path + ".shape").c_str(), "w"); - if (!shape_f) { - return false; - } - for (size_t i = 0; i < shape.size(); ++i) { - fprintf(shape_f, "%s%d", i == 0 ? "" : ",", shape[i]); - } - fclose(shape_f); - - FILE * data_f = fopen((path + ".bin").c_str(), "wb"); - if (!data_f) { - return false; - } - const size_t n_written = fwrite(data.data(), sizeof(float), data.size(), data_f); - fclose(data_f); - return n_written == data.size(); -} - -static bool export_dump_as_nchw_ref(const std::string & ggml_dump_path, - const std::string & out_path) { - auto cpp = load_ref_f32(ggml_dump_path); - if (cpp.data.empty() || cpp.shape.size() < 3) { - return false; - } - const std::vector transposed = ggml_dump_to_nchw_flat(cpp); - const std::vector shape = {1, cpp.shape[0], cpp.shape[2], cpp.shape[1]}; - return save_ref_f32(out_path, transposed, shape); -} - -static metric_row compare_dump_direct(const std::string & name, - const std::string & got_path, - const std::string & ref_path) { - metric_row row; - auto got = load_ref_f32(got_path); - auto ref = load_ref_f32(ref_path); - row.tensor = name; - row.shape = shape_to_string(ref.shape); - - if (got.data.empty() || ref.data.empty()) { - row.note = "missing dump"; - return row; - } - if (got.numel() != ref.numel()) { - row.note = "numel mismatch"; - return row; - } - return compute_metrics(name, got.data.data(), ref.data.data(), ref.numel(), ref.shape); -} - -static backend_run run_backend(const std::string & backend_name, - bool use_gpu, - const std::string & model_path, - const std::string & ref_dir, - const sam3_pvs_params & pvs_params, - const std::string & phase6_case_id) { - backend_run run; - run.backend = backend_name; - - sam3_params params; - params.model_path = model_path; - params.n_threads = 4; - params.use_gpu = use_gpu; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model for backend=%s\n", backend_name.c_str()); - return run; - } - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state for backend=%s\n", backend_name.c_str()); - return run; - } - - auto pre = load_ref_f32(ref_dir + "/preprocessed"); - if (pre.data.empty()) { - fprintf(stderr, "Missing preprocessed tensor in %s\n", ref_dir.c_str()); - return run; - } - - const std::string dump_root = "/tmp/sam3_pyref_" + backend_name; - ensure_dir(dump_root); - - auto t0 = std::chrono::high_resolution_clock::now(); - const bool ok = sam3_encode_image_from_preprocessed(*state, *model, pre.data.data(), 1008); - auto t1 = std::chrono::high_resolution_clock::now(); - run.encode_ms = std::chrono::duration(t1 - t0).count(); - if (!ok) { - fprintf(stderr, "sam3_encode_image_from_preprocessed failed for backend=%s\n", backend_name.c_str()); - return run; - } - - const char * nhwc_tensors[] = { - "dbg_patch_embed", - "dbg_after_pos_embed", - "dbg_after_ln_pre", - "dbg_block_0_out", - "dbg_block_1_out", - "dbg_block_2_out", - "dbg_block_10_out", - "dbg_block_14_out", - "dbg_block_15_out", - "dbg_block_16_out", - }; - for (const char * name : nhwc_tensors) { - std::string ref_name = name; - if (strncmp(name, "dbg_", 4) == 0) { - ref_name = std::string(name + 4); - } - run.encoder_rows.push_back(compare_state_nhwc(*state, name, ref_dir + "/" + ref_name, dump_root)); - } - - run.encoder_rows.push_back(compare_state_nchw(*state, "vit_output", ref_dir + "/vit_output_bchw", dump_root)); - run.encoder_rows.push_back(compare_state_nchw(*state, "neck_trk_0", ref_dir + "/neck_trk_0", dump_root)); - run.encoder_rows.push_back(compare_state_nchw(*state, "neck_trk_1", ref_dir + "/neck_trk_1", dump_root)); - run.encoder_rows.push_back(compare_state_nchw(*state, "neck_trk_2", ref_dir + "/neck_trk_2", dump_root)); - - const std::string phase6_inputs = dump_root + "/phase6_inputs"; - const std::string phase6_dump = dump_root + "/phase6_out"; - ensure_dir(phase6_inputs); - ensure_dir(phase6_dump); - - if (!export_dump_as_nchw_ref(dump_root + "/neck_trk_0", phase6_inputs + "/neck_trk_0") || - !export_dump_as_nchw_ref(dump_root + "/neck_trk_1", phase6_inputs + "/neck_trk_1") || - !export_dump_as_nchw_ref(dump_root + "/neck_trk_2", phase6_inputs + "/neck_trk_2")) { - metric_row row; - row.tensor = "phase6_bridge"; - row.note = "failed to export neck_trk_* as NCHW ref inputs"; - run.final_rows.push_back(std::move(row)); - return run; - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - - sam3_params p6_params; - p6_params.model_path = model_path; - p6_params.n_threads = 4; - p6_params.use_gpu = false; - - auto phase6_model = sam3_load_model(p6_params); - if (!phase6_model) { - metric_row row; - row.tensor = "phase6_model"; - row.note = "failed to load CPU model for phase6 compare"; - run.final_rows.push_back(std::move(row)); - return run; - } - - auto t2 = std::chrono::high_resolution_clock::now(); - const bool phase6_ok = sam3_test_dump_phase6_from_ref_inputs(*phase6_model, phase6_inputs, - pvs_params, phase6_dump, - p6_params.n_threads); - auto t3 = std::chrono::high_resolution_clock::now(); - run.final_ms = std::chrono::duration(t3 - t2).count(); - if (!phase6_ok) { - metric_row row; - row.tensor = "phase6_compare"; - row.note = "sam3_test_dump_phase6_from_ref_inputs failed"; - run.final_rows.push_back(std::move(row)); - return run; - } - - const std::string p6_ref_dir = ref_dir + "/phase6/" + phase6_case_id; - const char * final_tensors[] = { - "sam_dec_masks", - "sam_dec_iou", - "sam_dec_obj_score", - "sam_dec_sam_token", - }; - for (const char * name : final_tensors) { - run.final_rows.push_back(compare_dump_direct(name, phase6_dump + "/" + name, p6_ref_dir + "/" + name)); - } - - return run; -} - -static void print_table(const std::string & title, const std::vector & rows) { - fprintf(stderr, "\n%s\n", title.c_str()); - fprintf(stderr, "| tensor | shape | max abs diff | mean abs diff | mean rel err | cosine | note |\n"); - fprintf(stderr, "|--------|-------|--------------|---------------|--------------|--------|------|\n"); - for (const auto & row : rows) { - if (!row.measured) { - fprintf(stderr, "| %s | %s | n/a | n/a | n/a | n/a | %s |\n", - row.tensor.c_str(), row.shape.c_str(), row.note.c_str()); - continue; - } - fprintf(stderr, "| %s | %s | %.6e | %.6e | %.6e | %.8f | %s |\n", - row.tensor.c_str(), row.shape.c_str(), row.max_abs, - row.mean_abs, row.mean_rel, row.cosine, row.note.c_str()); - } -} - -static void print_summary(const backend_run & run) { - fprintf(stderr, "\n=== %s vs Python ===\n", run.backend.c_str()); - fprintf(stderr, "encode_ms=%.3f final_ms=%.3f\n", run.encode_ms, run.final_ms); - print_table("Encoder / Neck", run.encoder_rows); - print_table("Final Decoder Outputs", run.final_rows); -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, - "Usage: %s [ref_dir] [backend=metal|cpu|both] [phase6_case_tsv]\n", - argv[0]); - fprintf(stderr, - "Defaults: ref_dir=tests/debug_pipeline/e2e_ref backend=both phase6_case_tsv=tests/debug_pipeline/phase6_cat_box.tsv\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/debug_pipeline/e2e_ref"; - const std::string backend_mode = argc >= 4 ? argv[3] : "both"; - const std::string case_tsv = argc >= 5 ? argv[4] : "tests/debug_pipeline/phase6_cat_box.tsv"; - - sam3_pvs_params pvs_params; - std::string phase6_case_id; - if (!parse_phase6_case(case_tsv, pvs_params, phase6_case_id)) { - fprintf(stderr, "Failed to parse phase 6 case file: %s\n", case_tsv.c_str()); - return 1; - } - - std::vector runs; - if (backend_mode == "cpu" || backend_mode == "both") { - runs.push_back(run_backend("cpu", false, model_path, ref_dir, pvs_params, phase6_case_id)); - } - if (backend_mode == "metal" || backend_mode == "both") { - runs.push_back(run_backend("metal", true, model_path, ref_dir, pvs_params, phase6_case_id)); - } - - for (const auto & run : runs) { - print_summary(run); - } - - return 0; -} diff --git a/tests/test_metal_real_profile.cpp b/tests/test_metal_real_profile.cpp deleted file mode 100644 index c38d8b1..0000000 --- a/tests/test_metal_real_profile.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// Profile the REAL SAM3 ViT graph on Metal. -// Runs encode_image twice to separate first-run compilation from steady-state. -// Also breaks down graph_compute into encoding vs GPU execution. - -#include "sam3.h" -#include "ggml-backend.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include - -static double now_ms() { - return std::chrono::duration( - std::chrono::high_resolution_clock::now().time_since_epoch()).count(); -} - -int main(int argc, char** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - auto img = sam3_load_image(argv[2]); - fprintf(stderr, "Image: %dx%d\n\n", img.width, img.height); - - // ── Metal: run encode_image TWICE ─────────────────────────────────── - fprintf(stderr, "=== Metal: Loading model ===\n"); - sam3_params params; - params.model_path = argv[1]; - params.use_gpu = true; - params.n_threads = 8; - - double t0 = now_ms(); - auto model = sam3_load_model(params); - double t1 = now_ms(); - fprintf(stderr, " Model load: %.0f ms\n", t1 - t0); - - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - // Run 1: includes pipeline compilation - fprintf(stderr, "\n=== Metal: encode_image RUN 1 (cold — includes pipeline compilation) ===\n"); - t0 = now_ms(); - bool ok = sam3_encode_image(*state, *model, img); - t1 = now_ms(); - fprintf(stderr, " Run 1 total: %.0f ms %s\n", t1 - t0, ok ? "" : "FAILED"); - - // Read back a tensor to force GPU synchronization - { - double ts0 = now_ms(); - sam3_dump_state_tensor(*state, "vit_output", "/tmp/sam3_metal_prof_vit"); - double ts1 = now_ms(); - fprintf(stderr, " Readback sync: %.0f ms\n", ts1 - ts0); - } - - // Run 2: pipelines already compiled, GPU warmed up - fprintf(stderr, "\n=== Metal: encode_image RUN 2 (warm — no compilation) ===\n"); - t0 = now_ms(); - ok = sam3_encode_image(*state, *model, img); - t1 = now_ms(); - fprintf(stderr, " Run 2 total: %.0f ms %s\n", t1 - t0, ok ? "" : "FAILED"); - - { - double ts0 = now_ms(); - sam3_dump_state_tensor(*state, "vit_output", "/tmp/sam3_metal_prof_vit2"); - double ts1 = now_ms(); - fprintf(stderr, " Readback sync: %.0f ms\n", ts1 - ts0); - } - - // Run 3: one more to confirm steady state - fprintf(stderr, "\n=== Metal: encode_image RUN 3 (warm) ===\n"); - t0 = now_ms(); - ok = sam3_encode_image(*state, *model, img); - t1 = now_ms(); - fprintf(stderr, " Run 3 total: %.0f ms %s\n", t1 - t0, ok ? "" : "FAILED"); - - { - double ts0 = now_ms(); - sam3_dump_state_tensor(*state, "vit_output", "/tmp/sam3_metal_prof_vit3"); - double ts1 = now_ms(); - fprintf(stderr, " Readback sync: %.0f ms\n", ts1 - ts0); - } - - // ── CPU: for comparison ───────────────────────────────────────────── - fprintf(stderr, "\n=== CPU: encode_image (for comparison) ===\n"); - { - sam3_params cpu_params; - cpu_params.model_path = argv[1]; - cpu_params.use_gpu = false; - cpu_params.n_threads = 8; - - auto cpu_model = sam3_load_model(cpu_params); - auto cpu_state = sam3_create_state(*cpu_model, cpu_params); - - t0 = now_ms(); - ok = sam3_encode_image(*cpu_state, *cpu_model, img); - t1 = now_ms(); - fprintf(stderr, " CPU total: %.0f ms %s\n", t1 - t0, ok ? "" : "FAILED"); - } - - return 0; -} diff --git a/tests/test_metal_vit.cpp b/tests/test_metal_vit.cpp deleted file mode 100644 index 9b5fef6..0000000 --- a/tests/test_metal_vit.cpp +++ /dev/null @@ -1,166 +0,0 @@ -// Test Metal ViT encoding: compare CPU vs Metal image encoder outputs. -// Usage: test_metal_vit - -#include "sam3.h" - -#include "ggml-backend.h" -#include "ggml.h" - -#include -#include -#include -#include -#include -#include - -int main(int argc, char** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string image_path = argv[2]; - - // Load image once - auto img = sam3_load_image(image_path); - if (img.data.empty()) { - fprintf(stderr, "Failed to load image: %s\n", image_path.c_str()); - return 1; - } - fprintf(stderr, "Loaded %dx%d image\n", img.width, img.height); - - // ── Run CPU encoding ──────────────────────────────────────────────── - fprintf(stderr, "\n=== CPU Encoding ===\n"); - std::vector cpu_neck0, cpu_neck1, cpu_neck2; - { - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 8; - - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model (CPU)\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state (CPU)\n"); return 1; } - - auto t0 = std::chrono::high_resolution_clock::now(); - bool ok = sam3_encode_image(*state, *model, img); - auto t1 = std::chrono::high_resolution_clock::now(); - double ms = std::chrono::duration(t1 - t0).count(); - - if (!ok) { fprintf(stderr, "CPU encode_image failed\n"); return 1; } - fprintf(stderr, " CPU encode_image: %.1f ms\n", ms); - - // Dump tensors via the debug interface - sam3_dump_state_tensor(*state, "vit_output", "/tmp/sam3_cpu_vit"); - sam3_dump_state_tensor(*state, "neck_trk_0", "/tmp/sam3_cpu_neck0"); - sam3_dump_state_tensor(*state, "neck_trk_1", "/tmp/sam3_cpu_neck1"); - sam3_dump_state_tensor(*state, "neck_trk_2", "/tmp/sam3_cpu_neck2"); - } - - // ── Run Metal encoding ────────────────────────────────────────────── - fprintf(stderr, "\n=== Metal Encoding ===\n"); - { - sam3_params params; - params.model_path = model_path; - params.use_gpu = true; - params.n_threads = 8; - - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model (Metal)\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state (Metal)\n"); return 1; } - - auto t0 = std::chrono::high_resolution_clock::now(); - bool ok = sam3_encode_image(*state, *model, img); - auto t1 = std::chrono::high_resolution_clock::now(); - double ms = std::chrono::duration(t1 - t0).count(); - - if (!ok) { fprintf(stderr, "Metal encode_image failed\n"); return 1; } - fprintf(stderr, " Metal encode_image: %.1f ms\n", ms); - - sam3_dump_state_tensor(*state, "vit_output", "/tmp/sam3_metal_vit"); - sam3_dump_state_tensor(*state, "neck_trk_0", "/tmp/sam3_metal_neck0"); - sam3_dump_state_tensor(*state, "neck_trk_1", "/tmp/sam3_metal_neck1"); - sam3_dump_state_tensor(*state, "neck_trk_2", "/tmp/sam3_metal_neck2"); - } - - // ── Compare outputs ───────────────────────────────────────────────── - fprintf(stderr, "\n=== Comparing CPU vs Metal ===\n"); - bool all_pass = true; - - auto compare_dumps = [&](const std::string& name, - const std::string& cpu_path, - const std::string& metal_path, - float atol) { - // Load shapes - auto load_bin = [](const std::string& path, std::vector& shape) -> std::vector { - // Read shape - { - FILE* f = fopen((path + ".shape").c_str(), "r"); - if (!f) return {}; - char buf[256]; - if (fgets(buf, sizeof(buf), f)) { - char* tok = strtok(buf, ",\n"); - while (tok) { - shape.push_back(atoi(tok)); - tok = strtok(nullptr, ",\n"); - } - } - fclose(f); - } - // Read data - int n = 1; - for (auto d : shape) n *= d; - std::vector data(n); - FILE* f = fopen((path + ".bin").c_str(), "rb"); - if (!f) return {}; - fread(data.data(), sizeof(float), n, f); - fclose(f); - return data; - }; - - std::vector cpu_shape, metal_shape; - auto cpu_data = load_bin(cpu_path, cpu_shape); - auto metal_data = load_bin(metal_path, metal_shape); - - if (cpu_data.empty() || metal_data.empty()) { - fprintf(stderr, " [SKIP] %s — could not load dumps\n", name.c_str()); - return; - } - if (cpu_data.size() != metal_data.size()) { - fprintf(stderr, " [FAIL] %s — size mismatch: %zu vs %zu\n", - name.c_str(), cpu_data.size(), metal_data.size()); - all_pass = false; - return; - } - - float max_diff = 0.0f; - int max_idx = 0; - int n_bad = 0; - for (size_t i = 0; i < cpu_data.size(); i++) { - float diff = std::fabs(cpu_data[i] - metal_data[i]); - if (diff > max_diff) { max_diff = diff; max_idx = (int)i; } - if (diff > atol) n_bad++; - } - - bool ok = (max_diff < atol); - fprintf(stderr, " %s %s: max_diff=%.6f at [%d] (cpu=%.6f metal=%.6f) n_bad=%d/%zu\n", - ok ? "[PASS]" : "[FAIL]", name.c_str(), max_diff, max_idx, - cpu_data[max_idx], metal_data[max_idx], n_bad, cpu_data.size()); - if (!ok) all_pass = false; - }; - - // ViT backbone + FPN neck outputs — expect reasonable tolerance - // (different execution order on GPU can cause FP32 rounding differences) - compare_dumps("vit_output", "/tmp/sam3_cpu_vit", "/tmp/sam3_metal_vit", 0.1f); - compare_dumps("neck_trk_0", "/tmp/sam3_cpu_neck0", "/tmp/sam3_metal_neck0", 0.1f); - compare_dumps("neck_trk_1", "/tmp/sam3_cpu_neck1", "/tmp/sam3_metal_neck1", 0.1f); - compare_dumps("neck_trk_2", "/tmp/sam3_cpu_neck2", "/tmp/sam3_metal_neck2", 0.1f); - - fprintf(stderr, "\n%s\n", all_pass ? "ALL COMPARISONS PASSED" : "SOME COMPARISONS FAILED"); - return all_pass ? 0 : 1; -} diff --git a/tests/test_metal_vit_prefix_compare.cpp b/tests/test_metal_vit_prefix_compare.cpp deleted file mode 100644 index 8f75a9a..0000000 --- a/tests/test_metal_vit_prefix_compare.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static std::vector trim_shape(const int64_t ne[4]) { - std::vector shape = { - (int) ne[0], - (int) ne[1], - (int) ne[2], - (int) ne[3], - }; - while (!shape.empty() && shape.back() == 1) { - shape.pop_back(); - } - if (shape.empty()) { - shape.push_back(1); - } - return shape; -} - -static ref_tensor_f32 run_prefix_stage_or_die(const sam3_model & model, - sam3_vit_prefix_stage stage, - const ref_tensor_f32 & input, - int n_threads) { - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < input.shape.size() && i < 4; ++i) { - in_ne[i] = input.shape[i]; - } - - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_prefix_stage(model, stage, input.data.data(), in_ne, out.data, out_ne, n_threads)) { - fprintf(stderr, "prefix stage %d failed\n", (int) stage); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static ref_tensor_f32 make_image_input(const ref_tensor_f32 & preprocessed, int img_size) { - ref_tensor_f32 image = preprocessed; - image.shape = {img_size, img_size, 3, 1}; - return image; -} - -static ref_tensor_f32 run_block0_input_or_die(const sam3_model & model, - const ref_tensor_f32 & image, - int img_size, - int n_threads) { - int64_t out_ne[4] = {0, 0, 0, 0}; - ref_tensor_f32 out; - if (!sam3_test_run_vit_block0_input(model, image.data.data(), img_size, out.data, out_ne, n_threads)) { - fprintf(stderr, "block0 input prefix failed\n"); - std::exit(1); - } - out.shape = trim_shape(out_ne); - return out; -} - -static void print_diff_row(const char * label, - const ref_tensor_f32 & a, - const ref_tensor_f32 & b, - float atol = 1e-4f) { - const compare_result r = compare_tensors(a.data.data(), b.data.data(), a.numel(), atol); - fprintf(stderr, "%-18s %-18s %12.6f %12.6f %10d\n", - label, - format_shape(a.shape).c_str(), - r.max_diff, - r.mean_diff, - r.n_bad); -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [ref_dir]\n", argv[0]); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - - const ref_tensor_f32 preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty() || preprocessed.shape.size() != 4) { - fprintf(stderr, "failed to load %s/preprocessed\n", ref_dir.c_str()); - return 1; - } - - const int img_size = preprocessed.shape[2]; - const int n_threads = 8; - - sam3_params cpu_params; - cpu_params.model_path = model_path; - cpu_params.use_gpu = false; - cpu_params.n_threads = n_threads; - - sam3_params metal_params = cpu_params; - metal_params.use_gpu = true; - - auto cpu_model = sam3_load_model(cpu_params); - auto metal_model = sam3_load_model(metal_params); - if (!cpu_model || !metal_model) { - fprintf(stderr, "failed to load CPU or Metal model\n"); - return 1; - } - - fprintf(stderr, "\n=== Exact ViT prefix compare ===\n"); - fprintf(stderr, "%-18s %-18s %12s %12s %10s\n", - "stage", "shape", "max_abs_diff", "mean_abs", "n_bad"); - - const ref_tensor_f32 image_input = make_image_input(preprocessed, img_size); - - const ref_tensor_f32 cpu_im2col = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_PATCH_IM2COL, image_input, n_threads); - const ref_tensor_f32 metal_im2col = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_PATCH_IM2COL, image_input, n_threads); - print_diff_row("patch_im2col", cpu_im2col, metal_im2col); - - const ref_tensor_f32 cpu_patch_raw_shared = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW, cpu_im2col, n_threads); - const ref_tensor_f32 metal_patch_raw_shared = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT_RAW, cpu_im2col, n_threads); - print_diff_row("patch_mulmat_raw", cpu_patch_raw_shared, metal_patch_raw_shared); - - { - ref_tensor_f32 host_patch_raw_f32; - ref_tensor_f32 host_patch_raw_f64; - int64_t out_ne_f32[4] = {0, 0, 0, 0}; - int64_t out_ne_f64[4] = {0, 0, 0, 0}; - int64_t in_ne[4] = {1, 1, 1, 1}; - for (size_t i = 0; i < cpu_im2col.shape.size() && i < 4; ++i) { - in_ne[i] = cpu_im2col.shape[i]; - } - - if (!sam3_test_run_patch_mulmat_host_ref(*cpu_model, cpu_im2col.data.data(), in_ne, false, host_patch_raw_f32.data, out_ne_f32) || - !sam3_test_run_patch_mulmat_host_ref(*cpu_model, cpu_im2col.data.data(), in_ne, true, host_patch_raw_f64.data, out_ne_f64)) { - fprintf(stderr, "host patch_mulmat reference failed\n"); - return 1; - } - - host_patch_raw_f32.shape = trim_shape(out_ne_f32); - host_patch_raw_f64.shape = trim_shape(out_ne_f64); - - print_diff_row("patch_raw_cpu_f32", cpu_patch_raw_shared, host_patch_raw_f32); - print_diff_row("patch_raw_cpu_f64", cpu_patch_raw_shared, host_patch_raw_f64); - print_diff_row("patch_raw_mtl_f32", metal_patch_raw_shared, host_patch_raw_f32); - print_diff_row("patch_raw_mtl_f64", metal_patch_raw_shared, host_patch_raw_f64); - } - - const ref_tensor_f32 cpu_patch_shared = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT, cpu_im2col, n_threads); - const ref_tensor_f32 metal_patch_shared = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_PATCH_MULMAT, cpu_im2col, n_threads); - print_diff_row("patch_mulmat", cpu_patch_shared, metal_patch_shared); - - const ref_tensor_f32 cpu_patch = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_PATCH_EMBED, image_input, n_threads); - const ref_tensor_f32 metal_patch = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_PATCH_EMBED, image_input, n_threads); - print_diff_row("patch_embed", cpu_patch, metal_patch); - - const ref_tensor_f32 cpu_pos_shared = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_POS_ADD, cpu_patch, n_threads); - const ref_tensor_f32 metal_pos_shared = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_POS_ADD, cpu_patch, n_threads); - print_diff_row("after_pos_embed", cpu_pos_shared, metal_pos_shared); - - const ref_tensor_f32 cpu_ln_norm_shared = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_LN_PRE_NORM, cpu_pos_shared, n_threads); - const ref_tensor_f32 metal_ln_norm_shared = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_LN_PRE_NORM, cpu_pos_shared, n_threads); - print_diff_row("ln_pre_norm", cpu_ln_norm_shared, metal_ln_norm_shared); - - const ref_tensor_f32 cpu_ln_pre_shared = run_prefix_stage_or_die(*cpu_model, SAM3_VIT_PREFIX_STAGE_LN_PRE, cpu_pos_shared, n_threads); - const ref_tensor_f32 metal_ln_pre_shared = run_prefix_stage_or_die(*metal_model, SAM3_VIT_PREFIX_STAGE_LN_PRE, cpu_pos_shared, n_threads); - print_diff_row("after_ln_pre", cpu_ln_pre_shared, metal_ln_pre_shared); - - const ref_tensor_f32 cpu_block0_input = run_block0_input_or_die(*cpu_model, preprocessed, img_size, n_threads); - const ref_tensor_f32 metal_block0_input = run_block0_input_or_die(*metal_model, preprocessed, img_size, n_threads); - print_diff_row("block0_input", cpu_block0_input, metal_block0_input); - - return 0; -} diff --git a/tests/test_metal_vit_upstream_compare.cpp b/tests/test_metal_vit_upstream_compare.cpp deleted file mode 100644 index 420115c..0000000 --- a/tests/test_metal_vit_upstream_compare.cpp +++ /dev/null @@ -1,383 +0,0 @@ -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include "sam3.h" -#include "test_utils.h" - -#include -#include -#include -#include -#include -#include - -struct tensor_spec { - std::string label; - std::string name; -}; - -struct backend_run_result { - double elapsed_ms = 0.0; - std::map infos; -}; - -struct isolated_run_result { - double elapsed_ms = 0.0; - std::vector output; - std::vector shape; -}; - -static std::string format_shape(const std::vector & shape) { - std::string out = "["; - for (size_t i = 0; i < shape.size(); ++i) { - if (i > 0) { - out += ", "; - } - out += std::to_string(shape[i]); - } - out += "]"; - return out; -} - -static std::string make_block_name(int block_idx, const char * suffix) { - char name[64]; - snprintf(name, sizeof(name), "dbg_block_%d_%s", block_idx, suffix); - return name; -} - -static bool dump_named_tensors(const sam3_state & state, - const std::vector & names, - const std::string & output_dir, - std::map & infos) { - for (const auto & name : names) { - sam3_tensor_info info; - if (!sam3_get_state_tensor_info(state, name, info)) { - fprintf(stderr, "missing tensor info for %s\n", name.c_str()); - return false; - } - infos[name] = info; - if (!sam3_dump_state_tensor(state, name, output_dir + "/" + name)) { - fprintf(stderr, "failed to dump %s\n", name.c_str()); - return false; - } - } - return true; -} - -static ggml_backend_t create_backend(bool use_gpu, int n_threads) { -#ifdef GGML_USE_METAL - if (use_gpu) { - return ggml_backend_metal_init(); - } -#else - (void) use_gpu; -#endif - ggml_backend_t backend = ggml_backend_cpu_init(); - if (backend) { - ggml_backend_cpu_set_n_threads(backend, n_threads); - } - return backend; -} - -static std::vector normalize_shape_4d(const std::vector & shape) { - std::vector out = shape; - while (out.size() < 4) { - out.push_back(1); - } - return out; -} - -static ggml_tensor * new_tensor_4d_from_shape(ggml_context * ctx, - ggml_type type, - const std::vector & shape) { - const std::vector s = normalize_shape_4d(shape); - return ggml_new_tensor_4d(ctx, type, s[0], s[1], s[2], s[3]); -} - -static bool load_block_norm_params(const std::string & model_path, - int block_idx, - ref_tensor_f32 & weight, - ref_tensor_f32 & bias) { - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 1; - - auto model = sam3_load_model(params); - if (!model) { - return false; - } - - const std::string tmp_dir = "/tmp/sam3_vit_upstream_model"; - ensure_dir(tmp_dir); - const std::string prefix = "vit.blocks." + std::to_string(block_idx) + ".norm1"; - if (!sam3_dump_model_tensor(*model, prefix + ".weight", tmp_dir + "/norm1_weight")) { - return false; - } - if (!sam3_dump_model_tensor(*model, prefix + ".bias", tmp_dir + "/norm1_bias")) { - return false; - } - - weight = load_ref_f32(tmp_dir + "/norm1_weight"); - bias = load_ref_f32(tmp_dir + "/norm1_bias"); - sam3_free_model(*model); - model.reset(); - return !weight.data.empty() && !bias.data.empty(); -} - -static isolated_run_result run_isolated_norm(ggml_backend_t backend, - const ref_tensor_f32 & x_in, - const ref_tensor_f32 & weight, - const ref_tensor_f32 & bias) { - isolated_run_result result; - - const size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); - ggml_init_params params = { - /*.mem_size =*/ ctx_size, - /*.mem_buffer =*/ nullptr, - /*.no_alloc =*/ true, - }; - ggml_context * ctx = ggml_init(params); - if (!ctx) { - return result; - } - - ggml_tensor * x = new_tensor_4d_from_shape(ctx, GGML_TYPE_F32, x_in.shape); - ggml_tensor * w = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, weight.shape[0]); - ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, bias.shape[0]); - ggml_set_input(x); - ggml_set_input(w); - ggml_set_input(b); - - ggml_tensor * y = ggml_norm(ctx, x, 1e-5f); - y = ggml_mul_inplace(ctx, y, w); - y = ggml_add_inplace(ctx, y, b); - ggml_set_output(y); - - ggml_cgraph * graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, y); - - ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; - } - - ggml_backend_tensor_set(x, x_in.data.data(), 0, x_in.data.size() * sizeof(float)); - ggml_backend_tensor_set(w, weight.data.data(), 0, weight.data.size() * sizeof(float)); - ggml_backend_tensor_set(b, bias.data.data(), 0, bias.data.size() * sizeof(float)); - - auto t0 = std::chrono::high_resolution_clock::now(); - const ggml_status status = ggml_backend_graph_compute(backend, graph); - auto t1 = std::chrono::high_resolution_clock::now(); - result.elapsed_ms = std::chrono::duration(t1 - t0).count(); - if (status == GGML_STATUS_SUCCESS) { - result.shape = { - (int) y->ne[0], - (int) y->ne[1], - (int) y->ne[2], - (int) y->ne[3], - }; - result.output.resize((size_t) ggml_nelements(y)); - ggml_backend_tensor_get(y, result.output.data(), 0, result.output.size() * sizeof(float)); - } else { - fprintf(stderr, "isolated norm failed: %s\n", ggml_status_to_string(status)); - } - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return result; -} - -static bool run_backend(const std::string & model_path, - const std::vector & chw_data, - int img_size, - bool use_gpu, - int n_threads, - int debug_block_idx, - const std::vector & names, - const std::string & dump_dir, - backend_run_result & result) { - sam3_params params; - params.model_path = model_path; - params.use_gpu = use_gpu; - params.n_threads = n_threads; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "failed to load model for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "failed to create state for %s\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - auto t0 = std::chrono::high_resolution_clock::now(); - const bool ok = sam3_encode_vit_from_preprocessed_selective( - *state, *model, chw_data.data(), img_size, names); - auto t1 = std::chrono::high_resolution_clock::now(); - result.elapsed_ms = std::chrono::duration(t1 - t0).count(); - if (!ok) { - fprintf(stderr, "%s selective ViT encode failed\n", use_gpu ? "Metal" : "CPU"); - return false; - } - - if (!dump_named_tensors(*state, names, dump_dir, result.infos)) { - return false; - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - return true; -} - -static std::vector default_specs_for_block(int block_idx) { - std::vector specs; - - if (block_idx <= 0) { - specs.push_back({"block_0_input / norm1_input", "dbg_after_ln_pre"}); - } else { - specs.push_back({"block_input / norm1_input", make_block_name(block_idx - 1, "out")}); - } - - specs.push_back({"norm1_output / qkv_input", make_block_name(block_idx, "norm1")}); - specs.push_back({"qkv_proj_output", make_block_name(block_idx, "qkv_proj")}); - specs.push_back({"attn_output", make_block_name(block_idx, "attn_out")}); - specs.push_back({"attn_proj_output", make_block_name(block_idx, "attn_proj")}); - specs.push_back({"resid1_output", make_block_name(block_idx, "resid1")}); - specs.push_back({"mlp_output", make_block_name(block_idx, "mlp")}); - specs.push_back({"block_output", make_block_name(block_idx, "out")}); - specs.push_back({"vit_output", "vit_output"}); - - return specs; -} - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [ref_dir] [block_idx]\n", argv[0]); - fprintf(stderr, "Default ref_dir: tests/ref_phase3\n"); - fprintf(stderr, "Default block_idx: 15\n"); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = argc >= 3 ? argv[2] : "tests/ref_phase3"; - const int block_idx = argc >= 4 ? std::atoi(argv[3]) : 15; - if (block_idx < 0 || block_idx >= 32) { - fprintf(stderr, "invalid block_idx=%d\n", block_idx); - return 1; - } - - auto preprocessed = load_ref_f32(ref_dir + "/preprocessed"); - if (preprocessed.data.empty() || preprocessed.shape.size() != 4) { - fprintf(stderr, "failed to load %s/preprocessed\n", ref_dir.c_str()); - return 1; - } - const int img_size = preprocessed.shape[2]; - - const std::vector specs = default_specs_for_block(block_idx); - std::vector names; - names.reserve(specs.size()); - for (const auto & spec : specs) { - names.push_back(spec.name); - } - - const std::string cpu_dir = "/tmp/sam3_cpu_vit_upstream"; - const std::string metal_dir = "/tmp/sam3_metal_vit_upstream"; - ensure_dir(cpu_dir); - ensure_dir(metal_dir); - - backend_run_result cpu_run; - backend_run_result metal_run; - - fprintf(stderr, "\n=== CPU block %d selective ViT run ===\n", block_idx); - if (!run_backend(model_path, preprocessed.data, img_size, false, 8, block_idx, names, cpu_dir, cpu_run)) { - return 1; - } - fprintf(stderr, "CPU wall time: %.1f ms\n", cpu_run.elapsed_ms); - - fprintf(stderr, "\n=== Metal block %d selective ViT run ===\n", block_idx); - if (!run_backend(model_path, preprocessed.data, img_size, true, 8, block_idx, names, metal_dir, metal_run)) { - return 1; - } - fprintf(stderr, "Metal wall time: %.1f ms\n", metal_run.elapsed_ms); - - fprintf(stderr, "\n=== CPU vs Metal block %d ===\n", block_idx); - fprintf(stderr, "%-28s %-24s %-18s %14s %14s %12s\n", - "semantic", "tensor", "shape", "max_abs_diff", "mean_abs_diff", "n_bad"); - - for (const auto & spec : specs) { - auto cpu = load_ref_f32(cpu_dir + "/" + spec.name); - auto metal = load_ref_f32(metal_dir + "/" + spec.name); - if (cpu.data.empty() || metal.data.empty()) { - fprintf(stderr, "%-28s %-24s %-18s %14s %14s %12s\n", - spec.label.c_str(), spec.name.c_str(), "-", "load-fail", "load-fail", "-"); - continue; - } - - const compare_result diff = compare_tensors(cpu.data.data(), metal.data.data(), cpu.numel(), 1e-4f); - fprintf(stderr, "%-28s %-24s %-18s %14.6f %14.6f %12d\n", - spec.label.c_str(), - spec.name.c_str(), - format_shape(cpu.shape).c_str(), - diff.max_diff, - diff.mean_diff, - diff.n_bad); - } - - ref_tensor_f32 norm_w; - ref_tensor_f32 norm_b; - if (!load_block_norm_params(model_path, block_idx, norm_w, norm_b)) { - fprintf(stderr, "\nfailed to load norm1 params for block %d\n", block_idx); - return 1; - } - - const ref_tensor_f32 block_input = load_ref_f32(cpu_dir + "/" + specs[0].name); - const ref_tensor_f32 cpu_norm_dump = load_ref_f32(cpu_dir + "/" + make_block_name(block_idx, "norm1")); - const ref_tensor_f32 metal_norm_dump = load_ref_f32(metal_dir + "/" + make_block_name(block_idx, "norm1")); - if (!block_input.data.empty() && !cpu_norm_dump.data.empty() && !metal_norm_dump.data.empty()) { - ggml_backend_t cpu_backend = create_backend(false, 8); - ggml_backend_t metal_backend = create_backend(true, 8); - - const isolated_run_result iso_cpu = run_isolated_norm(cpu_backend, block_input, norm_w, norm_b); - const isolated_run_result iso_metal = run_isolated_norm(metal_backend, block_input, norm_w, norm_b); - - if (cpu_backend) { - ggml_backend_free(cpu_backend); - } - if (metal_backend) { - ggml_backend_free(metal_backend); - } - - if (!iso_cpu.output.empty() && !iso_metal.output.empty()) { - const compare_result metal_vs_cpu = compare_tensors( - iso_cpu.output.data(), iso_metal.output.data(), (int) iso_cpu.output.size(), 1e-4f); - const compare_result cpu_vs_dump = compare_tensors( - cpu_norm_dump.data.data(), iso_cpu.output.data(), cpu_norm_dump.numel(), 1e-4f); - const compare_result metal_vs_dump = compare_tensors( - metal_norm_dump.data.data(), iso_metal.output.data(), metal_norm_dump.numel(), 1e-4f); - - fprintf(stderr, "\n=== Isolated block %d norm1 ===\n", block_idx); - fprintf(stderr, "input shape: %s\n", format_shape(block_input.shape).c_str()); - fprintf(stderr, "isolated metal_vs_cpu: max_abs_diff=%.6f mean_abs_diff=%.6f n_bad=%d\n", - metal_vs_cpu.max_diff, metal_vs_cpu.mean_diff, metal_vs_cpu.n_bad); - fprintf(stderr, "isolated cpu_vs_graph_dump: max_abs_diff=%.6f mean_abs_diff=%.6f n_bad=%d\n", - cpu_vs_dump.max_diff, cpu_vs_dump.mean_diff, cpu_vs_dump.n_bad); - fprintf(stderr, "isolated metal_vs_graph_dump: max_abs_diff=%.6f mean_abs_diff=%.6f n_bad=%d\n", - metal_vs_dump.max_diff, metal_vs_dump.mean_diff, metal_vs_dump.n_bad); - } - } - - return 0; -} diff --git a/tests/test_metal_win.cpp b/tests/test_metal_win.cpp deleted file mode 100644 index 9fd296e..0000000 --- a/tests/test_metal_win.cpp +++ /dev/null @@ -1,252 +0,0 @@ -// Test Metal kernels for ggml_win_part and ggml_win_unpart. -// Runs the same graph on CPU and Metal, then compares results element-by-element. - -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include -#include -#include - -// Run a graph: build win_part -> win_unpart on the given backend. -// Returns the final output as a flat float vector. -static std::vector run_win_roundtrip(ggml_backend_t backend, - const std::vector& input, - int C, int W, int H, int WS) { - // Graph context - size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); - struct ggml_init_params params = {ctx_size, nullptr, true}; - struct ggml_context* ctx = ggml_init(params); - - // Input tensor: [C, W, H, 1] - auto* inp = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, C, W, H); - ggml_set_name(inp, "input"); - ggml_set_input(inp); - - // win_part: [C, W, H, 1] -> [C, WS, WS, np] - auto* parted = ggml_win_part(ctx, inp, WS); - ggml_set_name(parted, "parted"); - - // win_unpart: [C, WS, WS, np] -> [C, W, H, 1] - auto* unparted = ggml_win_unpart(ctx, parted, W, H, WS); - ggml_set_name(unparted, "unparted"); - ggml_set_output(unparted); - - // Build graph - auto* graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, unparted); - - // Allocate - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "FAIL: gallocr_alloc_graph failed\n"); - exit(1); - } - - // Set input data - ggml_backend_tensor_set(inp, input.data(), 0, input.size() * sizeof(float)); - - // Compute - ggml_backend_graph_compute(backend, graph); - - // Read output - std::vector output(C * W * H); - ggml_backend_tensor_get(unparted, output.data(), 0, output.size() * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return output; -} - -// Run win_part only (no roundtrip) — compare intermediate output. -static std::vector run_win_part_only(ggml_backend_t backend, - const std::vector& input, - int C, int W, int H, int WS, - int64_t* out_ne) { - size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead(); - struct ggml_init_params params = {ctx_size, nullptr, true}; - struct ggml_context* ctx = ggml_init(params); - - auto* inp = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, C, W, H); - ggml_set_name(inp, "input"); - ggml_set_input(inp); - - auto* parted = ggml_win_part(ctx, inp, WS); - ggml_set_name(parted, "parted"); - ggml_set_output(parted); - - auto* graph = ggml_new_graph(ctx); - ggml_build_forward_expand(graph, parted); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - if (!ggml_gallocr_reserve(galloc, graph) || !ggml_gallocr_alloc_graph(galloc, graph)) { - fprintf(stderr, "FAIL: gallocr_alloc_graph failed\n"); - exit(1); - } - - ggml_backend_tensor_set(inp, input.data(), 0, input.size() * sizeof(float)); - ggml_backend_graph_compute(backend, graph); - - // Output shape - int64_t total = 1; - for (int d = 0; d < 4; d++) { - out_ne[d] = parted->ne[d]; - total *= parted->ne[d]; - } - - std::vector output(total); - ggml_backend_tensor_get(parted, output.data(), 0, output.size() * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return output; -} - -static bool compare_vectors(const std::vector& a, const std::vector& b, - const char* label, float tol = 1e-6f) { - if (a.size() != b.size()) { - fprintf(stderr, " FAIL [%s]: size mismatch: %zu vs %zu\n", label, a.size(), b.size()); - return false; - } - float max_diff = 0.0f; - int first_fail = -1; - for (size_t i = 0; i < a.size(); i++) { - float diff = std::fabs(a[i] - b[i]); - if (diff > max_diff) max_diff = diff; - if (diff > tol && first_fail < 0) first_fail = (int)i; - } - if (first_fail >= 0) { - fprintf(stderr, " FAIL [%s]: max_diff=%.8f at index %d (cpu=%.8f metal=%.8f)\n", - label, max_diff, first_fail, a[first_fail], b[first_fail]); - return false; - } - fprintf(stderr, " PASS [%s]: max_diff=%.2e (%zu elements)\n", label, max_diff, a.size()); - return true; -} - -int main() { -#ifndef GGML_USE_METAL - fprintf(stderr, "Metal not available, skipping test\n"); - return 0; -#else - // Init backends - ggml_backend_t cpu_backend = ggml_backend_cpu_init(); - ggml_backend_t metal_backend = ggml_backend_metal_init(); - if (!metal_backend) { - fprintf(stderr, "FAIL: could not init Metal backend\n"); - return 1; - } - - bool all_pass = true; - - // ── Test 1: sam3 exact dimensions (C=1024, W=H=72, WS=24) ────────── - { - fprintf(stderr, "\n=== Test 1: sam3 dimensions (1024, 72, 72, WS=24) ===\n"); - const int C = 1024, W = 72, H = 72, WS = 24; - std::vector input(C * W * H); - for (size_t i = 0; i < input.size(); i++) { - input[i] = 0.001f * (float)(i % 9973); // pseudo-random - } - - // win_part only - int64_t cpu_ne[4], metal_ne[4]; - auto cpu_part = run_win_part_only(cpu_backend, input, C, W, H, WS, cpu_ne); - auto metal_part = run_win_part_only(metal_backend, input, C, W, H, WS, metal_ne); - fprintf(stderr, " win_part output shape: [%lld, %lld, %lld, %lld]\n", - cpu_ne[0], cpu_ne[1], cpu_ne[2], cpu_ne[3]); - all_pass &= compare_vectors(cpu_part, metal_part, "win_part"); - - // roundtrip (win_part -> win_unpart) - auto cpu_rt = run_win_roundtrip(cpu_backend, input, C, W, H, WS); - auto metal_rt = run_win_roundtrip(metal_backend, input, C, W, H, WS); - all_pass &= compare_vectors(cpu_rt, metal_rt, "roundtrip"); - - // roundtrip should recover original input exactly (no padding needed) - all_pass &= compare_vectors(input, cpu_rt, "cpu_identity"); - all_pass &= compare_vectors(input, metal_rt, "metal_identity"); - } - - // ── Test 2: non-divisible dimensions (needs padding) ──────────────── - { - fprintf(stderr, "\n=== Test 2: padding case (64, 50, 50, WS=24) ===\n"); - const int C = 64, W = 50, H = 50, WS = 24; - std::vector input(C * W * H); - for (size_t i = 0; i < input.size(); i++) { - input[i] = -1.0f + 2.0f * (float)(i % 7919) / 7919.0f; - } - - int64_t cpu_ne[4], metal_ne[4]; - auto cpu_part = run_win_part_only(cpu_backend, input, C, W, H, WS, cpu_ne); - auto metal_part = run_win_part_only(metal_backend, input, C, W, H, WS, metal_ne); - fprintf(stderr, " win_part output shape: [%lld, %lld, %lld, %lld]\n", - cpu_ne[0], cpu_ne[1], cpu_ne[2], cpu_ne[3]); - all_pass &= compare_vectors(cpu_part, metal_part, "win_part_padded"); - - auto cpu_rt = run_win_roundtrip(cpu_backend, input, C, W, H, WS); - auto metal_rt = run_win_roundtrip(metal_backend, input, C, W, H, WS); - all_pass &= compare_vectors(cpu_rt, metal_rt, "roundtrip_padded"); - all_pass &= compare_vectors(input, cpu_rt, "cpu_identity_padded"); - all_pass &= compare_vectors(input, metal_rt, "metal_identity_padded"); - } - - // ── Test 3: small case for easy debugging ────────────────────────── - { - fprintf(stderr, "\n=== Test 3: small (4, 6, 6, WS=3) ===\n"); - const int C = 4, W = 6, H = 6, WS = 3; - std::vector input(C * W * H); - for (size_t i = 0; i < input.size(); i++) { - input[i] = (float)(i + 1); - } - - int64_t cpu_ne[4], metal_ne[4]; - auto cpu_part = run_win_part_only(cpu_backend, input, C, W, H, WS, cpu_ne); - auto metal_part = run_win_part_only(metal_backend, input, C, W, H, WS, metal_ne); - fprintf(stderr, " win_part output shape: [%lld, %lld, %lld, %lld]\n", - cpu_ne[0], cpu_ne[1], cpu_ne[2], cpu_ne[3]); - all_pass &= compare_vectors(cpu_part, metal_part, "win_part_small"); - - auto cpu_rt = run_win_roundtrip(cpu_backend, input, C, W, H, WS); - auto metal_rt = run_win_roundtrip(metal_backend, input, C, W, H, WS); - all_pass &= compare_vectors(cpu_rt, metal_rt, "roundtrip_small"); - all_pass &= compare_vectors(input, cpu_rt, "cpu_identity_small"); - all_pass &= compare_vectors(input, metal_rt, "metal_identity_small"); - } - - // ── Test 4: asymmetric dimensions ────────────────────────────────── - { - fprintf(stderr, "\n=== Test 4: asymmetric (128, 48, 72, WS=24) ===\n"); - const int C = 128, W = 48, H = 72, WS = 24; - std::vector input(C * W * H); - for (size_t i = 0; i < input.size(); i++) { - input[i] = sinf((float)i * 0.01f); - } - - int64_t cpu_ne[4], metal_ne[4]; - auto cpu_part = run_win_part_only(cpu_backend, input, C, W, H, WS, cpu_ne); - auto metal_part = run_win_part_only(metal_backend, input, C, W, H, WS, metal_ne); - fprintf(stderr, " win_part output shape: [%lld, %lld, %lld, %lld]\n", - cpu_ne[0], cpu_ne[1], cpu_ne[2], cpu_ne[3]); - all_pass &= compare_vectors(cpu_part, metal_part, "win_part_asym"); - - auto cpu_rt = run_win_roundtrip(cpu_backend, input, C, W, H, WS); - auto metal_rt = run_win_roundtrip(metal_backend, input, C, W, H, WS); - all_pass &= compare_vectors(cpu_rt, metal_rt, "roundtrip_asym"); - all_pass &= compare_vectors(input, cpu_rt, "cpu_identity_asym"); - all_pass &= compare_vectors(input, metal_rt, "metal_identity_asym"); - } - - ggml_backend_free(cpu_backend); - ggml_backend_free(metal_backend); - - fprintf(stderr, "\n%s\n", all_pass ? "ALL TESTS PASSED" : "SOME TESTS FAILED"); - return all_pass ? 0 : 1; -#endif -} diff --git a/tests/test_phase3.cpp b/tests/test_phase3.cpp deleted file mode 100644 index 3472efe..0000000 --- a/tests/test_phase3.cpp +++ /dev/null @@ -1,632 +0,0 @@ -// Phase 3 Numerical Audit — test ViT backbone, neck, sinusoidal PE, RoPE -// against Python reference tensors dumped by dump_phase3_reference.py. -// -// Usage: -// ./test_phase3 tests/ref_phase3 models/sam3-f16.ggml tests/test_random.jpg -// -// Tests: -// 1. RoPE computation (pure math, no model needed) -// 2. Sinusoidal PE computation (pure math, no model needed) -// 3. Full image encoding: preprocessing → ViT → neck → PE -// Compares intermediate and final outputs against Python references. - -#include "sam3.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -// ═══════════════════════════════════════════════════════════════════════════════ -// Helpers -// ═══════════════════════════════════════════════════════════════════════════════ - -struct ref_tensor { - std::vector data; - std::vector shape; - int numel() const { - int n = 1; - for (auto d : shape) n *= d; - return n; - } -}; - -static ref_tensor load_ref(const std::string & path) { - ref_tensor t; - { - std::ifstream f(path + ".shape"); - if (!f) { - fprintf(stderr, " [SKIP] %s.shape not found\n", path.c_str()); - return t; - } - std::string line; - std::getline(f, line); - size_t pos = 0; - while (pos < line.size()) { - size_t end = line.find(',', pos); - if (end == std::string::npos) end = line.size(); - t.shape.push_back(std::stoi(line.substr(pos, end - pos))); - pos = end + 1; - } - } - { - std::ifstream f(path + ".bin", std::ios::binary); - if (!f) { - fprintf(stderr, " [SKIP] %s.bin not found\n", path.c_str()); - t.shape.clear(); - return t; - } - t.data.resize(t.numel()); - f.read(reinterpret_cast(t.data.data()), t.numel() * sizeof(float)); - } - return t; -} - -struct compare_result { - float max_diff = 0.0f; - float mean_diff = 0.0f; - double cosine_sim = 0.0; - int n_elements = 0; - int n_bad = 0; // elements exceeding tolerance -}; - -static compare_result compare_tensors(const float * a, const float * b, int n, - float atol = 1e-4f) { - compare_result r; - r.n_elements = n; - double sum_diff = 0.0; - double dot_ab = 0.0, dot_aa = 0.0, dot_bb = 0.0; - for (int i = 0; i < n; ++i) { - float diff = fabsf(a[i] - b[i]); - if (diff > r.max_diff) r.max_diff = diff; - if (diff > atol) r.n_bad++; - sum_diff += diff; - dot_ab += (double)a[i] * b[i]; - dot_aa += (double)a[i] * a[i]; - dot_bb += (double)b[i] * b[i]; - } - r.mean_diff = (float)(sum_diff / n); - if (dot_aa > 0 && dot_bb > 0) { - r.cosine_sim = dot_ab / (sqrt(dot_aa) * sqrt(dot_bb)); - } - return r; -} - -static bool check(const std::string & name, const float * got, const ref_tensor & ref, - float atol, int & n_pass, int & n_fail) { - if (ref.data.empty()) { - fprintf(stderr, " [SKIP] %s — no reference data\n", name.c_str()); - return true; - } - auto r = compare_tensors(got, ref.data.data(), ref.numel(), atol); - bool ok = r.max_diff <= atol; - fprintf(stderr, " %s %-45s: max=%.6e mean=%.6e cos=%.8f bad=%d/%d (atol=%.1e)\n", - ok ? "[PASS]" : "[FAIL]", name.c_str(), r.max_diff, r.mean_diff, - r.cosine_sim, r.n_bad, r.n_elements, atol); - if (ok) n_pass++; else n_fail++; - return ok; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test 1: RoPE computation -// ═══════════════════════════════════════════════════════════════════════════════ - -static void test_rope(const std::string & ref_dir, int & n_pass, int & n_fail) { - fprintf(stderr, "\n╔══════════════════════════════════════════╗\n"); - fprintf(stderr, "║ Test: RoPE Frequencies ║\n"); - fprintf(stderr, "╚══════════════════════════════════════════╝\n"); - - struct { int end; float scale; const char * name; } cases[] = { - {24, 1.0f, "rope_window_real"}, - {72, 24.0f/72.0f, "rope_global_real"}, - }; - - const int dim = 64; - const float theta = 10000.0f; - const int half_dim = dim / 4; // 16 - - std::vector freqs(half_dim); - for (int i = 0; i < half_dim; ++i) { - freqs[i] = 1.0f / powf(theta, (float)(i * 4) / (float)dim); - } - - for (auto & tc : cases) { - auto ref = load_ref(ref_dir + "/" + tc.name); - if (ref.data.empty()) continue; - - const int N = tc.end * tc.end; - std::vector our_rope(N * dim); - - for (int idx = 0; idx < N; ++idx) { - float t_x = (float)(idx % tc.end) * tc.scale; - float t_y = static_cast(idx / tc.end) * tc.scale; // NOLINT(bugprone-integer-division) - - for (int i = 0; i < half_dim; ++i) { - float angle_x = t_x * freqs[i]; - our_rope[idx * dim + i * 2 + 0] = cosf(angle_x); - our_rope[idx * dim + i * 2 + 1] = sinf(angle_x); - } - for (int i = 0; i < half_dim; ++i) { - float angle_y = t_y * freqs[i]; - our_rope[idx * dim + half_dim * 2 + i * 2 + 0] = cosf(angle_y); - our_rope[idx * dim + half_dim * 2 + i * 2 + 1] = sinf(angle_y); - } - } - - check(tc.name, our_rope.data(), ref, 1e-5f, n_pass, n_fail); - } - - // Also compare against checkpoint freqs_cis to verify conversion is lossless - for (int bi : {0, 7}) { - std::string name = "ckpt_freqs_cis_block" + std::to_string(bi); - auto ref_ckpt = load_ref(ref_dir + "/" + name); - auto ref_computed = load_ref(ref_dir + "/" + (bi == 0 ? "rope_window_real" : "rope_global_real")); - if (!ref_ckpt.data.empty() && !ref_computed.data.empty()) { - auto r = compare_tensors(ref_ckpt.data.data(), ref_computed.data.data(), - std::min(ref_ckpt.numel(), ref_computed.numel()), 1e-6f); - bool ok = r.max_diff <= 1e-6f; - fprintf(stderr, " %s ckpt vs computed block %d: max=%.6e\n", - ok ? "[PASS]" : "[FAIL]", bi, r.max_diff); - if (ok) n_pass++; else n_fail++; - } - } -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test 2: Sinusoidal PE -// ═══════════════════════════════════════════════════════════════════════════════ - -static void test_sinusoidal_pe(const std::string & ref_dir, int & n_pass, int & n_fail) { - fprintf(stderr, "\n╔══════════════════════════════════════════╗\n"); - fprintf(stderr, "║ Test: Sinusoidal PE ║\n"); - fprintf(stderr, "╚══════════════════════════════════════════╝\n"); - - struct { int H; int W; const char * name; } cases[] = { - {288, 288, "pe_288"}, - {144, 144, "pe_144"}, - { 72, 72, "pe_72"}, - { 36, 36, "pe_36"}, - }; - - for (auto & tc : cases) { - auto ref = load_ref(ref_dir + "/" + tc.name); - if (ref.data.empty()) continue; - - const int H = tc.H, W = tc.W, d_model = 256; - const int half = d_model / 2; // 128 - const float scale = 2.0f * (float)M_PI; - const float temperature = 10000.0f; - - // Match Python PositionEmbeddingSine.forward() exactly: - // y_embed = arange(1, H+1) / (H + eps) * scale - // dim_t = temperature ** (2 * (arange(half) // 2) / half) - // pos_y = y_embed / dim_t - // stack(sin(even), cos(odd)).flatten → interleaved sin/cos - // Output: [1, 256, H, W] (NCHW) - - std::vector our_pe(d_model * H * W); - const float eps = 1e-6f; - - for (int y = 0; y < H; ++y) { - for (int x = 0; x < W; ++x) { - float pos_y = ((float)(y + 1) / ((float)H + eps)) * scale; - float pos_x = ((float)(x + 1) / ((float)W + eps)) * scale; - - for (int i = 0; i < half; ++i) { - int paired = (i / 2) * 2; // 0,0,2,2,4,4,... - float dim_t = powf(temperature, (float)paired / (float)half); - - float val_x, val_y; - if (i % 2 == 0) { - val_x = sinf(pos_x / dim_t); - val_y = sinf(pos_y / dim_t); - } else { - val_x = cosf(pos_x / dim_t); - val_y = cosf(pos_y / dim_t); - } - - // PyTorch output is [1, 256, H, W] - // Channel layout: first 128 = pos_y, next 128 = pos_x - our_pe[i * H * W + y * W + x] = val_y; - our_pe[(i + half) * H * W + y * W + x] = val_x; - } - } - } - - check(tc.name, our_pe.data(), ref, 1e-5f, n_pass, n_fail); - } -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test 3: Full image encoding (model-dependent) -// ═══════════════════════════════════════════════════════════════════════════════ - -static void test_encode_image(const std::string & model_path, - const std::string & image_path, - const std::string & ref_dir, - int & n_pass, int & n_fail) { - fprintf(stderr, "\n╔══════════════════════════════════════════╗\n"); - fprintf(stderr, "║ Test: Full Image Encoding ║\n"); - fprintf(stderr, "╚══════════════════════════════════════════╝\n"); - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, " FATAL: Failed to load model\n"); - n_fail++; - return; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, " FATAL: Failed to create state\n"); - n_fail++; - return; - } - - auto img = sam3_load_image(image_path); - if (img.data.empty()) { - fprintf(stderr, " FATAL: Failed to load image\n"); - n_fail++; - return; - } - - bool ok = sam3_encode_image(*state, *model, img); - if (!ok) { - fprintf(stderr, " FATAL: sam3_encode_image failed\n"); - n_fail++; - return; - } - - fprintf(stderr, " Image encoding completed successfully\n"); - - std::string dump_dir = ref_dir + "/cpp_out_phase3"; - { - std::string cmd = "mkdir -p " + dump_dir; - (void)system(cmd.c_str()); - } - - // ── Helper: dump and compare a state tensor ────────────────────────── - // C++ tensors are in ggml layout. We need to transpose to PyTorch layout. - - // For NHWC tensors (ViT intermediates): ggml [E, W, H, B] → PyTorch [B, H, W, E] - auto compare_nhwc = [&](const std::string & cpp_name, const std::string & ref_name, - float atol) { - bool dumped = sam3_dump_state_tensor(*state, cpp_name, dump_dir + "/" + cpp_name); - if (!dumped) { - fprintf(stderr, " [SKIP] %s — tensor '%s' not available in state\n", - ref_name.c_str(), cpp_name.c_str()); - return; - } - auto ref_t = load_ref(ref_dir + "/" + ref_name); - auto cpp_t = load_ref(dump_dir + "/" + cpp_name); - if (ref_t.data.empty() || cpp_t.data.empty()) { - fprintf(stderr, " [SKIP] %s — data not available\n", ref_name.c_str()); - return; - } - // ggml [E, W, H, ...] → [H, W, E] (ignoring batch) - int E = cpp_t.shape[0]; - int nW = cpp_t.shape[1]; - int nH = cpp_t.shape.size() > 2 ? cpp_t.shape[2] : 1; - // ref is [1, H, W, E] or [B, H, W, E] - std::vector transposed(cpp_t.numel()); - for (int h = 0; h < nH; ++h) - for (int w = 0; w < nW; ++w) - for (int e = 0; e < E; ++e) - transposed[h * nW * E + w * E + e] = cpp_t.data[e + w * E + h * E * nW]; - - check(ref_name + " (NHWC)", transposed.data(), ref_t, atol, n_pass, n_fail); - }; - - // For NCHW tensors (neck outputs): ggml [C, W, H, B] → PyTorch [1, C, H, W] - auto compare_nchw = [&](const std::string & cpp_name, const std::string & ref_name, - float atol) { - bool dumped = sam3_dump_state_tensor(*state, cpp_name, dump_dir + "/" + cpp_name); - if (!dumped) { - fprintf(stderr, " [SKIP] %s — tensor '%s' not available in state\n", - ref_name.c_str(), cpp_name.c_str()); - return; - } - auto ref_t = load_ref(ref_dir + "/" + ref_name); - auto cpp_t = load_ref(dump_dir + "/" + cpp_name); - if (ref_t.data.empty() || cpp_t.data.empty()) { - fprintf(stderr, " [SKIP] %s — data not available\n", ref_name.c_str()); - return; - } - // ggml [C, W, H, ...]: element (c,w,h) at c + w*C + h*C*W - // PyTorch [1, C, H, W]: element (0,c,h,w) at c*H*W + h*W + w - int C = cpp_t.shape[0]; - int nW = cpp_t.shape[1]; - int nH = cpp_t.shape.size() > 2 ? cpp_t.shape[2] : 1; - std::vector transposed(cpp_t.numel()); - for (int c = 0; c < C; ++c) - for (int h = 0; h < nH; ++h) - for (int w = 0; w < nW; ++w) - transposed[c * nH * nW + h * nW + w] = cpp_t.data[c + w * C + h * C * nW]; - - check(ref_name + " (NCHW)", transposed.data(), ref_t, atol, n_pass, n_fail); - }; - - // ── Compare ViT intermediates ──────────────────────────────────────── - fprintf(stderr, "\n --- ViT Intermediates ---\n"); - - // Patch embed: max_diff ~0.3 due to different bilinear resize between C++ and Python. - // This is the root cause of all downstream differences. - compare_nhwc("dbg_patch_embed", "patch_embed", 0.5f); - - // After pos embed: same as patch_embed (pos embed loaded exactly) - compare_nhwc("dbg_after_pos_embed", "after_pos_embed", 0.5f); - - // After ln_pre: amplified by layer norm (max ~1.2) - compare_nhwc("dbg_after_ln_pre", "after_ln_pre", 2.0f); - - // Block outputs (error accumulates through transformer layers) - compare_nhwc("dbg_block_0_out", "block_0_out", 2.0f); - compare_nhwc("dbg_block_7_out", "block_7_out", 5.0f); - - // ViT final output - sam3_dump_state_tensor(*state, "vit_output", dump_dir + "/vit_output"); - auto ref_vit = load_ref(ref_dir + "/vit_output_bchw"); - auto cpp_vit = load_ref(dump_dir + "/vit_output"); - if (!ref_vit.data.empty() && !cpp_vit.data.empty()) { - int E = cpp_vit.shape[0], W = cpp_vit.shape[1]; - int H = (cpp_vit.shape.size() > 2) ? cpp_vit.shape[2] : 1; - std::vector transposed(cpp_vit.numel()); - for (int e = 0; e < E; ++e) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - transposed[e * H * W + h * W + w] = cpp_vit.data[e + w * E + h * E * W]; - // ViT output after 32 blocks: max_diff can be ~80 due to preprocessing diff - // (C++ bilinear resize vs Python PIL resize) amplified through 32 transformer layers. - // Cosine sim >0.995 confirms the ViT is computing the correct function. - // The isolated test (Test 4) with identical input achieves cos=1.0 on f32. - auto vit_r = compare_tensors(transposed.data(), ref_vit.data.data(), ref_vit.numel(), 100.0f); - bool vit_ok = vit_r.cosine_sim > 0.995 && vit_r.mean_diff < 0.1f; - fprintf(stderr, " %s %-45s: max=%.6e mean=%.6e cos=%.8f (tol: cos>0.995, mean<0.1)\n", - vit_ok ? "[PASS]" : "[FAIL]", "vit_output_bchw", - vit_r.max_diff, vit_r.mean_diff, vit_r.cosine_sim); - if (vit_ok) n_pass++; else n_fail++; - } - - // ── Compare neck outputs ───────────────────────────────────────────── - fprintf(stderr, "\n --- Neck Outputs (Detector) ---\n"); - for (int i = 0; i < 4; ++i) { - std::string name = "neck_det_" + std::to_string(i); - compare_nchw(name, name, 5.0f); - } - - // ── Compare sinusoidal PE from state ───────────────────────────────── - // The state stores PE tensors as neck_det_pe_i. - fprintf(stderr, "\n --- Sinusoidal PE (from state) ---\n"); - int pe_sizes[] = {288, 144, 72, 36}; - for (int i = 0; i < 4; ++i) { - std::string cpp_pe_name = "neck_det_pe_" + std::to_string(i); - std::string ref_pe_name = "pe_" + std::to_string(pe_sizes[i]); - sam3_dump_state_tensor(*state, cpp_pe_name, dump_dir + "/" + cpp_pe_name); - auto ref_pe = load_ref(ref_dir + "/" + ref_pe_name); - auto cpp_pe = load_ref(dump_dir + "/" + cpp_pe_name); - if (!ref_pe.data.empty() && !cpp_pe.data.empty()) { - // PE is stored as [D, W, H, 1] in ggml, ref is [1, 256, H, W] in PyTorch - int D = cpp_pe.shape[0], pW = cpp_pe.shape[1]; - int pH = (cpp_pe.shape.size() > 2) ? cpp_pe.shape[2] : 1; - std::vector transposed(cpp_pe.numel()); - for (int d = 0; d < D; ++d) - for (int h = 0; h < pH; ++h) - for (int w = 0; w < pW; ++w) - transposed[d * pH * pW + h * pW + w] = cpp_pe.data[d + w * D + h * D * pW]; - check(ref_pe_name + " (state)", transposed.data(), ref_pe, 1e-4f, n_pass, n_fail); - } - } - - state.reset(); - sam3_free_model(*model); - model.reset(); -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test 4: Isolated ViT numerics using Python-preprocessed input -// This feeds the exact same preprocessed image to the C++ pipeline to isolate -// ViT numerical accuracy from preprocessing differences. -// ═══════════════════════════════════════════════════════════════════════════════ - -static void test_encode_from_preprocessed(const std::string & model_path, - const std::string & ref_dir, - int & n_pass, int & n_fail) { - fprintf(stderr, "\n╔══════════════════════════════════════════╗\n"); - fprintf(stderr, "║ Test: ViT Numerics (Python preprocess) ║\n"); - fprintf(stderr, "╚══════════════════════════════════════════╝\n"); - - // Load Python-preprocessed image (CHW layout, [1, 3, 1008, 1008]) - auto preproc = load_ref(ref_dir + "/preprocessed"); - if (preproc.data.empty()) { - fprintf(stderr, " [SKIP] No preprocessed reference data\n"); - return; - } - fprintf(stderr, " Loaded preprocessed image: shape=["); - for (size_t i = 0; i < preproc.shape.size(); ++i) { - if (i > 0) fprintf(stderr, ", "); - fprintf(stderr, "%d", preproc.shape[i]); - } - fprintf(stderr, "] numel=%d\n", preproc.numel()); - - // The preprocessed data is [1, 3, 1008, 1008] in PyTorch NCHW layout. - // For ggml, the input tensor is [W=1008, H=1008, C=3, B=1]. - // ggml flat index: x + y*W + c*W*H = same as CHW[c*H*W + y*W + x]. - // So the CHW data can be uploaded directly. - const float * chw_data = preproc.data.data(); - int img_size = preproc.shape[2]; // 1008 - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, " FATAL: Failed to load model\n"); - n_fail++; - return; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, " FATAL: Failed to create state\n"); - n_fail++; - return; - } - - bool ok = sam3_encode_image_from_preprocessed(*state, *model, chw_data, img_size); - if (!ok) { - fprintf(stderr, " FATAL: sam3_encode_image_from_preprocessed failed\n"); - n_fail++; - return; - } - - std::string dump_dir = ref_dir + "/cpp_out_from_preproc"; - { - std::string cmd = "mkdir -p " + dump_dir; - (void)system(cmd.c_str()); - } - - // ── Compare ViT output ──────────────────────────────────────────────── - fprintf(stderr, "\n --- ViT Output (same input) ---\n"); - sam3_dump_state_tensor(*state, "vit_output", dump_dir + "/vit_output"); - auto ref_vit = load_ref(ref_dir + "/vit_output_bchw"); - auto cpp_vit = load_ref(dump_dir + "/vit_output"); - if (!ref_vit.data.empty() && !cpp_vit.data.empty()) { - int E = cpp_vit.shape[0], W = cpp_vit.shape[1]; - int H = (cpp_vit.shape.size() > 2) ? cpp_vit.shape[2] : 1; - std::vector transposed(cpp_vit.numel()); - for (int e = 0; e < E; ++e) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - transposed[e * H * W + h * W + w] = cpp_vit.data[e + w * E + h * E * W]; - - auto vit_r = compare_tensors(transposed.data(), ref_vit.data.data(), ref_vit.numel(), 1e-2f); - // With identical input and f32 weights, expect very high cosine similarity. - // With f16 weights, expect cos > 0.999 and mean_diff < 0.05. - bool vit_ok = vit_r.cosine_sim > 0.999 && vit_r.mean_diff < 0.05f; - fprintf(stderr, " %s %-45s: max=%.6e mean=%.6e cos=%.8f bad=%d/%d\n", - vit_ok ? "[PASS]" : "[FAIL]", "vit_output (same input)", - vit_r.max_diff, vit_r.mean_diff, vit_r.cosine_sim, - vit_r.n_bad, vit_r.n_elements); - if (vit_ok) n_pass++; else n_fail++; - } - - // ── Compare neck outputs ───────────────────────────────────────────── - fprintf(stderr, "\n --- Neck Outputs (same input) ---\n"); - const char * neck_names[] = {"neck_det_0", "neck_det_1", "neck_det_2", "neck_det_3"}; - for (int i = 0; i < 4; ++i) { - bool dumped = sam3_dump_state_tensor(*state, neck_names[i], dump_dir + "/" + neck_names[i]); - if (!dumped) { - fprintf(stderr, " [SKIP] %s — not available\n", neck_names[i]); - continue; - } - auto ref_t = load_ref(ref_dir + "/" + neck_names[i]); - auto cpp_t = load_ref(dump_dir + "/" + neck_names[i]); - if (ref_t.data.empty() || cpp_t.data.empty()) continue; - - int C = cpp_t.shape[0]; - int nW = cpp_t.shape[1]; - int nH = cpp_t.shape.size() > 2 ? cpp_t.shape[2] : 1; - std::vector transposed(cpp_t.numel()); - for (int c = 0; c < C; ++c) - for (int h = 0; h < nH; ++h) - for (int w = 0; w < nW; ++w) - transposed[c * nH * nW + h * nW + w] = cpp_t.data[c + w * C + h * C * nW]; - - auto r = compare_tensors(transposed.data(), ref_t.data.data(), ref_t.numel(), 1.0f); - bool pass = r.cosine_sim > 0.999 && r.mean_diff < 0.05f; - fprintf(stderr, " %s %-45s: max=%.6e mean=%.6e cos=%.8f bad=%d/%d\n", - pass ? "[PASS]" : "[FAIL]", neck_names[i], - r.max_diff, r.mean_diff, r.cosine_sim, r.n_bad, r.n_elements); - if (pass) n_pass++; else n_fail++; - } - - // ── Compare tracker neck outputs ──────────────────────────────────── - fprintf(stderr, "\n --- Tracker Neck Outputs (same input) ---\n"); - const char * trk_names[] = {"neck_trk_0", "neck_trk_1", "neck_trk_2", "neck_trk_3"}; - for (int i = 0; i < 4; ++i) { - bool dumped = sam3_dump_state_tensor(*state, trk_names[i], dump_dir + "/" + trk_names[i]); - if (!dumped) { - fprintf(stderr, " [SKIP] %s — not available\n", trk_names[i]); - continue; - } - auto ref_t = load_ref(ref_dir + "/" + trk_names[i]); - auto cpp_t = load_ref(dump_dir + "/" + trk_names[i]); - if (ref_t.data.empty() || cpp_t.data.empty()) continue; - - int C = cpp_t.shape[0]; - int nW = cpp_t.shape[1]; - int nH = cpp_t.shape.size() > 2 ? cpp_t.shape[2] : 1; - std::vector transposed(cpp_t.numel()); - for (int c = 0; c < C; ++c) - for (int h = 0; h < nH; ++h) - for (int w = 0; w < nW; ++w) - transposed[c * nH * nW + h * nW + w] = cpp_t.data[c + w * C + h * C * nW]; - - auto r = compare_tensors(transposed.data(), ref_t.data.data(), ref_t.numel(), 1.0f); - bool pass = r.cosine_sim > 0.999 && r.mean_diff < 0.05f; - fprintf(stderr, " %s %-45s: max=%.6e mean=%.6e cos=%.8f bad=%d/%d\n", - pass ? "[PASS]" : "[FAIL]", trk_names[i], - r.max_diff, r.mean_diff, r.cosine_sim, r.n_bad, r.n_elements); - if (pass) n_pass++; else n_fail++; - } - - state.reset(); - sam3_free_model(*model); - model.reset(); -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Main -// ═══════════════════════════════════════════════════════════════════════════════ - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [model.ggml] [image.jpg]\n", argv[0]); - fprintf(stderr, "\nGenerate reference data first:\n"); - fprintf(stderr, " uv run python tests/dump_phase3_reference.py \\\n"); - fprintf(stderr, " --checkpoint raw_weights/sam3.pt --image data/test_image.jpg\n"); - return 1; - } - - std::string ref_dir = argv[1]; - std::string model_path = (argc > 2) ? argv[2] : ""; - std::string image_path = (argc > 3) ? argv[3] : ""; - - int n_pass = 0, n_fail = 0; - - // Test 1: RoPE (no model needed) - test_rope(ref_dir, n_pass, n_fail); - - // Test 2: Sinusoidal PE (no model needed) - test_sinusoidal_pe(ref_dir, n_pass, n_fail); - - // Test 3: Full encoding (needs model + image) - if (!model_path.empty() && !image_path.empty()) { - test_encode_image(model_path, image_path, ref_dir, n_pass, n_fail); - } else { - fprintf(stderr, "\n[SKIP] Full encoding test — no model/image provided\n"); - } - - // Test 4: Isolated ViT numerics from Python-preprocessed input (needs model) - if (!model_path.empty()) { - test_encode_from_preprocessed(model_path, ref_dir, n_pass, n_fail); - } else { - fprintf(stderr, "\n[SKIP] Isolated ViT test — no model provided\n"); - } - - fprintf(stderr, "\n═══════════════════════════════════════════\n"); - fprintf(stderr, "Phase 3 Audit Results: %d passed, %d failed\n", n_pass, n_fail); - fprintf(stderr, "═══════════════════════════════════════════\n"); - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/test_phase4.cpp b/tests/test_phase4.cpp deleted file mode 100644 index 1ad6424..0000000 --- a/tests/test_phase4.cpp +++ /dev/null @@ -1,334 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include -#include - -struct prompt_case { - std::string id; - std::string text; -}; - -struct summary_row { - bool measured = false; - bool ok = true; - float max_diff = 0.0f; - float mean_diff = 0.0f; - float cosine_sim = 1.0f; - std::string worst_tensor; - std::string note; -}; - -static bool ends_with(const std::string & s, const std::string & suffix) { - return s.size() >= suffix.size() && - s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -static std::vector load_prompt_cases(const std::string & path) { - std::vector cases; - std::ifstream f(path); - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - - prompt_case pc; - size_t tab = line.find('\t'); - if (tab == std::string::npos) { - pc.id = "prompt_" + std::to_string(cases.size()); - pc.text = line; - } else { - pc.id = line.substr(0, tab); - pc.text = line.substr(tab + 1); - } - cases.push_back(std::move(pc)); - } - return cases; -} - -static std::string category_for_tensor(const std::string & name) { - if (name == "causal_mask") return "causal_mask"; - if (name == "text_token_embed") return "text_token_embed"; - if (name == "text_after_pos_embed") return "text_after_pos_embed"; - if (name == "text_final_ln") return "text_final_ln"; - if (name == "text_features_2d") return "text_features_2d"; - - if (ends_with(name, "_after_ln1")) return "block_after_ln1"; - if (ends_with(name, "_qkv")) return "block_qkv"; - if (ends_with(name, "_attn_out")) return "block_attn_out"; - if (ends_with(name, "_after_attn_residual")) return "block_after_attn_residual"; - if (ends_with(name, "_after_ln2")) return "block_after_ln2"; - if (ends_with(name, "_mlp_fc1")) return "block_mlp_fc1"; - if (ends_with(name, "_mlp_gelu")) return "block_mlp_gelu"; - if (ends_with(name, "_mlp_out")) return "block_mlp_out"; - if (ends_with(name, "_out")) return "block_out"; - - return name; -} - -static const char * label_for_category(const std::string & category) { - if (category == "token_ids") return "Token IDs"; - if (category == "causal_mask") return "Causal mask"; - if (category == "text_token_embed") return "Token embedding"; - if (category == "text_after_pos_embed") return "Embedding + position"; - if (category == "block_after_ln1") return "Blocks after ln1"; - if (category == "block_qkv") return "Blocks qkv"; - if (category == "block_attn_out") return "Blocks attn out"; - if (category == "block_after_attn_residual") return "Blocks attn residual"; - if (category == "block_after_ln2") return "Blocks after ln2"; - if (category == "block_mlp_fc1") return "Blocks mlp fc1"; - if (category == "block_mlp_gelu") return "Blocks mlp gelu"; - if (category == "block_mlp_out") return "Blocks mlp out"; - if (category == "block_out") return "Blocks output"; - if (category == "text_final_ln") return "Final layer norm"; - if (category == "text_features_2d") return "Final text features"; - return category.c_str(); -} - -static std::vector tensor_names() { - std::vector names = { - "causal_mask", - "text_token_embed", - "text_after_pos_embed", - "text_final_ln", - "text_features_2d", - }; - - for (int i = 0; i < 24; ++i) { - char name[64]; - - snprintf(name, sizeof(name), "text_block_%02d_after_ln1", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_qkv", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_attn_out", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_after_attn_residual", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_after_ln2", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_mlp_fc1", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_mlp_gelu", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_mlp_out", i); - names.emplace_back(name); - - snprintf(name, sizeof(name), "text_block_%02d_out", i); - names.emplace_back(name); - } - - return names; -} - -static void update_summary(summary_row & row, - const std::string & tensor_name, - const compare_result & r, - bool ok) { - if (!row.measured || r.max_diff >= row.max_diff) { - row.max_diff = r.max_diff; - row.mean_diff = r.mean_diff; - row.cosine_sim = r.cosine_sim; - row.worst_tensor = tensor_name; - } - row.measured = true; - row.ok = row.ok && ok; -} - -int main(int argc, char ** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string model_path = argv[2]; - const std::string prompts_path = argv[3]; - const float atol = 1e-4f; - - auto prompts = load_prompt_cases(prompts_path); - if (prompts.empty()) { - fprintf(stderr, "No prompts loaded from %s\n", prompts_path.c_str()); - return 1; - } - - if (!sam3_test_load_tokenizer(model_path)) { - fprintf(stderr, "Failed to load tokenizer from %s\n", model_path.c_str()); - return 1; - } - - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = false; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - const std::string cpp_root = ref_dir + "/cpp_out_phase4"; - if (!ensure_dir(cpp_root)) { - fprintf(stderr, "Failed to create %s\n", cpp_root.c_str()); - return 1; - } - - const std::vector names = tensor_names(); - const std::vector category_order = { - "token_ids", - "causal_mask", - "text_token_embed", - "text_after_pos_embed", - "block_after_ln1", - "block_qkv", - "block_attn_out", - "block_after_attn_residual", - "block_after_ln2", - "block_mlp_fc1", - "block_mlp_gelu", - "block_mlp_out", - "block_out", - "text_final_ln", - "text_features_2d", - }; - - std::map> report; - bool overall_ok = true; - - for (const auto & prompt : prompts) { - fprintf(stderr, "\n=== Prompt %s: \"%s\" ===\n", prompt.id.c_str(), prompt.text.c_str()); - - const std::string prompt_ref_dir = ref_dir + "/" + prompt.id; - const std::string prompt_cpp_dir = cpp_root + "/" + prompt.id; - if (!ensure_dir(prompt_cpp_dir)) { - fprintf(stderr, "Failed to create %s\n", prompt_cpp_dir.c_str()); - overall_ok = false; - continue; - } - - auto ref_token_ids = load_ref_i32(prompt_ref_dir + "/token_ids"); - if (ref_token_ids.data.empty()) { - fprintf(stderr, "Missing reference token_ids for %s\n", prompt.id.c_str()); - overall_ok = false; - continue; - } - - auto cpp_token_ids = sam3_test_tokenize(prompt.text); - int token_bad = compare_exact_i32(cpp_token_ids, ref_token_ids.data); - summary_row token_row; - token_row.measured = true; - token_row.ok = (token_bad == 0); - token_row.max_diff = token_bad == 0 ? 0.0f : 1.0f; - token_row.mean_diff = token_bad == 0 ? 0.0f : 1.0f; - token_row.cosine_sim = token_bad == 0 ? 1.0f : 0.0f; - token_row.note = token_bad == 0 ? "exact int compare" : "tokenizer mismatch"; - report[prompt.id]["token_ids"] = token_row; - if (token_bad != 0) { - fprintf(stderr, " [FAIL] token_ids mismatch for %s\n", prompt.id.c_str()); - overall_ok = false; - } - - if (!sam3_test_dump_text_encoder(*model, ref_token_ids.data, prompt_cpp_dir, params.n_threads)) { - fprintf(stderr, " [FAIL] could not dump C++ text encoder tensors for %s\n", prompt.id.c_str()); - overall_ok = false; - continue; - } - - for (const auto & name : names) { - auto ref = load_ref_f32(prompt_ref_dir + "/" + name); - auto got = load_ref_f32(prompt_cpp_dir + "/" + name); - const std::string category = category_for_tensor(name); - - if (ref.data.empty() || got.data.empty()) { - auto & row = report[prompt.id][category]; - row.ok = false; - row.note = "missing reference or C++ output"; - overall_ok = false; - fprintf(stderr, " [FAIL] missing tensor %s for %s\n", name.c_str(), prompt.id.c_str()); - continue; - } - - if (ref.numel() != got.numel()) { - auto & row = report[prompt.id][category]; - row.ok = false; - row.note = "shape mismatch"; - overall_ok = false; - fprintf(stderr, " [FAIL] shape mismatch for %s (%d vs %d)\n", - name.c_str(), got.numel(), ref.numel()); - continue; - } - - auto r = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), atol); - bool ok = r.max_diff <= atol; - update_summary(report[prompt.id][category], name, r, ok); - if (!ok) { - overall_ok = false; - fprintf(stderr, - " [FAIL] %-32s max=%.3e mean=%.3e cosine=%.8f\n", - name.c_str(), r.max_diff, r.mean_diff, r.cosine_sim); - } - } - - for (const auto & category : category_order) { - auto it = report[prompt.id].find(category); - if (it == report[prompt.id].end() || !it->second.measured) { - if (category == "token_ids") { - continue; - } - fprintf(stderr, " [FAIL] no measurements recorded for %s/%s\n", - prompt.id.c_str(), category.c_str()); - overall_ok = false; - report[prompt.id][category].ok = false; - report[prompt.id][category].note = "untested"; - } - } - } - - fprintf(stderr, "\n## Numerical Precision Report\n\n"); - fprintf(stderr, "| Prompt | Operation | max_diff | mean_diff | cosine | Tolerance | Status | Notes |\n"); - fprintf(stderr, "|--------|-----------|----------|-----------|--------|-----------|--------|-------|\n"); - for (const auto & prompt : prompts) { - for (const auto & category : category_order) { - const auto it = report[prompt.id].find(category); - summary_row row; - if (it != report[prompt.id].end()) { - row = it->second; - } - - const char * status = !row.measured && category != "token_ids" - ? "UNTESTED" - : (row.ok ? "PASS" : "FAIL"); - - std::string note = row.note; - if (note.empty() && !row.worst_tensor.empty()) { - note = "worst=" + row.worst_tensor; - } - - fprintf(stderr, - "| %s | %s | %.3e | %.3e | %.8f | %.1e | %s | %s |\n", - prompt.id.c_str(), - label_for_category(category), - row.max_diff, - row.mean_diff, - row.cosine_sim, - atol, - status, - note.empty() ? "measured" : note.c_str()); - } - } - - return overall_ok ? 0 : 1; -} diff --git a/tests/test_phase5.cpp b/tests/test_phase5.cpp deleted file mode 100644 index b9a9e3d..0000000 --- a/tests/test_phase5.cpp +++ /dev/null @@ -1,256 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include - -struct tensor_case { - std::string name; - std::string label; -}; - -struct report_row { - bool measured = false; - bool ok = false; - compare_result r; - std::string note; -}; - -static std::string load_prompt(const std::string & path) { - std::ifstream f(path); - std::string line; - if (f) { - std::getline(f, line); - } - return line; -} - -static std::vector tensor_cases() { - std::vector cases = { - {"text_valid_mask", "Text valid mask"}, - {"text_features", "Text features"}, - {"fenc_img_input", "Fusion encoder image input"}, - {"fenc_pos_embed", "Fusion encoder position input"}, - {"fenc_prompt", "Fusion encoder prompt input"}, - {"img_pe_72", "Image PE 72x72"}, - {"fenc_output", "Fusion encoder final output"}, - {"ddec_query_embed", "Decoder query embed"}, - {"ddec_ref_pts_raw", "Decoder reference points raw"}, - {"ddec_presence_token", "Decoder presence token"}, - {"ddec_ref_boxes_init", "Decoder initial reference boxes"}, - {"ddec_query_sine_0", "Decoder layer 0 query sine"}, - {"ddec_query_pos_0", "Decoder layer 0 query pos"}, - {"ddec_rpb_mask_0", "Decoder layer 0 RPB mask"}, - {"ddec_layer0_after_sa", "Decoder layer 0 after self-attn"}, - {"ddec_layer0_after_text_ca", "Decoder layer 0 after text CA"}, - {"ddec_layer0_after_img_ca", "Decoder layer 0 after image CA"}, - {"ddec_layer0_full_out", "Decoder layer 0 full output"}, - {"ddec_layer0_presence", "Decoder layer 0 presence"}, - {"ddec_normed_output", "Decoder final normed output"}, - {"ddec_pred_boxes", "Decoder predicted boxes"}, - {"ddec_presence_logit", "Decoder presence logit"}, - {"scoring_prompt_mlp_out", "Scoring prompt MLP"}, - {"scoring_pooled", "Scoring pooled prompt"}, - {"scoring_proj_pooled", "Scoring projected pooled prompt"}, - {"scoring_proj_hs", "Scoring projected queries"}, - {"scoring_class_scores", "Scoring class scores"}, - {"seg_enc_after_ca", "Seg head encoder after prompt CA"}, - {"seg_enc_visual", "Seg head visual encoder map"}, - {"seg_pixel_dec_stage0", "Seg head pixel decoder stage 0"}, - {"seg_pixel_dec_stage1", "Seg head pixel decoder stage 1"}, - {"seg_pixel_decoder_out", "Seg head pixel decoder output"}, - {"seg_instance_embed", "Seg head instance embed"}, - {"seg_mask_embed", "Seg head mask embed"}, - {"seg_mask_logits", "Seg head mask logits"}, - }; - - for (int i = 0; i < 6; ++i) { - char name[64]; - char label[64]; - - snprintf(name, sizeof(name), "fenc_layer%d_out", i); - snprintf(label, sizeof(label), "Fusion encoder layer %d", i); - cases.push_back({name, label}); - - snprintf(name, sizeof(name), "ddec_layer%d_out", i); - snprintf(label, sizeof(label), "Decoder layer %d output", i); - cases.push_back({name, label}); - - snprintf(name, sizeof(name), "ddec_layer%d_refboxes", i); - snprintf(label, sizeof(label), "Decoder layer %d ref boxes", i); - cases.push_back({name, label}); - } - - return cases; -} - -static float tolerance_for(const std::string & name) { - if (name == "ddec_rpb_mask_0") { - return 2e-4f; - } - if (name == "seg_mask_embed") { - return 2e-4f; - } - if (name == "seg_mask_logits") { - return 3e-3f; - } - if (name == "ddec_layer2_out" || - name == "ddec_layer3_out" || - name == "ddec_layer4_out" || - name == "ddec_layer5_out") { - return 2e-4f; - } - return 1e-4f; -} - -int main(int argc, char ** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string model_path = argv[2]; - const std::string image_path = argv[3]; - const std::string cpp_dir = ref_dir + "/cpp_out_phase5"; - if (!ensure_dir(cpp_dir)) { - fprintf(stderr, "Failed to create %s\n", cpp_dir.c_str()); - return 1; - } - - std::string prompt = load_prompt(ref_dir + "/prompt.txt"); - if (prompt.empty()) { - prompt = "yellow school bus"; - } - - auto ref_token_ids = load_ref_i32(ref_dir + "/token_ids"); - - if (!sam3_test_load_tokenizer(model_path)) { - fprintf(stderr, "Failed to load tokenizer from %s\n", model_path.c_str()); - return 1; - } - - auto cpp_token_ids = sam3_test_tokenize(prompt); - std::vector run_token_ids = cpp_token_ids; - - report_row token_row; - if (!ref_token_ids.data.empty()) { - const int token_bad = compare_exact_i32(cpp_token_ids, ref_token_ids.data); - token_row.measured = true; - token_row.ok = token_bad == 0; - token_row.r.max_diff = token_bad == 0 ? 0.0f : 1.0f; - token_row.r.mean_diff = token_bad == 0 ? 0.0f : 1.0f; - token_row.r.cosine_sim = token_bad == 0 ? 1.0f : 0.0f; - token_row.note = token_bad == 0 ? "exact int compare" : "tokenizer mismatch"; - run_token_ids = ref_token_ids.data; - } else { - token_row.measured = false; - token_row.ok = false; - token_row.note = "missing token_ids reference"; - } - - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = false; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - (void) image_path; - - const std::string prephase_ref_dir = ref_dir + "/../ref"; - if (!sam3_test_dump_phase5_from_ref_inputs(*model, run_token_ids, - prephase_ref_dir, ref_dir, - cpp_dir, params.n_threads)) { - fprintf(stderr, "Failed to dump C++ phase 5 tensors\n"); - return 1; - } - - const auto cases = tensor_cases(); - std::vector rows(cases.size()); - bool overall_ok = token_row.measured ? token_row.ok : false; - - for (size_t i = 0; i < cases.size(); ++i) { - const auto & tc = cases[i]; - auto ref = load_ref_f32(ref_dir + "/" + tc.name); - auto got = load_ref_f32(cpp_dir + "/" + tc.name); - - if (ref.data.empty() || got.data.empty()) { - rows[i].ok = false; - rows[i].note = "missing reference or C++ output"; - overall_ok = false; - fprintf(stderr, " [FAIL] missing tensor %s\n", tc.name.c_str()); - continue; - } - - if (ref.numel() != got.numel()) { - rows[i].ok = false; - rows[i].note = "numel mismatch"; - overall_ok = false; - fprintf(stderr, " [FAIL] numel mismatch for %s (%d vs %d)\n", - tc.name.c_str(), got.numel(), ref.numel()); - continue; - } - - rows[i].measured = true; - const float atol = tolerance_for(tc.name); - rows[i].r = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), atol); - rows[i].ok = rows[i].r.max_diff <= atol; - if (!rows[i].ok) { - char note[160]; - snprintf(note, sizeof(note), - "worst_idx=%d got=%.6g ref=%.6g bad=%d/%d", - rows[i].r.worst_index, - rows[i].r.worst_a, - rows[i].r.worst_b, - rows[i].r.n_bad, - rows[i].r.n_total); - rows[i].note = note; - overall_ok = false; - fprintf(stderr, - " [FAIL] %-24s max=%.3e mean=%.3e cosine=%.8f idx=%d got=%.6g ref=%.6g\n", - tc.name.c_str(), - rows[i].r.max_diff, - rows[i].r.mean_diff, - rows[i].r.cosine_sim, - rows[i].r.worst_index, - rows[i].r.worst_a, - rows[i].r.worst_b); - } - } - - fprintf(stderr, "\n## Numerical Precision Report\n\n"); - fprintf(stderr, "| Component | Operation | max_diff | mean_diff | cosine | Tolerance | Status | Notes |\n"); - fprintf(stderr, "|-----------|-----------|----------|-----------|--------|-----------|--------|-------|\n"); - fprintf(stderr, - "| Tokenizer | Token IDs | %.3e | %.3e | %.8f | exact | %s | %s |\n", - token_row.r.max_diff, - token_row.r.mean_diff, - token_row.r.cosine_sim, - token_row.ok ? "PASS" : "FAIL", - token_row.note.c_str()); - - for (size_t i = 0; i < cases.size(); ++i) { - const auto & tc = cases[i]; - const auto & row = rows[i]; - const char * status = !row.measured ? "UNTESTED" : (row.ok ? "PASS" : "FAIL"); - fprintf(stderr, - "| Phase 5 | %s | %.3e | %.3e | %.8f | %.1e | %s | %s |\n", - tc.label.c_str(), - row.r.max_diff, - row.r.mean_diff, - row.r.cosine_sim, - tolerance_for(tc.name), - status, - row.note.empty() ? "measured" : row.note.c_str()); - } - - return overall_ok ? 0 : 1; -} diff --git a/tests/test_phase6.cpp b/tests/test_phase6.cpp deleted file mode 100644 index 59b1b2a..0000000 --- a/tests/test_phase6.cpp +++ /dev/null @@ -1,287 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include - -struct phase6_case { - std::string id; - sam3_pvs_params params; -}; - -struct tensor_case { - std::string name; - std::string label; -}; - -struct report_row { - bool measured = false; - bool ok = false; - compare_result r; - std::string note; -}; - -static std::vector parse_points(const std::string & field) { - std::vector pts; - if (field.empty()) { - return pts; - } - - size_t start = 0; - while (start < field.size()) { - size_t end = field.find('|', start); - if (end == std::string::npos) { - end = field.size(); - } - - const std::string part = field.substr(start, end - start); - size_t colon = part.find(':'); - if (colon != std::string::npos) { - sam3_point pt; - pt.x = std::stof(part.substr(0, colon)); - pt.y = std::stof(part.substr(colon + 1)); - pts.push_back(pt); - } - - start = end + 1; - } - - return pts; -} - -static bool parse_box(const std::string & field, sam3_box & box) { - if (field.empty()) { - return false; - } - - float vals[4]; - size_t start = 0; - for (int i = 0; i < 4; ++i) { - size_t end = field.find(':', start); - if (end == std::string::npos) { - end = field.size(); - } - vals[i] = std::stof(field.substr(start, end - start)); - start = end + 1; - } - - box = {vals[0], vals[1], vals[2], vals[3]}; - return true; -} - -static std::vector load_cases(const std::string & path) { - std::vector cases; - std::ifstream f(path); - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - - std::vector fields; - size_t start = 0; - while (start <= line.size()) { - size_t end = line.find('\t', start); - if (end == std::string::npos) { - end = line.size(); - } - fields.push_back(line.substr(start, end - start)); - start = end + 1; - if (end == line.size()) { - break; - } - } - while (fields.size() < 5) { - fields.emplace_back(); - } - - phase6_case tc; - tc.id = fields[0]; - tc.params.multimask = fields[1] == "1"; - tc.params.pos_points = parse_points(fields[2]); - tc.params.neg_points = parse_points(fields[3]); - tc.params.use_box = parse_box(fields[4], tc.params.box); - cases.push_back(std::move(tc)); - } - return cases; -} - -static std::vector tensor_cases() { - return { - {"sam_pe_sparse", "Prompt sparse embeddings"}, - {"sam_pe_dense", "Prompt dense embeddings"}, - {"sam_pe_image_pe", "Prompt image PE"}, - {"sam_dec_image_feats", "Decoder image features"}, - {"sam_dec_tokens_initial", "Decoder initial tokens"}, - {"sam_dec_block0_queries", "Decoder block 0 queries"}, - {"sam_dec_block0_keys", "Decoder block 0 keys"}, - {"sam_dec_block1_queries", "Decoder block 1 queries"}, - {"sam_dec_block1_keys", "Decoder block 1 keys"}, - {"sam_dec_final_queries", "Decoder final queries"}, - {"sam_dec_feat_s1_proj", "Decoder s1 projection"}, - {"sam_dec_feat_s0_proj", "Decoder s0 projection"}, - {"sam_dec_upscaled", "Decoder upscaled embedding"}, - {"sam_dec_mask_tokens", "Decoder mask tokens"}, - {"sam_dec_masks", "Decoder mask logits"}, - {"sam_dec_iou", "Decoder IoU predictions"}, - {"sam_dec_obj_score", "Decoder object score logit"}, - {"sam_dec_sam_token", "Decoder SAM token"}, - }; -} - -static float tolerance_for(const std::string & name) { - if (name == "sam_dec_block1_queries" || - name == "sam_dec_final_queries") { - return 2e-4f; - } - if (name == "sam_dec_upscaled") { - return 9e-4f; - } - if (name == "sam_dec_masks") { - return 8e-3f; - } - return 1e-4f; -} - -int main(int argc, char ** argv) { - if (argc < 5) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string prephase_ref_dir = argv[2]; - const std::string model_path = argv[3]; - const std::string cases_path = argv[4]; - const std::string cpp_root = ref_dir + "/cpp_out_phase6"; - - auto cases = load_cases(cases_path); - if (cases.empty()) { - fprintf(stderr, "No phase 6 cases loaded from %s\n", cases_path.c_str()); - return 1; - } - - if (!ensure_dir(cpp_root)) { - fprintf(stderr, "Failed to create %s\n", cpp_root.c_str()); - return 1; - } - - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = false; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - const auto tensors = tensor_cases(); - bool overall_ok = true; - - struct row_print { - std::string case_id; - std::string label; - float max_diff = 0.0f; - float mean_diff = 0.0f; - float cosine = 0.0f; - float tol = 0.0f; - std::string status; - std::string note; - }; - std::vector report; - - for (const auto & tc : cases) { - fprintf(stderr, "\n=== Phase 6 Case %s ===\n", tc.id.c_str()); - - const std::string case_ref_dir = ref_dir + "/" + tc.id; - const std::string case_cpp_dir = cpp_root + "/" + tc.id; - if (!ensure_dir(case_cpp_dir)) { - fprintf(stderr, "Failed to create %s\n", case_cpp_dir.c_str()); - overall_ok = false; - continue; - } - - if (!sam3_test_dump_phase6_from_ref_inputs(*model, prephase_ref_dir, tc.params, - case_cpp_dir, params.n_threads)) { - fprintf(stderr, " [FAIL] could not dump C++ phase 6 tensors for %s\n", tc.id.c_str()); - overall_ok = false; - continue; - } - - for (const auto & tensor : tensors) { - auto ref = load_ref_f32(case_ref_dir + "/" + tensor.name); - auto got = load_ref_f32(case_cpp_dir + "/" + tensor.name); - - row_print row; - row.case_id = tc.id; - row.label = tensor.label; - row.tol = tolerance_for(tensor.name); - - if (ref.data.empty() || got.data.empty()) { - row.status = "FAIL"; - row.note = "missing reference or C++ output"; - overall_ok = false; - report.push_back(std::move(row)); - fprintf(stderr, " [FAIL] missing tensor %s for %s\n", - tensor.name.c_str(), tc.id.c_str()); - continue; - } - - if (ref.numel() != got.numel()) { - row.status = "FAIL"; - row.note = "numel mismatch"; - overall_ok = false; - report.push_back(std::move(row)); - fprintf(stderr, " [FAIL] numel mismatch for %s in %s\n", - tensor.name.c_str(), tc.id.c_str()); - continue; - } - - auto r = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), row.tol); - const bool ok = r.max_diff <= row.tol; - row.max_diff = r.max_diff; - row.mean_diff = r.mean_diff; - row.cosine = r.cosine_sim; - row.status = ok ? "PASS" : "FAIL"; - if (ok) { - row.note = "measured"; - } else { - char note[160]; - snprintf(note, sizeof(note), - "worst_idx=%d got=%.6g ref=%.6g bad=%d/%d", - r.worst_index, r.worst_a, r.worst_b, r.n_bad, r.n_total); - row.note = note; - overall_ok = false; - fprintf(stderr, - " [FAIL] %-24s max=%.3e mean=%.3e cosine=%.8f idx=%d got=%.6g ref=%.6g\n", - tensor.name.c_str(), r.max_diff, r.mean_diff, r.cosine_sim, - r.worst_index, r.worst_a, r.worst_b); - } - - report.push_back(std::move(row)); - } - } - - fprintf(stderr, "\n## Numerical Precision Report\n\n"); - fprintf(stderr, "| Case | Operation | max_diff | mean_diff | cosine | Tolerance | Status | Notes |\n"); - fprintf(stderr, "|------|-----------|----------|-----------|--------|-----------|--------|-------|\n"); - for (const auto & row : report) { - fprintf(stderr, - "| %s | %s | %.3e | %.3e | %.8f | %.1e | %s | %s |\n", - row.case_id.c_str(), - row.label.c_str(), - row.max_diff, - row.mean_diff, - row.cosine, - row.tol, - row.status.c_str(), - row.note.c_str()); - } - - return overall_ok ? 0 : 1; -} diff --git a/tests/test_phase7.cpp b/tests/test_phase7.cpp deleted file mode 100644 index e577495..0000000 --- a/tests/test_phase7.cpp +++ /dev/null @@ -1,244 +0,0 @@ -#include "sam3.h" - -#include "test_utils.h" - -#include -#include -#include -#include -#include - -struct phase7_case { - std::string id; - std::string label; -}; - -struct tensor_case { - std::string name; - std::string label; - float tol; -}; - -struct report_row { - std::string case_id; - std::string label; - float max_diff = 0.0f; - float mean_diff = 0.0f; - float cosine = 0.0f; - float tol = 1e-4f; - std::string status; - std::string note; -}; - -static std::vector load_cases(const std::string & path) { - std::vector cases; - std::ifstream f(path); - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - std::vector fields; - size_t start = 0; - while (start <= line.size()) { - size_t end = line.find('\t', start); - if (end == std::string::npos) { - end = line.size(); - } - fields.push_back(line.substr(start, end - start)); - start = end + 1; - if (end == line.size()) { - break; - } - } - while (fields.size() < 2) { - fields.emplace_back(); - } - cases.push_back({fields[0], fields[1]}); - } - return cases; -} - -static std::map load_meta(const std::string & path) { - std::map kv; - std::ifstream f(path); - std::string line; - while (std::getline(f, line)) { - if (line.empty()) { - continue; - } - size_t eq = line.find('='); - if (eq == std::string::npos) { - continue; - } - kv[line.substr(0, eq)] = line.substr(eq + 1); - } - return kv; -} - -static std::vector tensor_cases_for(const std::string & case_dir) { - std::vector cases; - const auto meta = load_meta(case_dir + "/meta.txt"); - const int num_slots = meta.count("num_slots") ? std::stoi(meta.at("num_slots")) : 0; - - // Tolerances: pix_proj/obj_ptr/attn_input/sa_l0 are tight (1e-4). - // Memory encoder stages accumulate error through bilinear+conv chain (1e-2). - // Memory attention inherits encoder error non-hermetically (5e-3). - // Propagation accumulates through full pipeline (1e-2). - for (int i = 0; i < num_slots; ++i) { - cases.push_back({"phase7_mem" + std::to_string(i) + "_pix_proj", - "Memory slot " + std::to_string(i) + " pixel projection", 1e-4f}); - cases.push_back({"phase7_mem" + std::to_string(i) + "_fused_input", - "Memory slot " + std::to_string(i) + " fused input", 2e-3f}); - cases.push_back({"phase7_mem" + std::to_string(i) + "_fuser0", - "Memory slot " + std::to_string(i) + " fuser block 0", 1e-2f}); - cases.push_back({"phase7_mem" + std::to_string(i) + "_fuser1", - "Memory slot " + std::to_string(i) + " fuser block 1", 1e-2f}); - cases.push_back({"phase7_mem" + std::to_string(i) + "_output", - "Memory slot " + std::to_string(i) + " encoder output", 1e-2f}); - cases.push_back({"phase7_obj_ptr" + std::to_string(i), - "Memory slot " + std::to_string(i) + " object pointer", 1e-4f}); - } - - cases.push_back({"phase7_mem_attn_input", "Memory attention input", 1e-4f}); - for (int i = 0; i < 4; ++i) { - cases.push_back({"phase7_mem_attn_layer" + std::to_string(i) + "_after_sa", - "Memory attention layer " + std::to_string(i) + " after self-attn", 5e-3f}); - cases.push_back({"phase7_mem_attn_layer" + std::to_string(i) + "_after_ca", - "Memory attention layer " + std::to_string(i) + " after cross-attn", 5e-3f}); - cases.push_back({"phase7_mem_attn_layer" + std::to_string(i) + "_after_ffn", - "Memory attention layer " + std::to_string(i) + " after FFN", 5e-3f}); - } - cases.push_back({"phase7_mem_attn_output", "Memory attention final output", 5e-3f}); - cases.push_back({"phase7_prop_masks", "Propagation mask logits", 1e-2f}); - cases.push_back({"phase7_prop_iou", "Propagation IoU", 1e-3f}); - cases.push_back({"phase7_prop_obj_score", "Propagation object score", 1e-3f}); - cases.push_back({"phase7_prop_sam_token", "Propagation SAM token", 1e-2f}); - return cases; -} - -int main(int argc, char ** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string ref_dir = argv[1]; - const std::string model_path = argv[2]; - const std::string cases_path = argv[3]; - const std::string cpp_root = ref_dir + "/cpp_out_phase7"; - - if (!ensure_dir(cpp_root)) { - fprintf(stderr, "Failed to create %s\n", cpp_root.c_str()); - return 1; - } - - auto cases = load_cases(cases_path); - if (cases.empty()) { - fprintf(stderr, "No phase 7 cases loaded from %s\n", cases_path.c_str()); - return 1; - } - - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = false; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - bool overall_ok = true; - std::vector report; - - for (const auto & tc : cases) { - fprintf(stderr, "\n=== Phase 7 Case %s ===\n", tc.id.c_str()); - - const std::string case_ref_dir = ref_dir + "/" + tc.id; - const std::string case_cpp_dir = cpp_root + "/" + tc.id; - if (!ensure_dir(case_cpp_dir)) { - fprintf(stderr, "Failed to create %s\n", case_cpp_dir.c_str()); - overall_ok = false; - continue; - } - - if (!sam3_test_dump_phase7_from_ref_inputs(*model, case_ref_dir, case_cpp_dir, params.n_threads)) { - fprintf(stderr, " [FAIL] could not dump C++ phase 7 tensors for %s\n", tc.id.c_str()); - overall_ok = false; - continue; - } - - for (const auto & tensor : tensor_cases_for(case_ref_dir)) { - auto ref = load_ref_f32(case_ref_dir + "/" + tensor.name); - auto got = load_ref_f32(case_cpp_dir + "/" + tensor.name); - - report_row row; - row.case_id = tc.id; - row.label = tensor.label; - row.tol = tensor.tol; - - if (ref.data.empty() || got.data.empty()) { - row.status = "FAIL"; - row.note = "missing reference or C++ output"; - overall_ok = false; - report.push_back(std::move(row)); - fprintf(stderr, " [FAIL] missing tensor %s for %s\n", - tensor.name.c_str(), tc.id.c_str()); - continue; - } - - if (ref.numel() != got.numel()) { - row.status = "FAIL"; - row.note = "numel mismatch"; - overall_ok = false; - report.push_back(std::move(row)); - fprintf(stderr, " [FAIL] numel mismatch for %s in %s\n", - tensor.name.c_str(), tc.id.c_str()); - continue; - } - - auto r = compare_tensors(got.data.data(), ref.data.data(), ref.numel(), row.tol); - const bool ok = r.max_diff <= row.tol; - row.max_diff = r.max_diff; - row.mean_diff = r.mean_diff; - row.cosine = r.cosine_sim; - row.status = ok ? "PASS" : "FAIL"; - if (ok) { - row.note = "measured"; - } else { - char note[160]; - snprintf(note, sizeof(note), - "worst_idx=%d got=%.6g ref=%.6g bad=%d/%d", - r.worst_index, r.worst_a, r.worst_b, r.n_bad, r.n_total); - row.note = note; - overall_ok = false; - fprintf(stderr, - " [FAIL] %-28s max=%.3e mean=%.3e cosine=%.8f idx=%d got=%.6g ref=%.6g\n", - tensor.name.c_str(), r.max_diff, r.mean_diff, r.cosine_sim, - r.worst_index, r.worst_a, r.worst_b); - } - - report.push_back(std::move(row)); - } - } - - fprintf(stderr, "\n## Numerical Precision Report\n\n"); - fprintf(stderr, "| Case | Operation | max_diff | mean_diff | cosine | Tolerance | Status | Notes |\n"); - fprintf(stderr, "|------|-----------|----------|-----------|--------|-----------|--------|-------|\n"); - for (const auto & row : report) { - fprintf(stderr, - "| %s | %s | %.3e | %.3e | %.8f | %.1e | %s | %s |\n", - row.case_id.c_str(), - row.label.c_str(), - row.max_diff, - row.mean_diff, - row.cosine, - row.tol, - row.status.c_str(), - row.note.c_str()); - } - - return overall_ok ? 0 : 1; -} diff --git a/tests/test_preprocess_compare.py b/tests/test_preprocess_compare.py deleted file mode 100644 index 8746ad6..0000000 --- a/tests/test_preprocess_compare.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -"""Quick comparison: does PIL resize match C++ bilinear resize? - -Instead of replicating C++ in Python (slow), we compare: -1. Python torchvision preprocessing output (already saved as ref_phase3/preprocessed) -2. PIL BILINEAR resize → same normalization (what C++ tries to match) - -If PIL BILINEAR matches torchvision, then C++ should too (if C++ matches PIL). -""" -import numpy as np -from PIL import Image -import torch -from torchvision.transforms import v2 - -image_path = "tests/test_random.jpg" -img_size = 1008 - -# Method 1: torchvision pipeline (what dump_phase3_reference.py uses) -img = Image.open(image_path).convert("RGB") -transform = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(img_size, img_size)), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), -]) -img_tensor = v2.functional.to_image(img) -py_tv = transform(img_tensor).numpy() # [3, 1008, 1008] - -# Method 2: PIL resize + same normalization -img_pil = img.resize((img_size, img_size), Image.BILINEAR) -pil_arr = np.array(img_pil, dtype=np.float32) # [H, W, 3] -# Normalize same way: (pixel/255.0 - 0.5) / 0.5 -pil_normalized = (pil_arr / 255.0 - 0.5) / 0.5 # [H, W, 3] -py_pil = pil_normalized.transpose(2, 0, 1) # → [3, H, W] - -# Compare -diff = np.abs(py_tv - py_pil) -print(f"torchvision vs PIL BILINEAR:") -print(f" max_diff: {diff.max():.6e}") -print(f" mean_diff: {diff.mean():.6e}") -cos = np.dot(py_tv.flatten(), py_pil.flatten()) / ( - np.linalg.norm(py_tv.flatten()) * np.linalg.norm(py_pil.flatten())) -print(f" cosine: {cos:.10f}") -print(f" pixels differing (>1e-6): {(diff > 1e-6).sum()} / {diff.size}") - -# Method 3: What the actual SAM3 Python code does -# Check what v2.Resize actually uses under the hood -from torchvision.transforms import InterpolationMode -print(f"\n v2.Resize default interpolation: {v2.Resize(size=(100,100)).interpolation}") -print(f" (BILINEAR={InterpolationMode.BILINEAR})") - -# The key question: does v2.Resize use PIL.BILINEAR or torch.nn.functional.interpolate? -# v2.Resize with antialias=True (default since torchvision 0.17) uses different algorithm -# Let's test with antialias=False to match pure bilinear -transform_no_aa = v2.Compose([ - v2.ToDtype(torch.uint8, scale=True), - v2.Resize(size=(img_size, img_size), antialias=False), - v2.ToDtype(torch.float32, scale=True), - v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), -]) -py_no_aa = transform_no_aa(v2.functional.to_image(img)).numpy() - -diff2 = np.abs(py_tv - py_no_aa) -print(f"\ntorchvision (antialias=True) vs (antialias=False):") -print(f" max_diff: {diff2.max():.6e}") -print(f" mean_diff: {diff2.mean():.6e}") - -# If the default has antialias, the C++ simple bilinear will differ -if diff2.max() > 1e-6: - print(" → antialias IS enabled by default, C++ bilinear will differ from Python") diff --git a/tests/test_preprocess_match.cpp b/tests/test_preprocess_match.cpp deleted file mode 100644 index 58cf631..0000000 --- a/tests/test_preprocess_match.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Test preprocessing match: run sam3_encode_image with C++ preprocessing - * on the same image, then dump outputs for comparison against Python reference. - * - * Usage: - * ./test_preprocess_match - */ -#include "sam3.h" -#include "test_utils.h" - -#include -#include - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string image_path = argv[2]; - const std::string out_dir = argv[3]; - ensure_dir(out_dir); - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - auto img = sam3_load_image(image_path); - if (img.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - - fprintf(stderr, "Encoding with C++ preprocessing...\n"); - if (!sam3_encode_image(*state, *model, img)) { - fprintf(stderr, "Encoding failed\n"); - return 1; - } - - // Dump outputs - sam3_dump_state_tensor(*state, "vit_output", out_dir + "/vit_output"); - for (int i = 0; i < 4; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_det_%d", i); - sam3_dump_state_tensor(*state, name, out_dir + "/" + name); - } - - fprintf(stderr, "Done. Outputs in %s\n", out_dir.c_str()); - state.reset(); - sam3_free_model(*model); - return 0; -} diff --git a/tests/test_pvs_point.cpp b/tests/test_pvs_point.cpp deleted file mode 100644 index c37bdc1..0000000 --- a/tests/test_pvs_point.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Quick PVS test: single positive point on the llama image. - * Dumps mask to file for visual inspection. - */ -#include "sam3.h" -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [x y]\n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string image_path = argv[2]; - float px = argc > 4 ? atof(argv[3]) : 315.0f; - float py = argc > 4 ? atof(argv[4]) : 250.0f; - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - fprintf(stderr, "Loading model...\n"); - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - fprintf(stderr, "Loading image: %s\n", image_path.c_str()); - auto image = sam3_load_image(image_path); - if (image.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - fprintf(stderr, "Image: %dx%d\n", image.width, image.height); - - fprintf(stderr, "Encoding image...\n"); - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "Failed to encode image\n"); return 1; - } - - fprintf(stderr, "\n═══ PVS: point at (%.1f, %.1f) ═══\n", px, py); - - sam3_pvs_params pvs; - pvs.pos_points.push_back({px, py}); - pvs.multimask = false; - - auto result = sam3_segment_pvs(*state, *model, pvs); - fprintf(stderr, "Result: %zu detections\n", result.detections.size()); - - for (size_t i = 0; i < result.detections.size(); ++i) { - const auto& d = result.detections[i]; - fprintf(stderr, " det %zu: score=%.4f iou=%.4f obj=%.4f box=[%.1f,%.1f,%.1f,%.1f] mask=%dx%d\n", - i, d.score, d.iou_score, d.mask.obj_score, - d.box.x0, d.box.y0, d.box.x1, d.box.y1, - d.mask.width, d.mask.height); - - std::string mask_path = "/tmp/pvs_mask_" + std::to_string(i) + ".png"; - sam3_save_mask(d.mask, mask_path); - fprintf(stderr, " Saved: %s\n", mask_path.c_str()); - - // Count mask pixels - int n_on = 0; - for (size_t j = 0; j < d.mask.data.size(); ++j) - if (d.mask.data[j] > 0) n_on++; - fprintf(stderr, " Mask: %d/%zu pixels on (%.1f%%)\n", - n_on, d.mask.data.size(), 100.0f * n_on / d.mask.data.size()); - } - - sam3_free_model(*model); - return 0; -} diff --git a/tests/test_random.jpg b/tests/test_random.jpg deleted file mode 100644 index 76884bbff867cbfcaf655ae60fbe57da273a6933..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 185354 zcmbTccQl+|^#410PY^W-VMG@#N{}GJjEUZj-rI~8M2!*>f-o3eM2X&+!RWo`gJ?6l z8KNfy86>&+{(g7eyVm{d-n*amoPVCPo^@X9oVE8pd%yoK{oMdC>S%$r03;*;0Li}x z@D~So2B4;-q@tvtrlO*vp`oUwXJMeHqoe0!zH^&}=PvI(p1a)Kd_YMdK7MflZf;=( z5pgMLIXO9AAthBs85K!cIhp_Y2?-4i4Luz_2Ll6#%zf_rGXJ0LZ!ds}nq-4Cij0IG zK*~fy#zgXW5WxFyofIVh%K-nUk&u#+Q&3V-)6mlWo6yP#ASEFqBPA!JpdcszH#_3r zcK|sP1@nDrb;>)24pjVpEHcq)pQr^i8v9s{X1)Vu9sOfyXxZ2~IPVHR5E2#P#+``hz$=Su#&E3N@;ALP?@T-u}*f(+UZ{H;(BGNN5ky+U} zxy2==WuMC{Dyy2BTUt@*w)T$xfx)5Sk_J4C>0rX@f|2B_| z3Gft$eyHo zO9578wTn^1eZ|Xsg${5>#}!vujq>1=<45UONk%3?J_RF-xD5bgx0^5ij#qd;mju{_40nf~}^n>ms|DbeMt-Jy(zBZJ@!CSD_(HlTYTjo)iOf z4>wKgT&XqQip55$0jrq7sv#pS$zc+>%KZib?2J4VncgIY#txDz+k1dOP!TI;*K6=IEk-uYf2fPfv3H} zxUQbJN(`wGIG$103v!7dlpwb_`2vlE97ltqKCSp$`~^_a`&*i9YEMD@XbQ*o#Rf%M z-n@g3G<(&&^;>SkZ#nudDRQX9c<}3@1I9cq4&E)HW`td_gKrTErnD8G5|f;&3bkad zD>bCEWS9Q}(t9>>-E})sms{<2T?aF;`g;BQf~eC#hHLLxJTTJRn^F5t)rW^6N&uKf zK&rafS7|_&aoDnR;SsF9WjS-(q;>VWM=T)5G#da_DDM&qKYi1RGed4m%d=hpH!B;Ra)TBD4Zb zpNSijE4n%k?oI;YeR8BCpPhZ^eCEnA=%qSU(Zy&z!O5t7fgJhLLWHx#^9qya6B_g3 zj}(&gJACSG{sM&dF$xh}`7VK8%c}>!Fb8FM%X{6Jv2qXTZ&yj(uK7{F)9|_boHA)0 z#{i1m0KQA0^H1Y+Tdy;o6|)*Ru(ge!bFi6W1+*ftqGi5gQ{qQe9e{jfd;SrU{hXns z_!(8cW1IPD)No&$EF;ac=KG0=$1x0+xoaTyBgS>Avl2JgR)0Ft_UFFREl1(9+#8f| z=@@>a;NA2t>|a3auW;1NUEwy9onYOy-*q3)cTu^lg~ATdiu{PY>F6A`y`(0&yjV)2 z{#qX)>L6wnAbHJXGHa_z`l?z(D=LvZeC|*@sy!ni*(=Ffk)_)4lTeai& z=Us2ZxC7Khn1&?-_<5FB4Bt}%zOza%l7}y|9)@gB-AWFOFxqcD?Z(AYO)^Xm7H5U*RXdnf!lnn&p(k{?c$V718vsE}B6?AU;7=250Gs+0i5nguBAFf~B2PX>~=Ub*F#f z&a$Tgj2H~-X0 z;!h``jypQ24}w#*!xWv>WMcv$&LIv7LjK=FH=MUC?E8b;=4ySMS|BcFA&wo%cr%-% z$ndBq?te7EiqrEvV2<(kFF+3^8Pb`Oe|*fBrE~>8T>Y|&RUa8=+)a-wkUL5jgW`1E z{K$>^AEe!U_2%nm1CT-le!7T4@!v?tlZC%}<>{@3t?uU$S8>5%ww1;)S+eOc50}8I zJ=i1k_{EuyMeS#Q3Gv;GVT*n^8_;#hBxYK=y#eA}P%heA;r58FC~Me)MpTkhK{-|a zF`|m@BXRdn!=LXjpABzHJJF<>)r>nO(t7*&tDWmv z{e5OFOsPE?XY9z1Y#cCigPT=DN=6}n0yWJy~^@*JzQ!tF}2JGfxBMI6~DS!%jRpluD$*p zq)Umyvb^@6*9X5Mabe^}J{Bc0`?!Y%8wlF8w(v<3E2}rqEW8A7U$g zp3_Y+?WV9ioA@W*(NQ|Piu`dgeVg_uFNS7KL3<&Dl0q@n=m8c1qZyE?H< zWcna)iORY4`-g-1Qz?hurJ5+_+h(I2q`3rtLiY!+(i+svVQYj>)BFl-+V;@tFMy$Z z%FC_w3Ra9*qwxW`bljP`4^sI>l1v{7Wt5^lb`>3{TX`ULTf;asiw#ZtU&1F+z_BH5 z>a%z%_p^9XghKe58_?baCMLK|!+XzSa7TnbT{=0+W#CR6xtk&|ByxkRT(7I;D-a4U zIWygGxQNsEk!AjSIkQ1c{4YSnW_gU*;cW08x&P%4m(6@(UVXOU1nXE=W)Cp;8|w6i zDfcYL``+=iJTx4+QDG7LQ%!ecndDqU046DgeIwLbjl}8=)kS6x~oGhgg)N2 zb6#RMM%eu5Ha_Q%J7EL~jrTF*R4~O`mOmn4mDS;>>$}~!RY3iF-DJ&sd(!Lrct~f-A{I!s-(ib6;e~=C!QOa@7we?gZgXD zR2$XspJx?H*EN7XGC`7xBz!x(b|SJewDwBO%g0WbP|03i@Y@V~*;T&|Q4Ngw3($wA z!yh{2bY?4b_GvnkwTY5)KD%?>aGQ)N_Qs;BQmm_Bb^QJNciYbhDh3+Ud}rb_d} z0(s4(a$sdI-??A{m+#GMYU`%UuF_WM%hqnlT!h?=mxIFh9_V`Q+^M&xjmU)~mC5HBV_nuywX24n6;dEQ+F7~(NvE0E^^qco#W1`1QpdOjq(rtYJ zVMWz<HQr2l*-4h*a+s@D=O*6s-NgpslA=LON$xDlg(3Kl@X&#|)R1aOd1exp#ILCR0hEi~0j z(QP?_3U5Z}LiF*Z!IVI9yv%VD*=aN`|EN{6!ss|`&WFF9LeH;}N2lTbyM9u#6fH{i zY3X;})_Ov9-1A?j(wVf!?0@;4;4sI2~L9@(PT_MHk5;FT0cMdgUk#OQOMh? zhPm)xTl-qCc5lpVR5FXT{Yy#t2;-?z?W6dci3guvQ==iTn^mXYi&sipbUS8?cbjdD zRAQ~^!%KLiO=_vU0#r?L`nI@r_=kvh}4Be_8K-EBTCM1LkiX`~~4j1DWGuu(ZVdU7zuw?ras=(uRnS3mRZgt>yG{FEqn zjoUbfs`Xz0TQJ#wTlqZcCV5QNc`oX2|Cw}d(NQPfTQ7>`=Ku3*HHuV^xZmxEU zgf3X|pQ#j&2WgwFXZ@%LM6E?r`|wW5Ew$}kC5hzEtp#O`Uv0nRjyuSK-|;%@k7Lvh zIy21RrKF)B^F)~qjeHBMdk*RrFA8B$E?nI($ye#A2rZq|vNd$L){CMZasP?#r-aE*8S$Io&G*8b!|EJk!W?CV z#nLoMwypo61xs^sq8&^8R)<4q6>b5rG<@NWgOUHath&2C|1b^xC@^^j;-KhN&V<$* zkNU6p;NW@Xl6&FElkRRSVxY`r>A6|xNj)?e)$@H(03`W&#puNgf#u!dSuLIvonclJ ztxg5bFSnx}$A)@^y!^GW@!~&y!B1d%_d-YE+k9kP*#Vs5Lmy~+x#txft{tZy)Cj#l z1Zoqk$6A@l;x;>u%mjphQCb!$HCdOkLVg2!J@*TAtL9PENx72w6L@NV` z396&}x7^wn-B~Du;#a@?xf$Ha!fG)1rrR4~Q9l0w-O|HuJXx8V`p)nZCtsx!@OD81 z&93iZn>@P}+NCr8)$t^Rk&B-$Cpf&&$*-*r3ODo(QRIrc4FU0`K#k?%ZhB`0x<`Qs z@C!*EU%zptq(ZI~f33j5{x$d9*^`MgHib5eu@5^kltQ$u!4IeNOcRv-aHQK4mSw!n zvkSzFMTmszi}pPKfN!(lbv=v(4{qKrvY-XAgmyVAg&I}(^g?DQFBVD#Y<0;4y%*}33-uft;jha3oJgB;zK8zpuGk^PI_qzI=ML7 zEH8KV%U(~O2!l9>L+t%$B9aEYlxZxBVr>5C5d3R79lJRE7Ri0zL{Q16y$7$K%&&KK ztUlF!=;}d5GxgO!*oT0tMXs`M&G;SHJ$%IVF6l=8yjuHL&|W!F zLR|5AyD-}A;i(y}>#%OMi?!fAbcVZ@8MktRIP)51MXFO94-~$2#S@!)WAI@!DJ*%$ zY29DM*0jC=t9S0(v!@PO_z#&oMy(*0iGBvw&z#dEI%bzQT=O{}xRHD(ICDJ z!)f_wyzb_@>nAZN!n8%W^r=HTg8zgT%Gcp72hKECq%i%i_2huvo~+U7%-*IeWs7UH z_5J$_C*noPQ*wJe_4si=p|MTUZ!dFj>|!xEZWbGuUz)5s^W4&LlwLtJ<~q_4IV$u^ zO|jf!gr$RZa8V3sG|5Zyc63=lbaX^8>X!Pwbv;_2!})zm8iq=k3qLG>HN(*o=40mP7#=$qJlgp9gM(hoi4 z_R$ZylX9`L1AaNmt>a@{tc^IG>sS$9?y5- za=+L=inxo@LNCIx#XVC@D5i*dSO+mDBYQ$jii^_#P}FTfzb)31-#C=Fa0cv7bngz! zrYn`2ti8hqsH(KB@ugJTRoVyK*6PqT^2)JK)L%K&n6z@@uy=BqR_7$5(vbpO{^}Kh zbmj;VAO76q-W9$(rITw0s?@+3i15fb36S^rcv5Q?k-?g!b*1{)Yo}#x`oikXSJCIs zyIl`)<~Qdv)qCqz9c;?WmdMwlmbLvdIJxgJlYNE>_FV?v8W!B=RQ@SE!f0@5LatI> z%~x41h-iObq!F0Y+U2_eJ@OB;iMXFWXKRcmbF=t8=XHS!`s8-!-RR_`5G7)CuTifI zAaT|@#BkjC9<0e)v^X0VJQ|eN(#ULoB*1mn*PEMxUu;Q9@cvh7Ttu7@)9)gZGF*Qq zUmk@8&6T^YH>_UL@Z^g*G4V5MA_v01#ED7`MM)rfguN+%PNFov(GE$j7K0Eb5F1CV ztT6AZ(dbBARwounr zS6x2XlSOzLOwk67H78i^wAnT5@@DbZgdyaCZk)nuinxM_kA~dCTrjQgIueCg%cO(d z>Mw6yX^o?!=&DG|e+2^~Y$Pyj&^#hAV|*MZAhi2Hp6}RRHXo!egdb6T^k$mJ8*e(ZF9Mp9Yw0JyU z_KBf0y)hxYr|DTvl&p5ksJf_(sRloMxxbOSfgV)NME5AIG=8(r@%fok+@-S!$IExj zPt<&A;DB98h_Menvml=IO6*oYL zrGh1zjn0%tpZt;tI&6wU0;(dFQ-4l%ruVO)yInXG4r+GOocs{~nh*?I|2dk9nPxqe(jyqGf|+0TKH}mE&N4`{)tlLXo!LH0wuawFZyayZ(BJaU zKW4F8DNskc&w1fKueS_eyMh!Qdy|}=*!NT7MzNGDP4aM4H)PmQcUxwZSG9}t9g&SB znW_SuzLAVSnEvPH@r0dTZlK-kbgS-3f5TGZO5OSS`ovj}z99--?O6EYv;-PH%a_7u zqy`b~1_SXTup*{WxHe0}l=sCQ8LJmex5t~9zcBKBx9n&DGRjP)ke(S0eyFgNLwYOJ z+-^5`w?##7nB;B2n_VdmCeHhf_R1X`K=V8#Rmv@nof9_RVE9wi===1Tv@>q2LhSuz zjXsf==7!Y<**!g5Evu~$D&q_TjW2j!d90&#?<}l`VZ`M4H*=5gsH7y&t=8uPd!LV9 zmknw(HE!dR^EYRwtfD2Pfih$Lk*|7c9ep1x=+$E^Y^$247)RHX^y zuP|V0+K^}Gc4<0@ZE#bu7t9DCGD9BaLTN0Xy_=W}{e1L`S5}H9RVNlr?$TtOuSDeO zu)6+@+uQ9%ebq}!_Yasu2X%PWpU-<%5=HzZdQywULO8TJ&V0?7@h(B}_Wr-Y_=juv z;~%~+{R?=Lpx$sMN%%1Ok{mET)hREffB=XO>5q~q^3}>|OCZbmCGQ;0$jkGJ7jhw< zNzMyKD~*o$WJlAWy^<-{#+5-O&fR_h<|C;TC_I>%&J7Sw?1cvlOP21Pot7_)^-Qm} z=v}Wzh`X2WW5Aa-lJ#iQIn&qO*z}Q#)~|1&OJlS-d2D@5c&0EMjr#cs$^ zV&M6Ku7?+XRs6^JCwu`vKvas5Vrmk_dRi_ZhA`$}olH`>200+GUg4nsMntmHFkAue z#^FL-ZzuI`K>>4;a%1-q&^M3?cPI2IYK&AXDDHai^ehgagtTwN=Zlu>eH1$ueSC6%~{DeqqqZ(45t=;*Hw`DWb~{E{cKjCq^xPOC?XPit{$k|Eu$WBI~W z^El&@AMEiTJB z7TV-6ipTF>9l6ZF;JG)2b+OO=k+X`)8mhUXBG{U1eWRiVAnEPrwpteXcZxFDtxSwM@b z>osL6@`v^05bG>w^cl(Xh@~YHmssQkNhWqMi6Qn?d*x%ltgtK3wZ$71^n>eb?cjUY zPo@u0$dkIMm3Xd+0*$RxK1mp-=`x>1Ct=v!FEDUr6lW6eO%hWCD7o__k76sQ=E;P( zt=3Yzdx@G&ChSHDv6O*@S+pN$$?V=1!A}wa=ST3xTLoCr+(3E+wI=?bsYR ztSBN#*trSeRgvHi!!L1A$Vda^Np{q3>5prxje0#U92e1BKy??|+{-&h+Z<=f6&06s z;KClQEgtOPdR{L51vEeEe#_^j4a(iJgu%+gg}~ zCSMC?Ptd?pDrg8PJPE6*Rk!RadaKY`mq8G3=lkMC5f`UWX>q74wdlbm;LB=%>L46A zE8bxBjok{|g>Fl9ET%BaoX^CQc@QJbU}ep1(Ji+z3#Noub0-I7=hrPWJqnF*=8TZJ zdEEJVYwqHg?g~su1;{z6C}wNx7Vsn9Rn)?pFG)7Mv;;Qo7ZoBdQ|FerLtV>8vWso% z-uMPh#+@!>H8!IpKIX+}KjSCv|L7!t`;|qYkE`PiCr9|Hf~@8WY8yFrtjXXaU*`Je zeTD7vn=v@hkYADOMJ+nt`M%N{BQE-`Jif{wWc zywZsIe- zybBcsP1AN}Ud6YYwAl#{`e9Z(Dp@CZk) zUoSoiiwrAA1+nr5M)O0?r!remXhtijrlw)67|20lAYwKMqy*Ug1@*(vS`aUMr+mB| zjV#0qoX-mQky1eWnkUAHTY6jIVZfo)*Hz-ruxbS_uez)`d~MZQ1WElV!?pY&Zmy1mdmeT%l*;XZ^s8!pxFYi_npPJvU#@6We0rTq63xf-{f zV((Ku>{H`PiY?PIYQD7TlIv=P1$*FLt6H{H`0ITMJO2xaU-+Hd)YRiQ13gYCJrzRi zh`&gcGi;RENS>CGA1ysgOV!%a1C)ERe}Vb1z?3mC7YgG`M+!~4X0d!cL5g}sOHU6n z)FQtSEH!l@B9NBx&(4pm*uo&Eh6AOVM{q>rYn<+?Ot zl2gY5V@1)EtaHM_cQ8Q^X=QySp&X{{yr9A@uThrh-(3>JwZ;;wFHQouK@l)c3 zSKAiiL_Y~YlpYb*SX$-NQUb6Vx5u!oNK5K-OkV0bHg)GYzZdrd&g#k;u`~T+#c%Tg ziZKco@k$cM&7n;Glm2W@ZbEl3>iR0&MTw5vCP!*s@$}B$%tX;Nix2%t-VHH{SH0 z(%)B;0;`W_j{$E6f3KF!scgwvEbHReJ;6TJwVr7t^0}Xw;e93noAuO!7$v6Vmz@ z0`ecPU4{R#6Z+Q!YtQf!)<$ySN`mLYt}x0EAy6Z_ki#1B*|j~o_Q1H06GVf8D%tpf z+pFA?Gn3&-xrNFo*(xg=)f@S5^{SmvGB*3#Oq1&khm4KnD$+W?bR(GBvdwel$* z@10lYVG$QL5372zu`lN04 zQ`QdmXu?Luvp_=7;U&e_(Fh||!M z;F7b$pSL}cm9_BJ9yk((0TyCza3K^8?`10A=pd|Dw7G6ZO*2kBpnd^qXih4=ZSt@3 zvd>hjp0kLQ4>~eop2uS;H!N>$XOg|zhfwgS!3ML)HIVTOA^(m3Yn;$)ZQsxSC>9B# zXwuv8WzSDbc-|5PJOg_8Zf-@Fr954QnPNxF8hC}NIjP96Da2d--)7ZzoG*uXqgFG` z{BwhlwaPxb-5s`AJG;VStC5JnN3&e+d>%P8{Aaf|sy{w_wc5sJP8_4~UHR_m^&JM$ zu8L zL&C{+b0bQ>x%oH7|3wjmonDb_hr9^D9Jwb{Pb0`qA=Gr!dA{CSzjnyPN<7=0_obCw`t^jzO6H;^3}e_uM_ld-m_D zVs1RQb0_*B~)S`Xa3KN90)LWuD2rkNa)vlz(7)i0$7i6C*{fpLlnIHS4ezn<2EKZHBr1rXg$Z7 z^+2%$S>9O&%6fDVX;R(nLpb4YBX-DllD0cBkN z_&3JLN7c8TVJ5hlSD-y!nD<^~E8W(H38cG|Vtgw*4>$~DPj^MNZawWF_g4=V++D`U zhuEjA)F4lb-jVyHvVNfw?L5K)G)xe(db(nY^_GI6R9C01z*nBn~`NV358reRo<@e@1=$PxcF2wW7>V{`+d^GRqSl4W^T6I8W zjX^3Ud1A)mThgK*h%j3pa`qBkI5>xc?~%yG;Eclp$T&V4wx!n0$)pak2=`h0jE!AOkDl-JA73X{z#A-;P|QQ!R(sa_cc(;jzO_T zybF$sM|u#w0c zPm|j#OJ;RU&w4-Uxa}(xH5fAy+pBUk;ATCEeZM-wy>Rv=P$`D()nC8@zUr7^*q@!t znY^J7k6-8_H2M*y&6LAMi+l5IT*=uS@^2>PS+To*SgMNDCVdydm0s~4_tUlI_y^fq zI?mgIlmoZ`XL(LF;DKJLKxk#~;nlLxYD?=+6gHFPbKZMQ&uV#H_3K65{Q)_%rWTZJ z-KF$Ic9BJ%L()caqRil9G?2a#z|8Pup7sk*DXEAGeqG#CjZ{x&W|*WOE*{Y0Th&`o zYBJkL*hlEZ9Mf6g8S>16vE~Jz03?)BdK&?8_w}ePBwSv>2N5{Y=Fc-;ypz}2$0-@G3SyXK%A*E(=^*rSExYgM`RnL6xo;zjZJ@S6R? z2c@jbH@Z(|?Lw?YKkJLC17f4<$H@Ew8(ZR!w-(mOfc|%Oavm^_Js_cFRU=E4A=y;t zK5Kz?wRXAeMD^CNS4h#9Lw^P?))YQ_r?xCBiFB0ov*E3N{NVjajRyu z`V2<>7w}-kUQ*FWkt`<2blxY-^-{Mx@wz&wYxZdARc@V#$e83pnE$`<8G1BxcnD@4 z>bD#65-G^BS`*7l;G_j7v2dpb@Y$>53T5|bup{BB8hDM_B2_G5V^%@;u@W|HWrb^k_OxJ#uCM{5S(7#$` zjG=9?f-8>7lRFBhrLW-&zHaYl<$^ykyQ#fE=I)hmEt_mA&UNy9)X&6-MSQWXTsywG zL{{>Y>cg42K1d(ijM0g)DC?xB-<#d1QRGw!`iv2Ql%iZASj_#{vv7a{H7bv8elX_r) zpJ2aQCI1d<@)oS|*6j?F4JN`duS^QpN$8M=M0?7XS1f&F$h-7qy=WdAUq~ZOLj`uE zDoL0ay?=tDNah4~Lf;wkWo{?y94jlWum^3K@RX#r-HJMj@7swPUpvHMM$98v zeE=e9IkR@K<|ws|q_3xId6@{DZtHxt^5xNf=l&xBK{3=Uns`F+d6I=$BMZ6p@$;%x z*V45;nBRk8(4n(+J*_1i2cnfv&rhGxVmOcs@Un+V-aS5brU9mHG?4ih-(cR7f2P@R zQrT!~K9ZLf0hJ|)=PU-)q#TZeUFo5|4lxD3Ph9)LW6cqz z|7PQ+H1mZ&DLo@?taXEpIIinw*yoGIa>zf?img$ShptU|*~KEEeq~{_0kvpkH{d7u zgLbw?3WL|N$pkG&1yzobq`XquIQ2h-CF%oHOMtV`*mw8T*1YUH^9nf8un;bQbBw{KU@(-b~)P zBJijTZLw7PPl}@KuOek+cI){RbaTP*m=A|z_KBEwN2+HrYNH#w_zk${K53XpS*&j^Xrcp%dm(yQ1)@r-25NJY zAt$8ARHaL%IK;g6U_WhCat}X3?<%h9j9=T;<;l;Vhn+(r30`5~l8%C7hj$Z9<5O`I z5NEa}Ztf6=C}Q+@luUr$SpGoy2vFj|a1gJDuydPtysJ}6@Ww3}dFei?yuCo#u7bs7 z*z<|{@aBb!KP|^suY0*?aRaW^Pre8-Qi zrOk|ctpU}At&%g3Qri*UrMFJC8m3x}f$`7K68XXg7S0XCSQ$fC5(RQ+YS)rS$d zH=8Kre+`vS0I$O9&IZ_ipX$Wx7B$5THT~*)%D1Ug7wTHle3pAOc{=3ml(M7z{;fA_ z!Xk&SeCkFKE$s(`oLt-_cODSSyD?v|o7!Mm!`Oq!ZD=(Vale#6QL?2g*6muW+tZ|{mE2#}S zhs3JfvWSh1@%%ksFyB8u_A=SEaYu7WA%~G!<2&tNKrQ??w2-)9dvN*lUkRIdP>s6s zs4K~s*R%L)dw|w{-42Celwnz`P(9(>Nuo)oqu-5woO9+Z5F+Egr<$+shI|XCRnJ zfz)q#@j1-!j7<9U^q2*XRRB>Nj#qM%(#+&R+P>yu0jXr}PML4)8`I(yNd2<4C~%B_5wv7NX&KZ4aIi#+zd zEMDA1eSl;3d^)bsg6DI+h!^%I8hpvluEFLL-o8Psj%rI50jDV&kJRQ$EOkr`_NCQe z@a_f9=iL{>YGk65mhmR{{6WUZYAQVsfkgMt`W8#g&CEIt?u`4E@1^*TbSGl^{sn;~ z-O!H2;q(w${YPb!wdCMCcf?R9nU=B%Bt|2Z#NN>(+pXlSpZeXXGkx>Ote}&f+$7iH z1(bQj&&86Cp!LA8>*>qVMFtP|L+4lbEu0fz6#ZHLQxBkCd|c1q|B{<&{pXaB%1#xB z7%vS3XdpcM?JG?|6e}0{`HjSBesYOuYY=BWO!8hJ2I9ahCn1#CB!9odC7YW~;y-4E zb{;S*#gE)+$HtZMq$B=*LPm2uY}A4cfFNS2#}qdcR!SIL6t&%E_P(9BuIJaH#+E$G z9vuI)HQy_l^HIi~O4zJ=%rvQ(Gi$R*70V$sCn3(93Nb|txa>wPKpO8VV1AS@UU;ie zI-<2KIF5dNMJ)cMfq%I@1q4pl-r$G3x1}Ur+K$jtr$Vh?_AuGUUK6b07ZXl*kvE(t3bSd37W~fzFe|S=y)enky|0W%<{y`Mf}=b8yJ7 zPHgR7YhAEq6hkqLGb>qwhA&J* zQ#W$&TPvoiTKBQ-lVZtd?UW+qQCvRx`6ub0F9A{deo>#azTE@!vs2#_6!YMNX>C+h zOC|9>#BZwk@7jM^HSm9I{XBC9Ugi+AiI%~}3Q!oCW`{?`FHzIBDZLy&;*c|2g@0zl zxXMod&}|!(;d0;l=X|&ir#EQmdhwZwQigBE&(YeKu+yIU;FbZD> zf@#CVZ$WnpV7{kfB~<2Ql5uEWiS~eAn@?IG(N_Ph;q9Xn78hIjOy=7eHARgd;><)% z&Fdyf&bH~PE6is9+-}O#emj;8i=V%Z)JCqq(w4R(dSLkSD;24BZmzUZ7 z6{v4XJbICqR03J{ps~|wjie;qP1E-3(yNEM5swzIUJ;>hipzQ3gu-kt&h}m_@3%Jn zX92G(lj&;Rz{wSQCr&%|AEh?Y*KGxKy2XwtH;ACw+3y*V$-IEAxP0y)o1`L<-$2x; zUypFQ0_AmLjs9G0NjOdu1m1r+O@yT~0>0b|ttLymQCjJZ{+Qvu-p4aU$nSAaac6UJ zlL3uk=<<29kM5ha{VM?00m{BFV+7&9BW+Hce|9ZoecdtL<`MLI(w*Rah)jNxctm8@ zyg-{9%06e#Gx+qj5QOADqVo9{;W}Ku_>I^MKB9buC6vS;FIWz>W0;~_g*KW+Uh=C} z71gc1&3DSTf*kSB*Ll~ra$Ubk?KsTADv=Qrnw37pwiSMgr&0lsiPQ2t|CH=AJG*Ao zEILlFlh;N7bCHll5xbAqvz_xj0_LoSzG0yM#J!>GRoy9jxdGF&bC7ck*aK$udeOWL zd2AX3ai0;4&M}{E_Kr(iPKx^vJ|?t7Rj$;xPu=mhfC&O}%x8OmeJZG-C~3M9XQcIt zsK)i?O=>{mXWgwFR3Q(N2pCFqa^YqkTk=?Y!1#ddc+I_N@p zBixhtz0$c6qN4&(_iZjLV=0>=RTM85h|HBSg;L}}F=}g8>#N;iRSiMlO@G;7SWLY2 zeOPZGMFL+uLDvRiVgEr7B<|87t!4ZrB$zz>c1NTs{U`9B@4ee%u?!nwHjjf$V<%g` zWE)WbnM=hqKtFVh1d#BCM3%`xJ&dpa^ zF40C~cBB# z8@`Y;g2|65sxDu{6VDFQ*_Jemcat^DLVJR}&T+@f77NVXsoyf|(I0Qf53|-?FfnT) zC5fw6&g_QRgc6(U$%MH8dVCxJCXmFljE6$SEHTmijbJ6l5Pum z_~?Cy3TAYBqraN?+gNM2FQ4s_HZnUp}adNpYgBC6urrhYENuL$6n=)F* zPjT64`Skf!yP3Sfa$(=&>lW6}A$lFKtAj5}0l>XWyYE+z;9fS%7jx$VXXS0$xpye z4RqM?<0Np5n~C*|OhD|P^ad?lry#5z|E(+Zs}`6){`lC1aT$|ACw&&)ORMzKMUX^D z2rhe?ZIu4D^RI?y7mbePNH^rdhj_xzxq?Q_qyt>#lRX0BK4?Q*KhACC$ z7tei!ygVvCvpwEhb;m3{SUI6S#QemaoSmM1^=>Hjw@S7%kddnyZSq+QZ=x1vcboRD zPK*AS$A^5R6LQqKbX1Jiv~ZSII}`SA`ac*t4@W5c28^G*h0KmJPPXh3mCccveO5Nd zSvkoFA%r-bt+Ge>)(~rn6CB4<$?$c15 zN0pfx!^4r6LzYWH89B^+i~gI!5hzoU*26XxlKQ#O^&q9h&eKKXmf&9^mBtrls2&J? z4KDGy-@)3STo}}|Yt{h=@r}it>iabYlSKyOaQkMsVc~8~S6S``c0tLPrUa$0VGU-f z8SO&6fRJLPPj|t-%Z-x|wbh<_*fj^wqC?k}ns?!<@cKWh&mO~qZ3#`za(3GicPPPJ zV-74D{I@ejaTLe*q(cT2M%wRrXp(j7+rH`ct7Dv)>?6y!<*ng)gnglrb7fy)bno%W z5$OV4>`Kt#c57#`aRZuSqGo zMm9=mqqE-H>a*|P>HOmu`?^3D2En`_LVKbfME59w5@~rd$o(v5K#nU*|3@kp15+S|g1?EO&QkIg{|oqq;2m{RYO#{ufBzdGT-RR^4y z3Ly)RTSCd@I2DP(#yoJ+-?=P=DsLYD`y;6^P^B7F{x}V9HI0u4(DKRCkid?eP-CWC!~AEc)86jsrR)9%%1}6oxP1KM+VD} zUW7X?uBL=1kl8#9L5cN?05us#e7|7=6j2p?uBj%8`{oG6TS74LPa$XiRu;HwdRydp zT}7w}d&9+1G2zAsR)+ctTl+XkL5wXhS^4gm^BRJfbdcxi^j=}FMvCUx z7f#2qK+}(^K=jDvtmOXgL@i?@@e3z!)RuChgDh=Tm5TT+eqO_Q>cr5j!)GBcw z40oUWGsU9(2jwyDehg4g1YtV5A|?kdROUF|mAi%rR%_v6<20VUh-2Oz7Jl^q4N@fXZyCc&9qRl>w z3uy+62(k*}Sg@lk)-Pm7S@Wn^b5p5UOO9T;VzYp8h*uK3b~^|G^+BJ>ZYbzSj!{rE z7G@(9ZI`GQlFqE*TMwHb=H>sw_ z`!O!ZvDlq$d0TKai>x8Xh76&jdD^BSu^5Y~OM1puqd z^rLCSXJudDwdP)d=%>h!*0*N9M#C&O^Qt#uOt>Y+D+tEPQvRqDC7DF9*3|QiW*nL0 zTmoQwZzX&Wm?ofH5V?C~m&ZQR(2RoqYu0L@1!Vxd`}uE=@$MA(-oB_kgb@A}bDUr+ zP;)1hIrkL&P(+)ne~pqQ806F9r;l8 zR=@;3y_+$U+43T}?d$*&nZpqfjE|{`O)CTYxmmcQ6D35;C7lQdOuZ1!fxN6f;x%E1 zHw*Psivisp&h~xe%%EjlWR>6$@)ieB7dxE{e3dQK<5wHfAzGi?m~+(g_^;!r>g;v` z;SyVkK18QAymR~oN*_^*W6*o9&|<_&9h#As9R~+|Ru`LpM$`LsM=rzzaY$l@8qeeBZ6Sy<8o;@y${um%MHNuI5RXPhk=&PR55usKl{!?T zzna889$FQlShw1cKr1B_uG{*B9y?64Ix!x&UBvdWxGsnKKF)@Mtix4C(lGPjryVuV zaP=7;Gh`_*JI2@@GdzlS9c|gj$W3FEefN@odMMGXzWv`H8p*B+-uNdnSEgoo=D;j$ z_wEyvNKVL|amlQ))bFRN{4K}+4N7D=S3wAyPn0$ES#TSBpZ~%Zqa&Ore3mY=**|x} zugcqOoINjEPqq6{C+>Z3pSHuj!K|19_}kzv8C=FH{a+a)z{>+k+$7U*5#ykm&*f-R zX8PD!H(%9%iE#4heX2w~6;M-p5M#ygK>F7)!OS_3hsqjQd^AYXl2L5tIG;OZM9sy? zJXmVg11}LiuX+0uS$54eLdDE5#tJMF^GVkO_{Fjw`g-8%iNb|Vi+g7f(So=Nzj@DI zDV>~p*M}-lJbT&DU_lrbJT1E&Pc1VXtuaMCZ%J*j5jzu(M??DVU#!`JI-uTK$yO!& z17?62-UNYD&;a+I#K@p{mijZp2~S-B=L7lLY(McOIr^BlqwU zuzR{p>h~~eHHXNL@4G)G1-^VDPmkWOk?>xA{pi(q(jI2kMx~0|j#PWP?o?O#KfvRo zQvZLGEG-y2P}iAx5{o(JuY%>K0NfIARX$ETE5Uw?OGNlTkHLJ7xN3>4 zBFbmqiu>b$1?ddQE_S4$T)kZyYQfAaY&JzR9Yu)rx#nd^3_$&icR5%bt9gM%Uwy?n z_KaD+e?8d|My{34eOK=uWgOWZ(}Lh^mxY_l+tm0X!uxm6w1i&q+8X%9+9iPyVD)-r z2)li6O`ai^Jc|Kzp_MGc1A?)m4!B2ZYwCaibyC6HQYwr@?rArkJ`5Bcpe@DBJt-Y}HA_JXk;iZ=Au9 z$Fq+<)DJyN3r3(&iT4j_rE+Jl#f)AT*e2pX+wh2*5VceH08Og6NiWJ{X(a_}?j<`R zJ1^<#Sa{?tzu)$)_C}$cN6k*dCepG<^q)sIVf#y$9c7iJMFNYylS+ZDFG3p$zc1-f9x+2w=U8XS0o90(k=0@2sd-V|)tXfK- z<6gI?eL=~{&B}L-X+m8x04^LUb=!wGvMSy8k<|SwZZ$5|$_Sw9aC@J}(itF${GC83y{-h7N!#5jI7~R*_Vbw0 z7W)MRtMJF_Es~B(6C+mkxNbtopd=o;x5y;F5neIgu#M+UZH!rK5IYBb)f_=gn58&7 zO7`d9h4;M)n!@4R4^n3SU?24UUXObCq@zzajN@5u=lSlbDG6d(Oj=tLX*8mL-o*72 zOHlk~M9#rjuG^g|;HfeZGc#eBHM8}K^C1$+@AJ$~-!G1K$M)Dmye6*s`0qj8`yv{c zr7~`(nY?umJe;$j5C3U6{*XJZduphTJwwSz)ZvG%fB%Y|9y^V(FtI(6AWF*Ia`#GC zvC~Pt&WYT_0Q@qk!!Cq^@R^F9R|TEM5aU!s5w7J8y;}i_e*TV`PD_(@h_cV z|3ryuw`;kzKl|dQ)GH%j*{g-WDIfR4Ua@BI7A?T+33Q>rcIl_NKaBX$-`CwoX&P1B z#_76fKEnE?=CdG>N!Ztz+9TmJc*{*w-5LnbCTqe5YnZ8S`jUyZ{A4uf(!nkny%GF$ zPnKK>$YJ!@V@5^AeFvV`exkg0y5GC}KLF_DQ_8I$P4yUVc8sq1(_e^kV{I}nh05Hp5;iI zFmPbb|x4_4<0{jC+&DN-ISa6T8&1zZi6Pn0yQTP z6LTaD@O8Y024A7~v2Q?up)xi~@sD!_vY@ReExcl)pwk`$um2x_Vi#e3fIxNmZFONv zB-riT-ua^Eh&2M4-(o&v(ajs%Y54%^k+7VAiWNtFxjOxwtE}N5HH{sl+ZnqMQ6@k}CXtW!3R{WbzV5HlvYN!VP8QPlxpF1K3|D^71&wqob&m*7;Pspd_y zydg)^**BQ1m{kYfjqG73n_j%y_g{-gIVYk1&Jmf@Ew9>Y*Q-iSpoeWsQz;QPIj|tc zj251|c%fw5E6It`S4ZLodlYd`OXBZjSrm=7J8O&CXgejvG7G|oaLEPtZv7o#oe*3rMF_mys7o19cX#raAe_#f*>8?o!IJ{dK! z{eC}UE*zpV#;w9rMW3d-P6!Ba3=2DKYxIZrBo`IEFgiP1wI`G`T4EY6t~~Lbwk|r! z<$q8AG$W}D2)K9)c=76_=m>aNx0+|@`^3H3I0bQqf9oKEE649gyX zs6iS1DKpO^Yh~jM|D_NLd;dk-%T?U->~Lu)Dd{=N`Sr)T*ZW1HD9`bB?$ir`p`V>< z&$*azR5O~hQ*_t0?Bo!C#c!NVScQePt9kLL;&+rcwq>-X1)X_uyF=vA&fSnz63nk0 zg+VgJ-X>puermWELr0bO2Z6p}X;89Ih^~}eD6MDwm=g5i|B{oGwuzI*g|wM4Q?RE0 zr`Y|mz+w8ymw@~Ay^U>M^7go&S(g#eMZ10ct49i5zs6fKtCEElD&mnz$dt#yD+&Sud1_`cb4BBivnsT%&dbK9=pNE^qFPk(d6ZW3_wYu9SG;xisSjwWf zzBitVncM?5=d5)DUP@BjCCByTWRyG$5^HBVm0^T4Qdv3VP%>?%%V@0JG6~wv-uq|N za#Bk2>iG@R%FY$SbpASp`(hP(NdE)O2CI4Kv=z*i-xCA|y1sdNFdU!$@p%VXXI4qj zSD!EbOzkPy*+lR1b}%J_AAt2OaUDc1^ZrdtJH6~_%2>SJ)8)H9VO{RF_TQ*egw1oz zwq(E1SaCbtrKL=zkm~!OX1fM{XM1|IU=Xk1-Gm5BrT8~1+@wkQOVO$ZJbVaqyu|2` zdb*eEI|*KT@_p)U(U^Z(rd44zPA8_F7+e3wkI*EuaS11Vp|Q{X#Yb+qx^#U44V+eW zfE_(aS@6bGLNRL~>6juuLp(9moQU9fsRuLwHOP+hMD>2)wRgBBq4}tm;x2LZLwCAO zaP7~S;aCw6V}jHarqQFKrX77NG|21T>Yx7s)Fm=jMhsNUjL?fb>Ii1eqLMnV`h>&Q{r%3h3oZg$SOW)rJdB&74|MH5P_l%J)~ZMi#>gh{ zztOEpidVW2uvewHB`wU-E~#Pml6l32M${NuXtO&}+c zD||-d?(=7vxg1DQe^6({%5$Hom@cj_DqlE|ion+U#HU`Wj8A$x3(ZnP;j4d_mewB^ zkm^4x(nd`C=IZ`^^n~GXS>M$?)1w1v*`>50uf%3MXR-vd%TDlKCw>WL7q!5~W8^yG z!GeQX!h;gs7AoYzPvjbuYWm$N-f!{pb*~f|{?1F}Qy6*5a-5Wd)b9B)JMOuu$4!!k zqk}9R)SbPcbq*+tTEj}$Qcj*Taxo2#;4woIBeQ`zg_Sj1%^0Bs9I&h(5&c9;5eCcl zU?PP?Vv zOt@y4a-c)+Mr%F#%0p#G;a$eC!P-q3qn+b`%;d92xbFAddQx=3@1Ea%irGX_6fr`$ z=X|d3&?#9-Ke^BZy{i86%^lpmHl=UE^6fElrc+;Xgzol}f9_f+_eQJ{BzD)XOAYKo z*s*~S5pN2K!!cKl8*MQtglRe(bIW~6KJ(!e=7~z+7deS1(>Wq)mPLqHMT_+{5pqrg z!S$~_$AQzQ{A(Z!E8DnNZ^ri6M90d?PF>e<>~ z3>-+A(dn?O0;?>dlq_T8h}yJztLh4(AM#hpN-fb@_VkxNH)pEVX7sbXO0hR5W^_4& zQbqk-ipa7CgsUIbr#9_Ro&AkdN=-<02eUE$8h4ybbd^8j{n6taUf$7d%eouFH*`cl zqBM4rBfJ3ksNTzyE~7bdbD+?H)xzfLubrJmJX)^v^sI9`b0v;*!vaKVPg>_m>>s~T zm$a4#bWRJ{We@X*y0wb(jH%6h?S?=2qr%aU2ExcltkqAQ-roZtZu9>vY84%Hr-GIL zy`b=8gV^i)zl#-xR+-q)xV6ft^Vd}Lg zf>PF?T)dLcf$I4mm06k$eeH1$yMl;KXss*6P1Ia*)iW%)+>9M2&_aeArenzJx8}F& zZD;kho*NorTE#Vgv^EQ?+=vkc$djv&@KaxftNQN4cd5{!9dWAeqDS8~nRef2(ZJ6T zFGukXY+)?Y@73M7%U_i|`BWzeYNfaaZ#T1j9ny>ntyN|ivK^eeZN zpSqOuKxjT%jz2jwZvveDtc}Agijco8gg0iAX$Hn6tQ>> zJ0^uN@^+;uEG4?pcMwvQDTsBOUG8MFioSGM5}PmlD?Xf!$g^z)w^QyVY5@bsVS6Pn zM;U^jlj92W8aQ;D8U2VEEU{dkMAjWF!=md5pdSw7Mi;OIwOdY_>d za)HEzZ}z>j(}y=W&f=4g;~(SBcZwfudnW1|Rj{Fr0Lor!VM;?3D!eBfSvj?lqu)yV zPoEdqX#E&5$as6jN}?cW&twjQt-mJq<>Y!r*p4U$ zh^Sp&_eME(J!-76R!uKy=FDlxDb6W|*OEkF!#>m+!Vc|jiF8E>Fk8jvTm+%M+b3{g zGT9*kd`Ce}599&VqZP@ayxp#(7Xr0_sB#Y|7qdTprc+^|bDD2=D<649`-NIf2E1JZ zkUNG}6!|>UDbz}63lJ!Rt?(gr$D>pL zr`K>|=Gy0yFb-Y2U2gCLB3TEL=^Q&veD5v$001zO!S01QegOmmd=awb$SYFTc;=$i z$B7gF^v{}UdeNbQdbsE8FGgJ3(?|g>0RlrI#Wjq6lv!dsC8jewEH*)^9;%W+?>6o( z#8y~Z#kJNb`z*-aASTtIoMp$~u{jf?9w!sbSzN4Fx<BQM$+j_V3%Q)1u{m}rHKr*Z(?030}G%_)Krq_ZCx zI36Hl&e!$UQ)Ei@!JVu)9><`r|FUlvFFK?b!FolY`?Y<0XAocB1KC|uD8yvHa;dKK zd<7C9sOX-|e~UhuDccnR0B^oY@{6jW>}Tp>1O09 zfxIB|pP4>9yUWgEQzHAOxStp5+hBilo{QJwXQ|ey6LS@HV(Bhx-v*W*zo_No; z#6g?%<4wuJV^!t>O<4*QiMCzHwvksCi+TJalgdDAJQ1!=dh%dul>Xss zp655gb3KY`$HZoI7fN(vlZPvZVCAN!2Xc5j8rcD28?5p`wI_+=COM7*+M-t$45oQ# z2PKOdlpAnAW8VDN4Az^$Ew|zi{=Jeqcb2}?S^WDgYNv6hlhn7{2C?(A=lsr(!JzK- z$~hSME%RS((-QNfg9CBy92Bb8LoM^jt>JTdwMoHoUPDgB2a-?OYf0A6z5~$<83sH9 znXvjWoJfICJHCgcX&YGhlDARV7AaK1{#(g5`U_Pora2{QfsV3~ZP!C5(BRhS4e2bE zk&VK=B03-(bAM^+u(9|b>6P#c5A)$?J3ULq$_E$N2RghmG z^_H}Ztzex15pys5lwF$i5$#JC*`w)Cx+$iV4U!4^rK=054W>S9WQ_4J++gPBR3ecO zYUSxrqZh4je`_NcL7jt7&_WmXH75e@?e{oeo^$?zhUd8_pVEEg2-KkXfLI<6vTcg6# zA7doxUbVg!yK`lj>J)h8qz1mD)?v9RAm}U`r(~WAMTtst zy%dSl7;qIAuMkrX_)tp<_|LySv@$1%wZ+ILI{8IUo3;J=B(!(3pFSDDx6+mDun4P4`T@m=hD|85*r4OqomlwLvUVB?mh_&5C zMu2uCq<_b+Nm z#q)QSF_J1Hgu--{Tj?))5C(#1K@m?e<<(5}uOOy42t#o#wov_KL=!+^oOlrt#zo~H z|I{_o|BW!qpDS!hnNZdOq5dH3?yFgESN@`VN@{O9l|b(7s-oTGRV8$QjhxSWv}dBj zj?#6erVwY#w)Kt*_X2!_ljEz3I?VXmT3t`l13sqIE~f7M7);*f5Nb|rtwSI%sEEZD z<{`qHWnoImW!`-^BAm^E&PkG9UlHS>!dHUq{GhRPw? zxT!KXFL7l6%?v}J21xGPIF#{HX6c=f9dnWB0pbiRSH(i#efezjLhE(}uFUeOgB7=RUhQKi zopv+L_I3+HG`;=p2gY|V@{Vt&O& z>j1ozBPbp%%j{o<-C`({%w!d8Z8!&xS^s0Bo;$lI^Y@5Go=#%@%m*6Ukb8ac>wAq@ z#KiI{96DKnJgGa3dU~B>d)J&F5@9Db&%(^9AHSU-M7*vh8@s|ZYM#;6w}JiZ7w;U| zPm1d5%2w-Zn0$4k2_I*srr%+UHjnY>43`!zAb29IyVG_NtNHPt@|v!@vwW(^el1}f9I+g{ zA9Vrwegml>wWXXn%({UdzUb>s5Tthbk;@-xes<%;G-_tboSGOK9EDdy3uKwcIaO80#a2BmqY+@7}O<(Yp`^j7}Rlb+9Q z0d8_T?JlfHGg|d;^v#g{!0E8Sxk*_m zKLZ>YP*$F4h$5d06uE*gI_4knuO>|2SU=UR$-+`1Q(#8+4qeI@1L|*3F+)X>gLO%J zM$TwmzFJhA%uxmRDCYUQM_RDcvt=PTP~F1z6_Qx69K}b-o&O&ot@~O~Y$51Nqeldc z)MfZTz|Qgdr(m1*@!mN|Y_j~5JbN30#sd@plEKVChUnkpelf|?jcSXVpq3rjg@~EA z?{KEQr6gLhs!n?GDYJQ;5oQivS|~$ncD>?gh2$5zZ!HpSYJWRsmhU;P)imL>G;A)$WiC~7k+dcOOhUn!W)_c1KTSVU?y|?McP`;`Eq`-ysXWPbq(!Bo* zlCdzDRed7*#Y;pjW?aN}@L5xTK{bABEB6R-3#->lV>;VU2 z>40zm^l^|I`wH@_FxRvKaqhPg`>$HsRR?@uAlKZmEB(vXU$H`08R6y$aPxjLFt?{# z@Qzy%qsy_Q{t+5}QArc3oCO#o5vg5R>O==-7BIwrk}`+wRDV=ofDiwT3j^kD9jaypb%u5tfuQxaRV z61LMH`x@mh-i_$;Zcys<*ZA4}^m9J<0FC#cM^gkzh%z&HQ)jJE>s=(`^Ai>MleN#a zQ+wXLtt~8I|JCvR*rJ@WvKDlm>G(-~F4U(?@sdARJ#=}&WK+w`4@JROk6mY@%z~7iq_u~sju4&%yM`; z7XzfePNDCWhe*yKO7Lk35Rl+%$#}z%{dxTfC=h*YQLKNWQsh z$tD*#T9O~KWA2z+dv*ECUM*BM%OYiG39sJAr;f6(zx}j<5IQv(BAiyL=SYP6QD>A~ zJFo>SDRBp~BWNb=PJBGB>wF%dbw%mapIb$?*qxfpQy+1vs*2Mcyi{)ZT%O3i*V9%a z@ZKsfo}n#D{D~^Y z2W8w+q2LI0HtP!Kq!Y*(4C_+M0>$QNy4K2&0Sg0M_P9qCh^M>Ac^JJnPs z#@g({nm zYcSSePwibuMmr_^^6A)gn2%a{#M9Q=-YYI|Topyo>|~+}P-Db^qu95Ia87E#^jt)B zWA+=CzVesu)I$~f#vSqa-%-NF9jr>7&^^Ojxz(LYn~)al8v!#M4D`W%{i@kc=X^C)JhuJiHRYARhrwvjJPNasC>~*Q+;gGX>3RT@0+Nq0Xkv zx);4rd)TzEH9k;pzi!+|d4hjnSv>Nkat%j7H5XQvY+vd?9LSr-Q)RBkRAigJc=bgM z7WSx#9m>Yry_hevofRO?aGoeH~m4BsHnG4W)EJULMxEypu1v^0)3PTqvQgc@2x_ux2(22a= zF^C+Pc}c@tWE1?@&cYVt^%5b`LcvhN9kU8bn2`JVZTrN3Qw6Lm+V}{vbdA1v72qEw z^~u7}Ci}R)Yi=L=(~)!-^q|Tam(AJdajkTnwA$REab{Vq`X%)(Ha5{k6{oFc+?>p| zWx%c2FCg{NnZOYY*Quh)%-6ab6+S^`hP!G#P*$27L#RVs&Zu#u0Y<@b$x*c!wIt#< zz*I+@Z}TW+5N;ZHPbQX4CU@qYYaOE(sJNjkk&FV6&iMhAR+x2$zQp8K7T<%c3PBdl zOr+|6oh)0fPZ}LOlU<$4Oyq*Iwu}vK0`6)vhX|OrmyUetQ&Zg&M3JoSF74!kr62Ga zD!gOs)N`h%P_l|y>yAy=por0v%m4!t4+0+k27}xYYxgDXO5LS@wnR>JfnD_-TuLT(zIUgybORDVNGw3*)?tRXQ4zj*9zQ$d&5Q|Sb7<)Q^hUh6Tg1Gdp;}YOC}8I) zqb?|)F3@hyS`HiSF`{=wx}JE*K`;IcF7%Z!?%Cd+53(a)o9@Yu3v}miV1{o{-;reaY*F)VcEw_&+ zlzjR+FK$MXZW4^rsHktr%&?>azDL{7`C4^|bJ?5PGK@xm#!CaW;E+@@5aKO51rm|! zjH)doY6Hch%Agz`I1QQwwWczP+q%%_QN7Q-TYH)!N5gY~WV18FD@`}dvk^Dai0^7C z-k7UcelmrWQPyD<7I=<_Zkmuq%)nG))*Aat(H6BT*}BevorWc?I8k8BrjXn%$0 z3!LXmPo(ub&b~GZm~g)Z8C+SYaDSI+9~PD{&2+TC4C}=9b{?+(lq&`f{zO&0wrX3y zSviUNkZEX3yjE9g8s;@7f_YoCU)idht{~Uya;da84MeCk#6PC^H#Ab1EF;=lCIr@j zc|(y6w-aV;Ie0@yUh)=1A{X%f-I~0~{N8AA#QjUx-~qQuH=-3KOF2dRX-6dl)pEcE zU$Pitv#0}(Q`-jy@lYa1OtyFfHFq*obKSQVkc~H7uKt3hHMr^2e36jVV=E6Y|A5^a z^TsjkbZyysO^nDgrdwyqN>-Jj^9&H@0f<`}ogCwVMN#Ra)0fEn_`RjK?sc*t&s?;2 zs;$-ZHn4Oc@|Rb=t~yH#!)&Mp(39t2Glg9RGnQ;u@XQe}^&=y3FCp?m|H5~fy^B(= z7o05*hno7fmNw^RP4=@nnL%#<4*QxndhQD)OIq@r!&8NlBBEGG3JSG!wHXK8_C5 z3Q$&}o}pD4T582<`?w-KIbTRlAq=clR&F;*uqbn>#q3|wj!Sgb8nieXGEF62OcZX# z-g6_ZReV#3g;2gdJagZ1^C9o3EI)4vnAQ6L?L$OJqvdiX3eD_at^0GPTz*IP`agQh z^E|_(&np43YTSl;7fg=A)27RqDFe3Yjr{v(IpA9ac>c#`AH<^nk`NGP&cA-2n)@6U z{BZl_M2@08d;Fy2tDI5MCt;My`S$^&z#U%9bW!WQtr@)~@^2xpy?ovD-m_x@$P81TPHo1FU)RO9g7W_SEBBZ)PzzD9&ZDKSw)O z4M&+2kHdw>@b$@1ZmaU|5jk1|#w+^#AO1Q1x6@ckH;eGSy@(GrS~&6X@BAZQ$%;4l zF-cCN1OC+H^YpPjOmoPnow=Ay#TrC5?D#w&D2$!)$g6#wrQq0|0p=XuKdXjr(6F$) z{WT+m-CXieiZ}#$z%wzdrd%XEE-p0n5)cz?nAM|685bz6PvNj{3^W-J2sG|5kTjsv z>bV)B5{6I))8XKVi5a;_T>ba1u!#NTIcU^1bJ3$KpBvubU$uSg(xtAv@Zx6{UA~ii zsg|@k+`tVU&WT0Hj|6C`laBb)omKR$AZim~it=UtAHd65Ng?%5^JupkDxP`->_Aqc zN^DLkiPFGJ)RZ%M5Q-Hss_yD*Vs-mn3$$?)N?NRg6RtC|pX;6}moC;ndV#+%h5cs5+G_{hDaROU z8QNk&zR}YtF)Sjlf5VL^B9bGJtt7TK4=hi(LDdD{xhGbv9k5!QXV%;-tnUQ43RErU z96;4MOM50f$H;YC&+KF%)hurPM4Cx3ootS!*6;7h^xY zg7S53Y6dcg6CMBK9=q9oGzRx#&D;7gM%ipn>p`jN;~qLQ{nx!tg)~#PMMWOLOtuA3 zbc$%B=_tsRu?pOm4O){P|WNL67)zS_AtBa=4@j7{%#qduO^5PjmlyE|sY zPR{{ijZDaB9wk`2liqLQv^hk`>k$Kixx+|qLv7-Xn{j3{HS_%H!fF5A!DTeY9r0lVnS{B!<2u~z+zB1oO4Fhp=E4R6e}vqtdz=}b}8Sw z2a2q0P21P9$@zQds?YIrqvuKJ$AyXs+y|oySfS6}@3!`W0ygV4f>zgV@)s_3$&`Nm zF@fxz4Kco;A`;nJ+H@B_izif2JD|A%e41D1(jv{BG0^D_ctg4K?V0WcbfrG>O3?P3 zKAiTM(0H5XiXOGOgVNT`nhb~P^ayqj>+FaP&#G9#82r=7QyBgPOI0Iv+tSLDzQ|$)MrTEV1~WXdJpE&4)YO|MO1=1= z$ig%9^kvl3`8p0?=l2D#;n&`OR=!ugJipcI01Cu&8zy4g@F@w~jLI^=|F%_TdFVwJ ztSuK(7oMJhGUX(h0Dxu>SDk!H`P#0%#dO;ohmlV87Th|Y9{A2OiJUdC_O4yN?4^N~ zMTKLwfYjpKAFg$H-7sO?83Zy^YU!ELEuLN_Onvmju1Uizf)y3YR{boSODfB{cWoU5 z{k^KNoON8}Xs5{K-RF3317Q<}?i8+=gPOD?1n?(&?Gf>rHtXor$_v9SLR)hEnN&oiRziDEDvE%lys&QPYN%dj`Pm2P=^NzqW>G9 ztl422r3v`Krn$zXHK!ZoSAV}5Eyt-And%3~_ym@;R{!WfY|2=4YRtUw0adt)g`V%< zWKbhZ%ugN{MmC_0_Pin#j(?|iy(SfFfy;W&%FiOV_>**IZmVgC(^5oa&>gS`8fKZ; z@n=IEAW`ol(+4cv0W9wk`Qwg5x!k%N@?OoX_^E==IyY}2C)cFX1Z(55SQ?yEg3B`+ zkr-!{BTpnvfvpa@HsF}kAfAlT{K0r}b8(e=u$(Cc_nN_Tz0#C-IRv!H(yHT0?b(78 zTDDJ0XZuGz-{s*hYGG1`^wm*LRpF;fdphBX>>}TsEls8~RX$$Ma=XY2L>?GoarOTG za=};IRfYQNy;M(ADGhqMK=|p)nP55*9*T$DQ_8uVL?^``&V*FM%-B6w9(p) zHyQnJ2-BTDAv%UblMk|=A{^PYGcP9$%G(*;AMxC9dPmmg21ZQGU!9+tHku;;#cVbF zGVb!<_h>6YF2GKvYpM&C6Xz!s#bv-|^@&119bWLO za)dHW!b=zmWoAyRiv9->g|Yjox5}Eucr=%&j!vrMl1DB~A^Y`>-%EO(e6N86YwFY9 zmstH9ANTaIe&PbN^dpIf>*%88a|L?1XT$0WeXQO7L(^?mGMEOWq8Y&VO=-V@v7`6o=z>9g0Fd|ww`Y+L>qYMdAAI2C{IgLDFYNLmp^7j>|t zRGUbN5v#)7Fk+TJ;gcG8&CE^czgKQ#9qLQLyTL)N0ZZ2;tm7+Z2+WIH6FiyopyRL> za@4kNb=qEabf>pMt~=A=AtmOt8>4GIa9@M`trZjM?w1T^VA5*&LdA2Y$#+SDw*TgC zFo>$++@yZWk86&JWSb_k$oo9_bsgmK(s8&anvMZBsZe3-f_hC5j%5zi@B;)PCet2@ z)<=#txa2hdi}`WCc}FRaXv`l;5;RN-(e?*> zSb;j~!XL~V4%ZJU*;kb0z?!F$2FjlWYskjT#l{)PQCV^G0&IC>YzKT4vkRtVuk;AV zs^!nSo5CiKH}-fZS1nknz!$N*bTgN5((`ZcyRL{B#JKwp z_KFmce|tX@tlBI$e!(dfA~d7?z?$r~ikep96QPidaEE}V-^Hj`s9fdc7R?vmpBJck zXE*%)lx%SEK9>wA5;jq*ijU2D^^Un1j>OQB&(L`<*=}djQr=FWy6N4=c7g8p%YV1E z7jAuLnJLuA;;GFG@-!te_ z|7;GWq^cS%lPeRC9J}kGjJXh+V&Nm&T6Qmiyi~rG;JrAdZ4mhgU2#Iu+`424r+Ctq z#(R3@$@U#an3jo*{Bi(6&~^Id0Hq9(oQH!<7oBeWDfQi8X~Le6R0o`>wa6K_KHbxX ziBn8@LKurz7H=j|>218EJ7$ofc_Kz$d7xcjzlKMx)~(IVzBc|-dpo|e_GYEb|0vh8 zKETu|{!)o9`VO6YXD@JSLSflL-nDMrS0 zPD{u8yZUJy8-Vbk$2%5fpaiD4kW>~86~&+`e}^_};`KS)xpW~q-hvo-S(0+G-?N0b zLG~OcMwQOFVcKIoi!+Pu8dxnUv1hH~8SoFe2z_90`OflY`iH z5fIxaOqkF|wWK+O$}(vw-8H5=M2}fn!c~sLhT`%qwJa*@!-psEvg8Iuqm}yZrp$>` z;~<`R{p+vMZk;l8LQ`Y>Ss%2ZcGi0ib6|+$`T6--3$`mP&(Pt3Bw<&1ZWPt&gZFNP zUylf<&)ilweh7l9bOCp9!YNWCt70IM<*OIj_@95ex!9s~r zQnP<>Ck{aMk|s-WrK`}KAe`(U!f)gHnLDlV&7{(owh=mzn3F-jxU`%6tQtruW@;eT zGBXP{!Z;>FBb@X<05YDO>wasTg~Oub=gvn`b83)<@p?t&ta#_1i2ng>Z0zCtnCVQ# zFONL4O{f7{DaRz^%$`)(1v_1Op_%WAASB!FwyLoW~@9TC|#c>h#L7j>d{_e1k|%54Rl{W zL9RPtCAEU5d$2ZS!o{oTd5u|N>Xjbo8w7hOD@){-whauJ)OLo}s94 zqNZ73u#7ev)Sb`TjG*(exhG4GN~)t7Nl)FsQ$0foO?_hxYh=S(vlJ5aRy^Y8MBi4x-;d@JzdR;sIyMwHZ{g1%Xw3G z6#jXZSi|8RomuY4iNbg}301XiU`t&*ndNIB;|~2CRLwG?-og?xtdo9=U}#D0*p6#A zpM$(LlVNQwvQE>jK7;*2_rO=N3;t~9bQYNIqx!wBn}$`mV|zVkCsJzTn}7XXI}M*V z;ZUL3d$ZF0%zWTz%dEJEA8&}+-?)BENC}7Yx3@{aTwTee!^G=&b0-xVg_=WW{MdSmqR4GlFl`E$ydBxieD(^wS}m)#d#km8!tcRK3?+ zf?8F^{U3uws`y|RgnKLyQ3?@(fu2)hv9jX$*7131&rop=r{Xq$YwE@@I8-0A8{UHJ z>urgh@sIXB*9uPdZ3stCf8^ZEv>v7iGt%WD){$RFN(dqiG9Er=v={UV5y8^EY1gIR zF)`H%UV3qLKg&Zw`XlMC7vD)T&)ErY;g6}KyOua;<7+Aon`#=mo|bpsyrl>CSfpP! zbHzq1+Yx8#VE>$11eUn8mdafS$DChZVEo{_Q1o%@?K5;7an}hIqwSY)jygv{kHYJum333z3(d>8j5QOQ@8*S-ekt7#? zHf6}*pk_EtNKcP{`5viR>_0O}TN7>23M570goYcD`pUx-KMZ*sFJN)o{B$ zWh(b;YS8iz77dc@%MwXZLR`GMz1ASm)n69wArw1H(s~NYlmjGF-up=*Qm07Lz!iyk zIw~^G(j?a{Gh6X!+R{vtz9vr^w4>@ov5oJFuM*e4RD;a`hX)xm(zErn_~uSS^WBrRRrMQRVy>1xzNXMb#Re1%IE+Z(Z)s4gCrR`;rO5@S%@QlxR*z;&SNb%+|XvCf~@n zh+EGj)`T?TBrQ79-7KY^EfmWQG%p1>Us2D~kf0DC@jS?U#3YeeRC?M~qQAgQTkR8; zc_c?E*fo=JPM(Ojb5~+T!Sz@~+tS9*ucqg!B3|*KeBnaJJk)|n`Lo_QwHp2n=|Mt& zKx2~9$A-_gg#SRF&TqkCqYN&MsLMqwpFVqKJl;F>1740-l8L3rC%qw+R4edt^Xn*W zC!;SF$)AWu4|viaW7u=;hAK%j7o9+^Rz%@lA}CW3e*vw8-sov2$4zfXH&TA?NLGIg zeVR%nZ*&$!9u#S+>iRinS!{O8K-3y5J!vVujJErDLfIbo??*bZUDBQ~-rN)5sx5dU z8zmm)UvqaclyTZ5mEqtSak2K>+plrSvu;T%?V+8pXfHAoXW_I3BWIS-Ip(KlKO%5{ zjuyd9v~vgzd=2^B-3P7w=EP;qPPJ#O<`QEJgUvyZLj6vPI9j{yUCq=g+N2*o? zIk^uyVR=@S267w}0g3Rbdg5f$TR@r<}hx zZ^T;u;Tec0{c}^Gw~s$9yA|6dSf|pkOL&`0hIrAA`7L*r32`{l%qTx5FMnXI+kWRS z1T%YspQt^G6oMkoV{W_p|NOE=G*21wqcT)yxgX0A2@lMWrI@_5+e1!MjvnH;*+%Lj zte>9VGGl29`R8A&{!yjSPG=LU-=bD~iE8BnnA(fZh29Xaw|HHMX%}jqIh6#=rSH*g zDb0M(Ix3gxkbI0*t{i({|9M7?lrQ^Oz()P=m%RWC;S75}RIF%tpe6Lm@Ly3&6edu? zTjAvv?$_G0sGDtkD45+@`Xk*y(6=G`)op)ofl{zfBf~)io*h0J5WuIxq;DCM`<}TT zj3_%k&2vwcX2_Tj^G$rv{{uAV%8&R{oh*M*o9&z#vAphOhxbF0T|Mpmp+p#{@^awR zAJRSJU)igIl{2J$V&dfyj9Sq(<`!1t7XXspOCbevC`pjqHi3};}2fcq4ToQ_ML_87&2G%XTOmVOO$!=p9W#;T#+15 zcccge7-?gVUy4_IhP->v9XMji&j|6*m>|}p)}vOVd0)_GR`bl$`?o^dk_CiG|2KN4 z^<%zl!yHKBxA<-6&cz}vSr4Dj)*W?GC;ZmZHihf}D z2h%No2NXxcTDKb$yotcxOrE#z!qT~aGkJG>^j-gXnl(cz|D7{w-5a6#1PHf$F5ToC z^MO9rMb~th>#-$ij)ezI+W;PMzBb3Mv`BxH1Eg2u@qI?GJ{H{oRcyUzEt0*t=sT|~ ztQJ>xP{kG5{&t*$m|d>DI(J@r9UMtF3P0WNToXz1-nGDi(L8t z*lH=152S?2W)K`vdyV@xZ=#tKQ0bZaZJZxLp@DU}&ct@=K*|2<2C{Z(nxw_B{S^06 z%{S?Q@e1H)H|$Js2z-45aZU@NyfVoqbiq_z6FDqL`5N#;$4044psjB$EqUeIyPnI> zp6TaVnBIe-5xCuaNAV{;>u1CKWtD~n#fDz%j0nf>$;+-PHyJ)x_+E!Qpd;sRb`fGd zDu+PpzK=oJcznGS@@*bd@#F@4c=$#S9l-^&B(=xd+AUUYPsDEr9=#RLdP525ywFU3 zfm57OwrsVV39_$-Tf1Au@e{CX2X52$nobSoQwyso#k~NH^S%uKRmai|hnvgqt3{V} zT;dLWHW#tKaXnGj2GcVjo>iQQ@w2HS4ghdJRVQBWN78$dr#O5hN!DoN;jlZLYH!s6 z95Qb60%$>(WRggf(9e@5H$_0G5Kd6ym7R^t$y_CjQ_43(Q4mGtAbVT+qOE6y74fK}ac`(>^e4-5U5w#| z&#vEXlNy4LiAVP^@!4> z&pE|#5h`W&)`*jgk-h9ZyU&Q#_fP*3Xc?Tf1(Kx&AM=$t8k2kz;uYA0Gq;YOaJK#6 zigohvgbQ8uL}eKuR_0*x36nThAGzih(O1pq|Jk%Wzb98Aby(74D7Slb6!ZUc`y4nW zsD0;KB6(k?*TF=Nu<@=x-vuI|irzZ9SGmwg573cr#k?f-)@^-!F2d4flxRl81{{^o z?HoZ8rX%SUNtZFWy#luXsA>!L!sT|q+Tu;#N5LK|zXNcG@yR`X@nX5!a>lD2zqm>n zy(AXh1ow4=B}zGMa(*LMZ_wYlyjq_loZK&m5EsiAmUdU2c`N_Qttuhn%-%cO`af7x zWx?LHOtMMm>-(3D7&mIQnHjN`PbJY4S@VFRlj)qlBnk`*-I2w#ko5kf_bP}L=_prV zg-C=g_jd+KgJB2Kh16b+xk*` z;5@OgrI0TdUx26CV>Qd0_I$1&i0crC(}?PQK?DaSd-_Yk0dDTvQYqL_8xpV zWRkM-T|T=m)`sprd!Tk1o%imXQePyV%7V;>7$((Czb8GGE!B(Yp=HyHFYSO_-pGGp zh%U3V=9c(J{R==lB^x_sw9)DKgKMb&*JJ6}#h=SwH@o&5+Y7Hn9d)+7npG7F2EwUgD-%`4?m*ZscN`s!EFsk3mxB248P~EGAFh~mHi^SUdg4JS#aU#By3N2 zQHSbr<-d)$!lbLC57udn6pvi5M$#6Dy?#HXg;cooo+^um5dX)w=G>G|;QfE4v_!+=6R(Zy(y2=W}_M26_|x z?#7)MNH*&TnbDWouyo}hb-Xn|{rE+pxc|9pSFec%(D{p0vv>KBIyDuRVuzNB;y?9? zn_4}#V}lqCy@ct^iH-@+NZvwezytH!0~LVY46;aT$VLpuM2Ol2Uv;}Pz3dd2Ax!pA zs{OdTkzXV12DdqZF)C$k{XOfAnCIr6LZ59BtLr;`7k-EsImAsE0iT6LLPLiGn1Tf^ zo&p}1z7y$3{14DHw#TX{*nsJFaeFB@;Nvft=S`CF12U(E+czMV$A#{buZs~~z{hcr zBN%#iMd=_ztJ8ozk0)Yni{Aw#qA{c&N8@zJ5Cme@^%pO`?dH_2nit&SD@~wu8cHH%k33AWSqNMMB5!>U`4%ViPc#JUo()!1uqb{(;fz z?5Xxz9vmyK-!H6_X-F4v!x!drUP%*~guTxr|DsltdS&=dr$ah1QVZfO+e|V$k=c;v z?k)lf`WwTl9kOU@oV{adiRgDUd}J|*?)+Yx+^#QWSm)jZ_5L5=9^vt+VDoa-+b_|i zYwn~xo-cD=%D?jGg#0|ACfB8w#7Z!28U(T&`+)zmsm&h1^Xo8@?PxRwSeHwllkUk= zjkvs0HgiF;waF$gQXU(oCgnR*Pvr3dOJ|ih$ijk9#LctfFfr^D|0?(*KlUW%>&OLt zSdy>k3}GE8LS&-z>ea6^`0z712Oq1hjkboO{%RQ55qja%bkWj!>$BSQZ|=U~v*|wK zQ&(xOa!I(Q*SNBn57G<&=mAHbV3yIe^PTu@DS4B0d}kjx^2JCaZAJsLN=fX2e zCjj$}uyS{{n=?w7@UjkmL#j6&M$>VcD+1tVFb=e6K5H{?^{vtjVs&R1g>fflN={3T z)1mg_WpX|jcA##UYh{x=-6Iu)SvSuWFP+WAX`Cc2Qq{8H0-=3v5WJ!J`4{ zz2gu|J4uZ==Qq4LTmedG;-o#*(TaAEtJjLWaki^>c%x{NiJk>f?&Yo&fM5zqD+-y- z-w+XiDYg1Be9`l;0j50S8W_>2d+3awO5#eU!fT*qZ$wFdM|(kBzt}jx=%6liT1s!< zWG6iWxr~#L=h@ULX;$~(e+_LI54V+mX;yk24lFN+@*fifpcohBfDPZ3QN@3s`g53c zrVq&N8aqba=_^*_!}pvB^~?vg#a;vuIA?f!c4B){q2j2?ks3|>h~R8c%Jw^2&{6+D zGU=9Y{nevwwSFxnD;ExCBx$I4oQ1K<=e}ZWL;Ui>Lh0Z6B!beS!#5p}D1S2UR2>1_ zJMmL+OWYMR2mUjF#;W=>+50J(w6r^s*R=X^MQv#@709?eUAY`4K-LBA`0^Gjr0{L5Wh) zkN%#1-%h<(jTXPJ8rWc&ixB-=-mPVWWy7*(w%Y-_e&9Jp$oXd>3~igJZXOYhZrlez zXU5M=<%cw!>GZ)j2~Li-q8d*gSIXWF=Bz=eWkHKG-I(6`j%YfG<_M&mj3~dw%_!tI2-=V)oW! zEB<611*enjC*?|z=TP7umgjDH(VKEcW&Rd5 zxZ@8e?oD9-Z4B7avSqam%yQI~scaSk&&FhpsI#<#5Gt(Ldx{+H?z|~Wl?f;>*?eiN zUzsB>FngKwY-O5+oK~8MA`k#j8=Exdkj((B@bqP}K9;56%^DdNe$ri%(8JO`8KA(l zIkRBaU_HMw`$BSz_JiF8<&ZZ8S*f0qC7&O;p8FyBab)B<8)kQ?xMpM6!0&tq*Zkms zzsr`u9!Cr+e1DDVQ888H?Z`YWo-5pssO-036cURyQ72`X8Koss)_qwZ&`ayh5^rWsaJWud`etOUD>nnV9{%(`N%OguD z{1Uk#ByC(ayT9FSw~Klnt;(~C^0 z@yPx+A6-XLE=7JVW@Hdl-S#Mi_Fd7ONutUR>=LoO{ynAmE^=&kNl5}yLTyUrp`pBT znrO6+!bk4FMv?fpLYjOGb--T-tI2ssbL`NtojEY%#Q(prr$%*8L(1(R_D6*x zx9iZK4E}@EeU=~$NN)lTpu(Hf59(R1o5=1^Vv6{H?RQ8e!~iTb)!Sr@k0aT18MZgM z#ks^o6hGQageYOum|e5IJgtErYH4caEYUC=J`&nbcBBaY(`dE|4C(zsGY9RK#@gt7 zYFWx%%<=bS2##Nd9WoUP9v|tfW*MbiqI3Pf4|^W9H0OU`vAXbdyAbth=~?!vIXkdbpCPbAJ?(wVIi z;IRK_Ayob+Szgfmz!4H)ZF4_Ivo`ea`>O7bV*Gu+AKtzW z?OFdTPXKbVuW%aYL2kn1MdMp|rTU2F6Co1~OJqiXMG$1P_;nOAJLL=G@C9Eyf7-Fh z7PeXvn2s|YDQT-;;BerSarJP`o{YxW&}NN&s}k!6la?fm&<5!cP$)$q5xl92$9>cj zPiJ2QQYgPMx5pAqo&|RahW*`QRz%{EEeI$Ww|u(sqeGTem*4&33E`*Tc#SKo=?Ny1 zekC^lo@6e-E#r|-P{lIpFiwZVEg3*CWr&*;;%ZPOT30OAE;<=*Hhh2!Ec@}|krUvX z)jtxWJX5_FD_y-svUeJ1Vx6p}I$-6hM;%<)ZmHG_GfBm$^U*cl$7Xo(|JSkoBIk!7}8*JgT_3K-CrL8#gz-FmM zMc-giC_1-=XhDSE#y_d|faO8dilK7lc#_t1Vl zROZu5#hxXgBgm}CIqxk|Jk`usqHWU|ms{zrak@kZ5I$N3+k-H{?8C{-{s_7H=V_AB zjG#ZLKVGW0Ye@ZV{rQ3&en=^)(&NF)!aNeGYF5Tf>XvFXd=)#&ir!Z_5b4)gTvBuj zLIFWpXCuLwW1=K&8-3F>Q)KJ^09C(+8;VU}K?LB=we4=@#cFGfQu6_uWxKkx9cG%1 zHG|J<-OWubAUo$1j|iC4c?LbOi;>=%Bdby zD~|{X-EIV{cziQCIhBb_luw65ZEMZrZpPV9_#i-gWxeS#1B>uZ=fe>pmG~3ca+4zA zrQB?}slrRZ%c?{nL2@4MI1|SA`L$u0hTaXA=dDf%#yRmhnnXmg^SkM~gkFHiCK^PS zJu*o}vpNI(d>+$tjfAa*M>Y8@7p~gs(1CRZpVainvV))Lf9tBm>+Dzb=WTIJG39nT z-3^7l5I)-K`sA*rgwFu^{+sjQTlb@V*U?OMl<$ECy$@7e1ZYzPENc8+6S@C)p&7Gp zVZx{PSO6{kMqMeUkVi;Iam1a3SM7Uhs?@_muPZQb=4b_tikq;-C@1~a5z-9+;3~N0 zAa}LUrvu%@^hpdVhYbf?Pcr0KuGqtef5@J8A&OwePiJZypf9|}FVqGboVJ2Am zQ7M;eM)jX6YU@9vK_iDmmcLeynV*XJpTJ8%0`_}x7x!}S7G_B4%^`x}8cUm#mi`WC zt8Sq@2E7q`fzG@e*&*mCE)d$8Sti7h7k zI_a?Xs&wrFdK%Mwwz_`1YZCfdytPc2L))KEFJ3OMOOjH7Auc&cPwbvqmL3UlGupQ0 zFkf0Wu*rsUg_)VpFI_UCJuL9@puCUu9lAP@_FgUcFI)iJGn5J-Y}6%f^hN_2O{{-V zVh!9v_`FQZ^bR-GKoKuH*tLicV7dG)xN-+20Zga*&v;2!C zS|>0)$pOHX*zg3cu)DxzGjH(mM9de5val3IH3Te8-%SC7XqfE1CX0hCzqYL@pT-Wo z7PUPWs56pzE4K>Wy;%EtyA7%7v86?6lKoP!qEBTr*QABIp(#zROG*!Xv`Z7XT`P#Y z1u_+SPza>y7aR2?M)`&ylY&q596V;IYZD{gS&_Um#sy~|61iDBZGnkYibxCVw`8E1 zo&N#kFNdK_jH5vKLQwW>hmCp|Vp$WIG!Af#%373uOTr|K-6jF3k%hb4(ifZMm)(iV z`Tytn`l_K}mNDt~uT#(1zmC89Y$V_RdM8`w<6h>O_+WzXTs;|Y#?|gdq=pXu9CW|5 z@*Pm#_Y0RV(DJe|g)ixF>|A6@Bq(@v_JpKC&i%f%--I#n!bc@=Y}Y-MYi7LQU(82q z?vjUr-6V_pX!1-31cL2nv);%Or!I)^-qraSZQMIWMR8bw9(kY0=Mp~l)Ba&d)Gtd! zlMm3-sHx6feCNHxzw+AQZ2i3!*y*tElRyYjNZGLrvy)_iHEyrtO#!wHqgF!)^(9G)$`Vigd zR@s+*tFzsUiC?e@nRY2(7}TpymyrU&^yWI~dd!J81--}V<%4#iLF>`k{@@#bZT$7J z_ZHZ9Fx_Q1vJrEAifkO73-X>-OaC-=^UhD4QDR`a+dh}XvHAq=pmXo7SbvI%)F_&2 zQPL_N?~(fD%cKo|JobpU2<*Pu$l)SMzSlTA5BjPHct2RSCC<&X8XAjvUbbQI)3yGz z*XFyZva+Mr>V_7s4`bMu2@~yI7rx+w^Q2zB8}l3GFTmODIWf-wv%4WmPNre;%MxD@ z2zIth4`GmdUV_u$rg)e1TiR- zLcAHCuVC5X$p6_bGS*gYiRIPNxcmU{s(9E-TIgZA&d=GHH?2DxjzszFDq=@#% z&(&h{TMj@A64yfl6?OE*&k@-f?d|QxXUHt4cTM1rgp54m2PW(=kIxgpM4gaK@>Xt^ zzuNSo522R{O5)5OD~U`8>=(S{yLd7e&z1fn*V%%f=DDcPvhQE}GB{%1TfV3W;=A+v zJG4ozra7<8z0z_S$+lk0@uFj#mPtRq!yc%Gvb@AFN}TUS!K!J4RGk>np*-mwesuZmYFaA6paA?!5{A<~dnQN#TpKUtn4_|HV z7i{grw{EO@5<3o-)_m4}h%>wpxID^6jLo7v`1p0M-T8n?UjbxC?#Kb_(%iXRP1!)+ zpbtVb{QuFeFh$OIEVZcW`JG>rOGb98dotQzDaxoMX}Sl)EoUe}6r|^Sahl?Xh$7RQ z?7nKD4x(54Xzj~sbe&)vquv7}N*=n+0#t*7cuVZx@L^r#!I8I_(V~IRY1KyEmlceG z&!+D>qMnVzup5yka!)w-o~-IA`3(KK4q$Q<()4LB4%9&^*ljXa`5x^dadexsq#e$0Id+tH%d{rK7nLA$&&X;=* z1FIF`pLN2kB(PbHdfJnbriA5ulx4_aYZcp{vvQ-f7TnqO;8|HwVeI*L(cdi>w+{aU zgd6yLK-~VMeb&=I;%de}Fs-L8HbwCM>a5VHUW}LakaT=56AcWwxg|20Wbn{c)AOAU zXez-ymyiY=I+<_0wDk@c=bCT5n&QTO&>IH{;Z~gk-+O$p>!Ntf3 zxih+KTix(N>jnTGNNf&wsEVHg9=%9cwM!szWa}+LZ#n7h2&jvJJs3{{lph_@#2sZS zab1-J+OnJB;WzS6t5?KsQ)X*Fb;V$bZ33olUj?Je!V+NWOV&qFA8@fN3xu?>aaBZ2 zT;!Tof!B9v1z;L$F{#9A9f9a2kmK3g>YA4APp>+|^UhX>S{MFqT=$^=c zZ`x(O(w#F+3+PayPfG_u668mmvxgybZg-}LEw*6EBI9g+VvZl18S12O;xbvTmEW6v zG*k&tEho)M8-L8nFB)^j2|w)IBWMX{#a3x99%qs2WcB{8D(|H`sr%tlK*ac=QY^^UExH-9Sfg z?B97_mCn1vHzhz$-lv2Mb_dkW(yqu8g@VJ0xN8yexs?X9AUm=6&WitkfXJhB z9g$MWrk-B9cqP)3NW8@g6tnc&-m0t?o0T~DapUKC4044qrX^#oM~Qo?8Cx;QCNRu& zjiuyBw~@)=Va~0w=$lc+dX~q^HjcU!`2hM3;EQ;*{#&;_9Fbji@-&{(EJwrMkW|Ki zWl2&?RE8{j+5Auxf!(*7%X*U)E=-}qbfPu5!dE=o=W}{5*WD#TIgAgO}9-D@?`f5eJR7=-=JNuixwTuLk)PKAGojY4vsQj^%?Xschsn z)!N04$jBd&fx~R{6qh9DD-{{hTJUe&T<($tD9wx@zn(k2wJ$}5N}jK&Er+}I0MfvV zdcsAoXicBhX%mn|HA80Ien3wzvyh=kXWx2+T0O@-C?)LTX{QGS)Li;g0rKgdwaPgr zN0bg1t^MB1%BT2SOn9ZGL7}p^?NH{8Wz??uHDb;%gs(ey26*f<+^s2c(I?C#t;L9> zo86Pwr4FJnF;-i=0WqK560UGp_Vdg|ahy)g5x~}!-Y~PvUo(Jre)>zKFO5{ED~jY{ zJScL$-pqS*iOR%(u+I6J^>74Ih!31njgaQ7LNOBwjLZjJ%DNj}hw4K zK>U9BE4#p7mp%Qj6ewT5=a=)t#LQg|8#QilhP&8~*|?`ta;_*&XUdL}nFmQ$IQ|zP za|x4_wC(YQ@cD3f45scM5~Wkmu5e^iVn4=N&R})U_D*{FSm{c<#h9kzA`X;`Xe{zB`O{Ln79KmU6IE!)jP*@?V=3u)Yui1F0^p76f(#-WI)-4?(H3fQPTDr=xvG*HrY~fYpirNm1Y*3 zD=A%W@CZAQFqHh=H?iygiwGKW>zD~X;XrbZCpy(Q$kD}9C{;{6_X9f0MGYU!cu@=?6pW-O^!(EW`P&*{0?{M)D^|(i(w! zJl!<0m~GhxAG&dTYXSQE@~6wi^x!fsMY}j7LtUQ;f5N9~g}uj6>)pJxbNHmDUVyRA zUh)(P!dUL5FAn~>WpE_**y_sFCz4f)cXBo1N{Fl7DqtVAooylHh%G^`3D!;YMfwez z;a_;*acKRUuJG0H%E5nkzm_w5a?Y$6954&Yv=hVy^AItZ8y_tbIi;BTK_Cgh)u%FE z`TIsP$I+czef=%{8&wlcUo<9xLHC90k*Tx{Tn7kVQsV7}pR=oib@Z?n;nIITlqgPYw#oOZ%0R6Am)8@5rp+-#kdV#E(R8T)zs#FWG{ z0e2()=?O-tI~t=VwsN_(Gm|I95(_nB_1l9+)~F}#Z;~Bz<7HBJtV}u~t#{kct=7+N zxBo?H=1mp794z16?F2dgwk?uI8`CmCJqUR0Tiv^(COCKuUD(L7E~bM(cCqD85m-h79JFuYg!H_w^7 ztx)|XM*wpa3S-v4Ex24)!ylp`mXwXLx2=H+(O-JwnSOvY681v(2j!XCt ziX+`%N)ikO$2k|Y8N0&#af>qPBy-d$rZeun-uyOIOkb>gBGxsTS!rFSvAe3_%*m7N zJXx<^cV{iI4s=dQ2DA1*eUkqeCuHzYp`EBavsWA`^6DGE8?5sjIp;`#-9#}q081c^ z-(zIY7lv+-GfkU0=L!;jRbd|o$1Wmn8&b%>YzDu9a}^&0X^9s;Ni{%7$G!zq1ww#L zV&2b&Z1RJK=fm=X_It7F`&2ygBA06Px%KNZHT^Oj5jZw^%MmW5bkYpvNfG?$rRz%+kyc_Tl z?fTfwozp`!O>M%dHadcTbGKNev?AL7PV7t3`VGkTQ5Pw?F@PJsX`r-3Qa_`TWp2P< zu(NTi&i8t!o9I0b!=F|KP<8C@E@3VS0&~9iJ{N=n%ZrBRdU}Ry&TF$Rr4pHr!z|B_ zYWs(NgO8w}LA70lIEmV;y3U=lU?*UZ`);H%VjxLcj}xC3ToT`B10Ulw5p7fGm} z4K87TNpZJ+1iA%{m~9=r1G;dUK9ZUB`p^%0F!?D3MO%_JPOYmFJtA_>_Kn@Uim_$8 z_((83YliE>87{zEdS*dSG_zegNSu{=Wcu10A38MW)&mV+nJ@DcF`}~hdX3(JzQ#2E zRu27~s%+3u3FCYLTk7L+O4YpzL?Nu*^0G)D3Gu$$E4i}3x%Cpxv~2$(VcO8QWR$5g zM|<6`I0$4oi?|WyWG)I;mmtX7t+Rzs%#xSxsfwjz?@P}h zu1j;v{YhKN_O8iI5itL#gZWh^>$}4*jkxAE|Nf=ce%qb!&S7<(<^0Zs)s257{!IwX zjbNO;?t=tn2YcCRHZrHMiRoE+p*@+$4UpZCQmng1cbaI0PeIEOfl25(?>k0>lH?Qu zh!LnGGb6!c@gUN@q}?F#^L?#UVLu=MpV%q$WWJZoD0==Nne^Ah&ySJ5m*XOGb3@^x zXTd-S=lI0J^os*vmeM4+WZ{6bM>XxYf%f*Rv_#gyI=P&4OwTXl;T(f10v>qlp-!3U z=$D9HSJ8Z|1!vR&tO?GAXbLVfV4HZ>*RO(TZ9bXzwy}xBKG1As4-44jBW1)ShjtdI zoXQEgMQTk+>=Sta$!08EX5N}aqFLikdq2SxqE@rsfrv6o7z1 z`V~;YBtePY?H{vN{^j?sp>x)VMxEW&Q%CoIbW$heU-OBA9(42s?E8&wqxF-)D;CP zi=*@5?=4nie(B#B-Sn;DT7}ob1JjY$%Y=T%ZOrgfGb}z?x!?4H(Yi8K2-^=IWB7nz zwa?tqi8bfIDmFzmS8CkSGoeXM=ftEdwe}e*o$Bf4B>Cg)B)(8`={3pCIkfD;vzwX? z#1LN#%$@FvsntngfWr_MsJwq#8X?AgVl27Wix!`YZcOdvJ94tftxzN-X6G_{oGSwS za_a-dK$OF*u9jaxSTJd}O}mc9&n(I?UoX1WdsaZ~-wwfZtyLeH(V2h07pp|*zkD^MN#|$610rlVLpY9!I z&(^*&bp9Z3K>VKfKv|_KWO_NHPR!szHDo)B^Np&_Nq~SsN~=?MmtzM$H9H$d18I!WZG%Wju{w3#-;Yk+DL45FS z$^-Sfk1U$T?53Oo#jm&a1s5~ioHUJYkhQxN{C!eHiR(`i3eA|oC8gK2cHlHbhY&Z|D5 zuC6>W^wx+gw`64&XLbh+>~n>R#sg)X9-?51Y@^uGfHtY@_GLAkpmQdH)Bm!7XmTeh z`FyiO)>W7G=3U!S+t$QC+o@v(`&1LU{(%7)R`}%zSDO0*<;?0uE9PcpH?;ZeP9U{P zVAXHtiL%2^X=xGWn#Wx@hQ*`;k$H0anDnixZ=%3YWQ3x~fE(@^mEuUZ>nbp%l|!A^ zB7;er`qMayA>geS1|D1e7pH6X(p&yayqWry0)sX+Z)BM|qsV@{n;olvE~{A7p>yVX z9lF@NQUMZ_Y>6^l1y&TAv)LaA()Jf-vIo3;0}Ik#HS@PY1qC&a6+<$qj;~fU3N}9Ak|PmU(|oYVUWEs>h^VVKvB2z@WS(Q}_* zKakqLDL%$>>i-$1;BW{hGXIg()Lykbc{x7SDCpWT_}693Sph~D)pOqq zembuN(xMD6FL*F-Oya%UF157Kon5-gX5;!+ADMG5>ITeyNeh)sdWqs8cT%|2Pp>C! zI5qp5b1wfr;Qn%(LZk?+A*i_B)wK6+n9uFka~ZeSbWLfZcelv`US~H+dNZNf51H<` zWdH>I_@q0hnAE3?T61(x-dWq%-zRJ{}(IMRsQJn{m6W?K_vQ=n%stpq;RmZ&G|8%Ft;5tnR~z}y9~GS9hQbKMydvSbC<=L$P}J;o5XqNEa`n<^&h$=a_xoLb5$ z6u4x5;Mb2`R}xr{eNSjxHWy#*0hz{}KnldygWCZ6KKcJP$# zxsU8^W+wq6Tiw4-NrQJ!&A2Pn{}J9X`c=AV6)@-e8bWs%Z0FZgwgS=zk6?;C17=yA z$X{kfkhYnXT#B3etue2K3`6hQT7QRiUJuE1^Lj$PLkOUc+rjfS$Hs5h@mg0B=GCkG zqmiW2jZbD6tVmRxoE6cMoJKlW(zLTHtkOWL!S_g<)x1_Zz`6ur@c7plab0i#SZR_o zFccDiuBokwk|1*TDm~<)LhH8&Q9g=^C9HKLgGSIVl+a>rR2lb%COXg1}SCDWtqrA?ZvUIG2()pPq-( zWcJrGCwk4!3^tbd`ENv=4^_r#HkOh>>SLdz(9G{?PUyL0DO`)+l|97 zA(dYRj{;VI^S+FkWL8+?73gXiLyNZ&W{G<*zPt=7r5xqjnC}oy5n#}cI3m{DsH=NFN;{ZzmVgp>m#>Rv>OXgyUDN~6*F7kR5=MGX%QM$O!|*-qVyBd z7f=JlP+ykDtwOBN5{pR#gG^2qZFZ(kY&6OHi-&vNiKp;g5BP#aO$JPpS=LRZF-loM zgSee5T(F7cp#Uk4cdFf(^;fkyfWj* zet{FKQDP@x;)rre%?`Y|)dhC1WG9)?WM2GdX&!IA*Df4S;`*Oeg_BGMAVQ{OWXsNlM~_^ z+Vz#cje78iy?6E?m809qJN?vtLGc|;+s*bm z!{$LZq;FC?f1)Q7^Li}vteLahD7`=pSkvy zTWLT^SZkxYS59N1uJQO3f}223u&cADCs~2NhN<3^{G(I7BL^^xYIMXTg0q3{Xx`Ti zirVjW6KTG`pD&@AMD-a^kr_Ew@(W$5O#2*nG?CrZ{jR)cJdaD)m`>oIa z0jEG%zinFklYc9Xj_b^kF0U=s(rb2kV^s%ijk#Q-Fa=+sBjzAv72s7!wCvoA^CI7r z-7VCT258EF+~)*0w;nsU^0NQ zIcxZ>EhA(}5q31!9&1UwD>|;=p#B7M+lBxj)0%{whiCN#+TGV3Xm5h_U>qg1no9Q`mlFpGluHld;^8X(kLcb;5I z>C?!%(%#)%@3)J{TNHRtkseA$jzeuH7zd()thgU8+~HG9)ML54iLLHKUK~tND?E}e z+%DL_3x@LJZpC&Fy}NZ$i7qu8h!umuJKQXRV?Jq&sbaDonfuDPVjpsxh9u_ExRUi+ z;jOQJ(I{s-9!6(X1%6%UkWrbHR1TTI+k?*VSllj29FBMhgFFqZ5-s(*T(znDTz3&je70Fa zZ$*9J?IBM*^c;bnxqBeA(_n(-t)jI>M73y2VrYW^UAK@hSh3`QN%^ulsFhRAYjrYN zTJ9=$&RNn=EHrq@`^&cmFdJ`S&T8dtx_7?1ZGY=WrSE&-z2&~Kd-Pu?ub*8W>eej5 zrLFCRHfPP6+S)k7SQ0HiY-X;gw-$@1%_kLUB!q~mZI%5xU#71_X3~s#Y3P-k*QL95SLxB}t#qQALh#(P zTmz>(j?T*yC}r3Q2LadpTPoy`z%Cm&8B5i-zPqqnr-}52a26=#FS(Bq?j?xW#Ys>{ zJvd>Sg8t0u!K8Rn)umw5N|LZm!A9a^4fl4lsW@TKe-Ya}=swSJZ)q`|>>4Pc+`yGt z7i(pQAHr~;bbM_r*s7GRJ14)FUiQA8Pk*~jt5TC((R=x?p0@J0PbH?4TtH@)c`Rm; zCi0#LC5e!-DJ)e$P#X$3+T(^}oDdqQTs^1TW@u)fM`_f4?njf$01&=V=Wuc|yb*=; z7+pIDiu%RwW4`mGc*`JAepd4s;~=W97|#p9Q}X9+P?kTnTU*Vk-ArSYi2S(j6v>l? zM%;FXOp?HRf=SLxjaN>ls?A+lX`)VDR+rY=ci9=yk2EDVwdv`iS5J5MZ`s+h-%Gi? zx`GRRM$yb`5*aQ$xgwFXU#ELY_||MP-KO*%U;QuasT3m10hIsSMy?4B&bJm%}SwNM8EtDDFv)6X!@3{>$a5 zDuIdn!wuLE2nkoSc9QE(xo*w~`}E3d;l98-$4sx``S0AGlQQ zUJ9uo6M+8!dw`}U`5V_&>CxKD)!Nqi^h1oJP4dNB_w=`Qz1Lp8d&L#?lxuG>Z)3M> zSfQ0t6;_E+NK%1gIWZC>*|c&ACnQzrcU@eDdwaj#Z;l}>JfT!z8_Wz)I5;G?eDXRp zEht;k&`QdM6_!M8#O+esWI(KpNh(VKcpzltm0Ov<#KK_KS)JM9l4nU7CHWMXh5#t7 zwC9oXwsrtn*!INA1pExUB-t5wMw({5#+}*@2jme(bp;hu>*&4EJ2xB`+jBGm(Y!EpBk%|1f zg?Js}D)FO5ChMsDj4Kujtjx@yoCD}bP0TRscx=ASeH2!up9-{W17g(?Cm z-Hvmgx(Rh$qa`Q1?`<|s+V)+Ym&>c^oS`jgt+(rK^z4?Mbk^Hli4+!*S-L=|j)<`{ zmocD@3C2Oj&<#?G-g6S0tN3SP>{Lt|Pce$3f*nrgQm%4&^uSVSF4_jP zhT)@?ka#8A1S{@8-s%nsV@+w?HNCX(1=}H!r*!|GTIQy^ZrgXZ>y8R{sElE!F+i<+_`g3QKWr`yqZ9 z??qk3O9fEQNZ<^sWd8sSPZyCb#Fo(@-uYi87?{HB^DwyOobWS}+~<&XiLKeKLM6;L z32_-vj+aJ38xtTZ;EW~v=8h=DHFhTS+oANgGV;$A=LRBLfN-XWk&= zf(hGRU(?w~g$nsJg8^KITp-%LwndFK$0}hfXC(C6$c63vPz#tuiIrf&*!*6Vb{D|%x zJZ4mK%Oit-bWzzS5@Eye0G`;2iTU+$kUt8R(r7RO&ys^n~6DX2l1a3nPe&{#@4YikO z0|kC;fH09tmTMi}((Q1jOSTs1PngA*XyB zomGefYmN$Ki6kB~&mal{h_x7&$_rVYdrx>lR&Zld0EH;MRMC&YplqLq_i~u(6VsHpxMhN2xa%Zs9E?yR$jP~(Lq9_TF zND1UNHi9#ejl}I4Vou>bip;0&tfifp3CysK%%duPRFSu)#Z|%o0G?Uow$n8qJ+AEU zv)65PTWq)9{;t`|QE_UO?6r5(&fEDltGd}Cml{>Ih5Uv@FB(U-Kb7Mu#X~;T;Iikb z&p=n@;aZKteU8vY6~aunOsa_#XdYg74V!@~LWs^EBRq|u3Urbsu9YK85AGH?y4rEE|^UIL=K*;^s)>Ye^)ANSW;-mI&1x zkV1?C`8>JGXMppy~_B4Ys1sj!t8&GxVO>Cmny{lZ; z`FHOh>H4cpM=39NSEt{*093{9H4^J@Zwyjk#Uw&d!R3td2?UaI z*xZ>2S#U`iRk#;PJeEFtGNwV1m8V%BYm^xvp4bjgK;$U{<<1h6YB;#v#_rEuI_=)p z{Ce4M2|3A6o0;40+kag*zS~;Fvb<2VmeY-nB_d&QDUW9FFK9*D5RyiAxarqu0;&sm zps|`pwG+oASz|P+(Zn(Pr;r`3yC?xGw;P6jVZ@SNq1STPvfITh;RD)9x+fq7R$+h$ z`TqAp6b?oML8o6qXQZHecnV$2Vt+DLcJkS_l-dU znyrlI%Ln_#d!?qf*LBhKzP5XOe6_xzSNP1SCZmSnP1BQOu5Pt*LLr^4mJ&; zs>CyBqs)@`%R@7a2_#t_NR-BWxd8)(D}j((=2BNBvtC4&Yio$@Ev1wU>>}fI!Dk6clNvO)Gic+I`w? z+IH-%qPfW$^?9C;MXygiSLuH%=sp`*WW0>W6ssJiCGw+UgMSOWWP!oVfkr_KmBO(C zv`b@WyX~(LiK1aFQ(S^(VYo5bfPPnEFh=GikT|KNw2kCzxfjR`jrM3InP!g+0L!pe z&QTa+c6uo)3gzM{;jxoYlW3MpeX4^j!c6sYt@8{te>*S$Fa%`e&03bcW3M;g>DKyh z*G1OM))AaroV1Iz{dH^Rv{!p--nU}d$#W!%w&PEc7kMouzWC*09}F1=a!b5?mdM|N z8MZ_sI|X&Pc%u#IPWo9{!@OT`!wxLt3;l8;nOBn8}R7y+{ND zfTwRH3=9+2yIfBCf7x{KqR!!CiEhLnHqsAfWgA}^$vEBeoxj5#(LZ%*EBE>T0K;hS zt(LlTs(#hp9n)KA{Z{=Jk4LWMIKIhwaV7d0KFy}US`RFVa@znsHjn~@;E>z$pqw1s zFvESR-P=!UZjG#BG>RuIyC^>w*T+!vJxdCNfl$T29p?%LMkaT3y@x_3HH9 zn#P= zcrINcM`n`iGM%I_-0QgS1xD2%?EsQ+0cL{M!b3C>EE7d6QG}R0rQ`r9UJ^V>f?YrZ zyY~z>rM1#V@#z!E8nlyq(5B%VWkVm{AOtAM?9I7J$=!;iok+>I&3kL7&-#9yfzzKy z%?ph17M8JPDn=Wt$$rozm~0OuKhKK zDMoFlrpne&XK#CLwYJ}OiplmkPuuR1E%snvO4)MZG6P6L<+flD1LI~n$il2r*=tkI z*B0?QnEuYbSPvY=HDZNa5P*hr!Q(v(90+pL$sM4ySz1W4o>)kdDGG&Fm;Kf_DmpTN zK-vv9O-5wX{?IPU+(_}s_Iqg*v5lL{V!0hb8v*D@VhP=}rsXMew$_%jcJ%1&=lSTH zB~RGX_fn4AB>H#rT|VpT#W!;5H%|bNtdl}_n3Ym8?hs(M{INoIjFHO`lhK(AU);fP zv77g}wnbKEOav07cww`SyT*HD5~S`lliN7Ap5E`txAO>z)gxzVJfnax?0$WrPC5eJ zLF;+qx3<%!lHYU^T|!Vv9AP$|Tgu)^A%?-XoG2gwI)Xg0kGkz^CEH!~UR@WLWQWSG z?H^q=-7olOud7U|vP|}tGD`x(6OggU>`F=JDi8|fK0b4f#KuqivNh6NAF@rCYq_y) zo*QE}12ZlGOk=L)Jb|6NO%0MnCi1T!xYJeDuC65*c^@trTx};CK^Y{d+qh@=uSar% z3wRdQnIeZM&_Yj^uFbiLBn8Vh;&$KwRQ14fw=-`=dwjLl+pE7nEp@rbpxyMBuivh( z@bc55Trf-M;!obl%qD`^?S+ww10gI^@8A=jNj#8oijL*&q_kCv;HA8A`EXpT7n0ne zW+3kSz=uqs>%3zfr?4@!vfMR;$kG9G3o4Ht%C?E9AS;E7@zN@4EZ6 zSKmRGFDzJ(?Q=06+gOC6%S59+^QtP&weoKAtuV;H)m{L)FxAW7ryCzDIjE?bCAfR7&#RYkXXXePVIKK z#o@}3p$!TT5xHzuxqFb-wId ztDB+mhX9h_D`Poa1|*h}X)UgFQ{_V-vXz|09jY-@4B05e z^MTIpHl2&WWueSarNgzO$|FRVYq5eD%wx;XnCtTv$OqIFBf}4hBF{jF#>DPa{?|qj?y^}MjrmxMVw|%^~y7NnImZT1% zYiLAb+stJO+bvZY*f9zKRXbRr+6NqfNJF$#w;EmC*4MI0sKD}W*)Q6yK4x66&gFP4 z8@T{&88{3o2*B6tdl1+CHnjdPPx>o@)x-pD^Nh3W)WNMaHsdWjwwka%b zeru^&6-Lr_vJiK$7~Qn+uBSMmRZ(qzHu-gHZIk~1hX&nR3RKmTR$c9Tzw29DN*Bdq#CQJh33xk3(Mt*4tiYtlQRC}E{WyzR;R3k_uU?gA(B(CF) z!z1J>P!l4+ZglZ#-)U(ihBTUaqYuFu05Czn_hfw7?4y#zhH17<_NAKI#M|Z&Swy%o zs7C1Qpf}9vhEcTSaskabP032}yV+~eTU7M(OTPV45mAljs$CL#U3I!oPc*I3`BE0l zmj3{_7E2@SLT%VIQk;2(nB)RW4t`^S-;Q4Fmp97+o#M4=Vn+t)E*CpkvOesnKPzAr z0G9_C0fd85La|%3+ly&kNSfd%5teAq;BW&I<+j$w2N)n^)SKqCWHUI9&4g;j&?t1; zM4OITLV!RAw%`ZKwGri-QnF2Ty}JDO*L(Nu&6K(0&eFejo4=>!x1P*}h3USySX2EN zvWX#Rfs)@Nb8SKh+)hT&3uFQ_l3Mkhs_U0i-#Y;+?1@5#1`M-)Kk zvV7Z#JAnJH1wyG#P-^eC>h?)&-%i)vZ@G#|%i3Po*S5;n^tJbT`Dj$SwVK4;!75xr zN%IwvGFmdbBt?kj4-9z&usFZ~U$nC}LSYS%xkb2lEe)K@v5`gqUAY9X+BxeP$@zc~ z++D_6+INJTDD5A~L-#_c#!8RlP{nqJ==lsVGR+>{-CpAH+sYR23Ng7cqRw)8Wh%kD z9OU&GKQ&c4>e@?Wm#PPbum_DHPm z1aM6YG61#_Tish**hvNKHtgy^XIvvL?ZktVv@t7(Y=9S@ zP6~78YEZgMcYnV6YqgSB>#DK_>N&}!Z8c4PefGVt(Ob0nt zRBo6DJixZMQCi!UXhaRTq70PS0R`2h7zj6vHU~E5 zaFKc!vL#%+s@(-E~ICW z5hW4FE}+KnYEd27cPKks?BQ?#U0dhNa1!zAhvbiZpssY&+;l0k`8mzEPm}MFKHzXYd_2$CYPf}w|(yNm`|3FH6{7~!*4 z-u>@1)=8Qn2nq6eo?9?hQRXq^Dlb48j&NH!z$q-kIg;Au+S+t^GOZi1SXiOuR23pM z3h$hZ?FRrd;wiMan%&>Xf-`eAAu~eku&(|4SPn-rMmCJMIr{Aco9CL-ri;txqf6wK zwJFq#j9QKF=citZ$=^=)U0v)<65Ar)Yn|^46A|;7{EgBm7~#He-OJQq0>t1q5WvvA zt;Ex;kjf)5D-rX=cE-C2&Ug8pT;P>%3Z39+#wL4Idz(mF;^?${@w^DZQlNqIFbw4M z;1GG*1yhR7-s<*f4ZLrD#G=O7!jcmZL|p(|*Yuw;-ShyBSJw2IfZ_K__Zt5J+h{eANS5K$gpE zeL5Je+1^DGGZ>?0DH><06NMO6B}#V5Ac}j4Z>;e$$eE525qFH2XxYDqkO)oNwt%IH zz#)xIBgy;7Et+#|iqoBhVVIriweSkQLY=tJ?@8}c=)QhMa9} zh{8DFk(`%KNk9+G0)Rf}7|AAkN4;#=6nQQK3zkTf84z%iWHG_ocmNVqsK)?<92QZm zsIfE=-o=kC8P(3fbQNtIkM$@#!IJ4nV;Eu4~DZC5e1%SU+E(qP;9mp)n;e$^u2 zuOW+oH%3Ps5VK_!L)o)rN*zUd?70ILN7YDY9tJ&Z9Tz>>K$d8M}`XvKE$Sb#=Z zfNYJ;ob#7UXAIXcO$tYQB+MX{1F=Au-6Os^Ly~sn<0An`$u<7~(=Mj6N$;-bR&d58 zP2v~I1>839pcUnT2Wk7?aOW7Q(^0+G>el*czKd_nr!HGLbk*tKYb#r)RlCzt#CJ1> zxSA*}^vii0%KrdHfT!-*U=7ZgBOw4FDEYZyd8;#OV%9-H_UN^#(BqZ%Mb4irZ^PZqhLWXMs1I-Qsq7!cs1xuqTU?v{$)O>fouqLd>&*1GoI_xayV*L{T2bf>b7WV8(T zFNV4;B08`;m6$MGu*3J{9081zR?|_Pq((b!jjSN-iV$X0N3E*&Kw9`Yk&7;%VfXE2%(}BugIleZkd{ zg1NvM$OE67uG6EVIm>(9d0T%y@6zhmR@Y-FB{k1AZz%7zo~^rE<c!Tf(Q)ppntkp@^+fF9CO@F9CqSSZ8Mo-LZKc>RQZb80Ph44NK(Do zO8}%nd%j14?jv<{B=fFj3n&q?q5G#`f0Ps$2f0#Po*K~q0BS&CwbbVc31nxxyd|AZ zGP^@NI%)oMUj+^n3Q(MLUi#_ec5nHNTVG&7yOCqKeo_eq)ECkl1#=# z^4BG}Lzdbb2RLO_$vlx5iCI2ZrkbbOyXdyRo%FNb#Y*v9>q$FpXSS`{?%l7~uC^x9 z?vnoiRWn#jmdM1cC`+;`)-y?e6eXHCHWZhPpxb~l zzbQN%V~pd8H5&+}xS5M;3i2pRn66$yyW}sIR3!BT{I1>J&!wH$ zL`SwwQVWKAd#D~lsBQucjgT8+hc3jh2acKQrGp9C8~Yv2=b3RTIEq)U?>1*9Q>f?3P`6_&ZX}6(|4kJH?M6pOILj^&RN=XQBTp|T^9De z*F^8VoAMJ@xpsRvFC~}ktqZWYl|Ty3pDj_gk({n_4%~(81|`0Cw1!h_Zv^Fb$#`Rt zB}g9wErEl{jFl%Go;U`d1T#v>440RPU`PGfkRz;NF@=u=GyJN{pW!S<2pE%M%^Y&c z3{ZIve4r!Vk`_SVNTlIU-8udob@_`dHmNA6D?ZNK?|(jO@(n-jNNwP4HRQLJ0Mo3p{i}1xs)d7Y z#?{DV?dW>bgGu(QD_eV(x_P6GvK6CL0d@e5w<~~glA*Jmzc9l^J@KA7u4CV(NYR*# zPX+@yEr44AQ0^dyjx=_Vy7Jj{nlpNcHdCaOn-MKp36nkwXL?>ThpS_8RW6I zl6Ycr^DVm@DKZSEr%3n%Wz?c-X#+Icnppx?JJ4NK|V2lUM*)A}I2||zu8wkl!hrNy5&Mc+4M~s0XGTC|Q z0~8=4w%ih>fyfF8cK#qYXU$G9_LFw|_Vjl7^?K+!aJx#Iw@2k}-*v9GPsomW1olfU z-J+wGbY>HTXycTjbSmXeSOPw500<=asMhNAUff51u}5;@klRRzra;+d+T@HW^7@g< zVZb=Vx`2f(qf;%-t;#b^<)f^7M$$rv*MdU^P)H*g```j8`M+h%5L({<0B9CIeY9;L zX$f%|0ZC$jDI;({dn0hhD|jT8r5(Dy_wL$j<*IG!w{ZzY+uh%7FKzu^uUqTA_ePbp zt@a2*q^j$gK52IZVsR5lFbO+|QH+zcoaJ)WH71q~LvqIyXaPnNN=if-)d@nxZX~jj zz%ByyIUq7`H6D9O5ZeZqX+arL8;!260KgmvZKR$9H#<>k?XAqwM(Fyi*HTS%5uu6E ziG0Ra47n$1QIH2rZc;Otm$?h|chc`(x^>rEbibDPRHJ%&Sz7DYO>}o%FMGb1S4f6m zwM$`eZ=fXzEt%HCO3R2kuREF^Me*Z^1BMhQ+%KnyZl2dSsr&2M+7#u3CKbl)Ylw92HuRd;Mf zV(Lq6_uqz`@^xo1EHzyk32Lx+K?MRQ!ob3Rjf9-P-h8JEqguTdnPG#TB@M%Enn< zHnp`Fjww~bu*_;qY)?64$iY7*7mz?9m)G*$kXay?&u=zUBx$gZnNXD4S#1(}A-7AG0`(OhSmV1W zrDcWA7|Zu?4oMpbP`(gue`mEm<4aLXuYA#%7o2PbgDC?J9tjA20o zcI@K`!P)M&+pAmoY5FZ0m$h{#ZMVOBy?X82OSfxz4P8Mmw4b$K+(jJC=IvIO%nV%Q zZVQ$OyO(Q>k+-qgm&Nv5JBcQa+8E4M>U^-l9SL3EGGLLzl5);-hW_fBOtQf+kL+93 zR#7m@v1Z)NU1Z2YHlb3)gYxijv|Ph+CA5*<$rcPNG!n><6uZKKyp~{doM%1C91K?U z-z2@~qVw+hS!>ze>wfz0eqB22+pV3tTfVPV-EVeXMIl%g<}0aZnOfwpk>#s7UBQ7k z!ClI5IuZtU{M4bo$A4+I$~`_wE>y(#Gecn*BW45=_X=6o3+;n<`4F1pMKHg&8Cg1^^H9C$tetVzY+rqbQ2N z#=~r&xdo1U?qb2PT<*XIt66=ieEXpYfwnv^5DHZ93g0pjg~gPaJX0ZG0|4@-60yV81XpDzB0TNX8F*GcEHX_ zHE6ob_up@lQ)KL=QsgLNM)v^Vu*k`90Kmut?_k#RMa}cqb?LRQbMi!sp5t zb1O$1i*vegfFK|RB!8PQ*H7}-Tj;mBL|w~Z?;AX}g;6AR2xNf@vBQPIR#Lnu=N(t2Xu^pY)2BA_ zrO<$x5?OZ4+thMdkhXA60Kq(r@IzO%c>JiA<)v7n4;7S-K=Voo0WMh^3{W-{=XV1u zQcGxT3ruJ7EY@h+SVgo`uw>lZn2+JZ1G#=s8;tZcs>V$zXsxH>o%gffe@owY7`eg5 zKI?t@zRKF^`s}RO%|&hg)@w4#k-*cAZM`~mz+Fa3v_F{8N`bt zyGb)X!q^{oc?cWtC|{c#-=+4l z?)vv#?bM3q+2#`J7N#b%B?HW{e9GBHb&dJQ%tpl-?hi&3{p@3oU$k4v^Ru(vuqEQ3`;z!a@gI7!v$hS6|y&HHOQLm{{Z0WuXWdLo0ggvB`<~%Np-fGdEZS| zoBj7!z8_h-o>&A{mZ=<3$}YC5s}P(ct7jzTLV|e4c>q+B-bF3ggHyE=TUrCvP!a#SP`K z5G}+r#OfX7jqx@BD&*j7CuznBJp8%IH023HSu~ZGO;fYG^;Yyhs+Mv@3KaF zsW&6B05DwiBaQj_bCN=WF+?NDv(fr1>b~u?x8q|Q+BQ#TuU|CQzFjYOtJqyZW4SKY z%(s%mJ3Nxgyf)C}fq+ywE4Vp5Sm0o`BDK4{)Gi)r;)V#_o*7K5ka9t7uE2U3A{o@AwQik=-@T ziy2~)bUTXRNQ49{wfPFAkw(#zkE z2KD3*?+{4ERG}_Z(z4s;ulgpxBBeDO$D6kO`roda+kG1HX)V$?l1OJ`HNVw*Z*FuRyB1aK<6_m(!X&0%vU>*+~~K$&3*XL3U1k+p{0@<%O^8%nlu zj8v&|r7d5b+SOah*hHR_CC#RxB(fP}+WL*N8ExE+keAx5 zp+<1qusP2SlZ%dy-7pPVr`7#S^jEvj1L?r4lt zDzwl@tl>7{!2@HE1{Ct31;_&%Yg!xWtzIaS;jtqu(iKA{)>b%D5FQGUNFRm@yDb@h>i<1mqCV3=2lOT6$`lX zK2erog$yvsPuj0qTfnpC-6Kb`ZH8P)Ai;8X05&OX@$%$gXE3eJ(rFjIMZ?2$A(jP( z5}VoEaR)tFm=z~$XDp`xaALXs&w?MZTwFsxkugs)6dU4l22?0_8;6z-2?qdxL0r^A zk1B1ezRLS2Zoege%GV~6jAdwQ+^@6l``gj={VXJRH&D%XVTD@SeWu!0HxD3Xz+K^v z4oFf!1m~Q5*^MBM_VVW1;hAQ%R+39;A=nv0xgpt4EtX?|A3X_df*vKFE67!J$YNP{ zD@0>y31uUdVTC|&te^!QG6*7{EPrm)99=u4X*;{iZ$3y0r9n7lZh$c0uqXioilp0= zoSm0_bhVdv-&=WEw%x5tUd>4>uC1??y7qRn(!wW`4a653leM}OKPXccS+=Wz=7R1x zCvaBCVaOzJtsD~)&eu(B=<4%Gtg1t8P=$df_(~8?;u&%?m8VH_#I*vBN(X0q}yFR?%L~h=dV?&zhWCpn_Jmh86{hfk-%3AlMr>0wzhIQ zGjcc?Ami^?L3wX+7_!>0x+4Uu13I$&%6C5uW65F)j{8sWk)rd(8?EcSaHP!6E=qwj z0iHoDc4Ux@R16he+kmXe) zX541&E+=F%uiB)R7*<&T-?*v65(&Xw+%YYKjAEWqOhU%l-Wkx85jDt1Rzdgj#{{-l zBRKg;T!qMETUe__1Xgj+BvHp0kv6o9(Y6&x=K%I38~{{t#bWH!gx#F4ZI_?Q?P+e_ zS20dr(I?q-*Y43;_gZwb>^AVUCDwaen|62>WSSY-1acj~AjsTL1F3Ab6pxrd26%=_QSiBdeT7z%bM+;TF+ z{H^mIRdZV2-aAR5xm#y4oQX{200QmFC^5-ggUJD(mgconb7?7jmGxHFSEkE-G+t(^ zblfd1SCUq>*Il&l>bs?}tiyA6CZ%T%pLUkJ_N11~4hdFe<}h0X?ayU9vAFS7BD>F|$ z3)J-Fo0k6oY#OW04eh6&ih?GH@>|!veTvYt8R&HYvmhvw--Ke@`94epR*ZSyvlEu&XwZiJAqslzBGL;aA83Rr0}$vtf})+@k>F z=EntqM?cH}W^3}+ykuLHw-sEd2I zM$}Oeg}8MPyt{zN*mJyv0|Px*8Bi+h7qL9I7dK9@wb)r5Wx8B#z$`*^EJ#tb249zH zILIt(m=a65?gUUzDJ^i$2qT4MZK_yj2LKn_*QNr}ij=>Iy&BffM*6>Z$z9*Jx5cET z%_pyCqw2TQMK2}Y?{X{Kd`g$H$276R&ay=dDuq$^P;s8SfNX=6BWNnLnv{@QUA*sY zd2IwC{IIYqOCH^!z}ua<4Z#F({E>`P{KGxWS2uzugz%fCW=7hsLNVGhSUZ98o!bjJRRQ4Uo7}Yfp8g(t33A+q+8Mch^hbYmK9$ zd#h-?@3y`7)8^4V+)s4T39j!ajv_$&Jby0L=W%5!z=4s%j9{qT00vuGqKrW;*bu3c z&D|RV5RI~kNeH>eBcUT|jm1e6kcZISNWNzO5uSSSieexn&|Tf0f>@2^eMwf6iwEAFi-GM1BCYo}h9 zZ%rPS^L|^>x4f{1ONo|yQ+8fm#bnx~ta;q|2Xp+)Mig>e83*ROZ4Ldcyrx28n%$1s zB~!4fki3FM5pevC$_p`Fqa-9K{{Y&z0^wZABx*!6x@LIy1!BcR5%T0Q`L?O*zjQ$x z#+xi;yH_&IA`&cXD!AG~EF6G%JBb-mToSzG=~kPms$H+Hi>=+3%Kf#qwb&b}CCk07 zy3wwhulPD?qfYjqcS%fw^5z&7kb@~z+Z(sbyC-V0vbWtBVnD_bv13t&)B7scDMhS^ zWet~g$w0(2o=C!O{`LVRkO;tv))}qmvx@RHxsGTQHKIghXxCv-`5!YzNB|B%-lr!7 z8gk2QZo*x|b3C?%OmbQIfz+NrXOQP1M*I>vRs;eIQdGZqxgFBh>fh#{`ERh{7NvN- zbiK6ECu?+j>w9ROmW69&xU{!OqmF;G%b42am1SvFRZ5MZu*<0&@Y!tlW48^(wa(kP znmLu@kjAhejf*f*CO4cA$9FkAs~qu)%DQ`tn=^EXw$j|g9J>_@32?^%?!eCGQ?zx$ z2n3r|gv%I`CSqjo!o+XRjW`@k1Q_VdwG5jzm=K1qwO4d-J4!k`7QT) zJ89`;F0&~st{J*k>p?R{-(W`xfK5+v@= zI3iL}aLt0)$!7b<!P|0@k2||^_5Rt1A6Ks6sm0||r$fr^dky4f2?3Ycis{B@(_S(iu`bs>m-EQpc zuG4-?=WTs%M!#U4CbqM=SyeWf9%d1)*^Wj=PSObko|zvn0;(8qEp4Ead|~5_kVzq8 z?J@y@y|PG9e6gG#l;vA(M}KE)HJU->+se!tT6cPm zl3WDmXf2*eIUg$dnQj{a%LD3`cP1Y+po&EbVNzqaExeYHZY=CpOb}1{&Da5jShfPx z-`c!>bolcX;Y@k?SioKe?0w+HR30Aylk(e(+3WLe`mXx)-F3aR3D;F4C`Uw=zbzH? z{Z~x}MzQJUS!K9;cr zQmcOekHw%6|KdcV)}IG&=u2>g0*H$0**8}GD2 z9R}QcsQK|@ zMCr}6ztZ;@6=Zc9bxIKCKiIeJzrMa%*s#EXKh~kT&beX7Mra2p^_B7!`_O5PqPLOX zy|}LzKW9`7UnUVzC{WI?bF&+XCv zZNJ93fK-o*el9`ImFrBvK?WxT!!^8eEop->l^&7*{rfa?$v_|ip0Fg`+_uofR)lJn&}nS8cSSlI5{mbpuFUx(e(vxd$|4^7mct@3d9 zjw4)zIcy_b3AsPXQ5!c+E|1`$a*N2|1&l*ry?ZCy52=8>mHE2%B?GdvZ5@r}x);x^ z#_izCSG~O?Mgx~_{~0LMwZ>g~(Bippf2&6WCu6d_$q{$pTg4d2Ox z*%IqF!xy=xNh*p!_lI1upXXD_k55U70l+yV+v^Q<#lTdH?a7w@@ubmSnpnqxHpX1{ zt=&<1L!D^6mJ;4NGC94bf=S|&myXS8IZndgs99_T#ANKVNa zRldWF^WGT~K_c7sEXI53TE}it0p&69U*WLwz5$7;yQ?N+LttSD)CYwNhI$;K-XWJy zKwEA>J?_ue#NN-Mxm(2sNf;juT(_x_-tP_Lrp5&Q|A&?d-2OS;7JGLlR`Rvz=_gm6 zT^gg6UgS26m;5*;U){CzHoWGi{>?GQjenHL*Tm z)evXHqugu?ACbY`74eDl?*6ojg_9pA7*r95In>iscLrDBCDHs(@(QYT5}`n2AcWIX zww5!iKca{5m~MjUbjWehgshV;&TV^>03ErjeDK*6q({zJp`JC^ku87(AH$xRx#&Af z!Jp9`BhW5i4Te*HXezhIv6YS6PnRliKg!{wdrjiu#Ast_Kk`e1AXooOMEyG@Bv18< z{n>(kC|7&RWYzXR{KE3W{{YX%7?La zG>FO!Q=P){LbDuO*h!6PL&%DL{&zv0#e>A9r(#X`aGU4P1unE2&*iX)!=^A2UAbtN zY+mP}+&ZU~g5-nu^#r`vNOdn)6{K0YH1 zx>Rl@{yzC(LsK)>d*ZIdPc%d9sBtG8Ln&{{d{brRZ z_)AI@O+O8*y(+GFJ;q?h7Ce|Zs8`)wUK411U%_v<{Y*K#C;iFgs`{P~i6e)_o1}OR z_(iK`7v;Q-Q(lzywv&kO0e9UM9o4tiop4z5Gf%geNvj!0@;k12HeW8; zK_E~dGZvq0dUEv)-T!yX4gPn1nZ+t8s{$Xn{2SHRvlWb%1Q|_4Fryh&3ql@gcIH4Lf9(uP&!s_TY@1Gd7S zKA+t{q0M^U{W~1pqRrmga0yvBn!m#2Gp-uQ{iIsv3!^^P4NRZ;SnACuDhjX@18_S5 zw*SqOp>e0rJF?n8xpTIkX6KvGf(gT?B2=P}G=?I{wWq|VQ9Y$865^mPOa?NQtka z1*s&j;3qSgVQj1y37*SsDlL4BCa^iA5=i0;+u1|whg625?LJ#ROu9kUL;H_fHAVAe zKeE0oHr!R>jP6K22w>klrLs~Edp2`hCXLbrU((mB42>E#pBL4{7y#&n$4u~%HkVLI zjNS7BnC*s{-tFNL;qHoJ^VbHH|FxaT23+Fa)}DZ*33UiWn(r%+P$y%;XqhuJO0o$0 z9&NE%$H5%lgA?+h?Xzp&}y3CO4&`8WS$XD1E$8IklZ@9F8XVaKC$TXHosbpM!D}z`x@8(N~DY-)RTUM>M(%BF@UPMdOIKg_I-$#XK(LP z2U@*89K3cn5_eaT%`|jING>)dIAwEZpR$6dx3%S1N2Qv_@RVKKBtYw3nl4T{`r00% zc!KiA6L#QF(hzkvx=)>b{NZFm(B$Tl2L? znPS4wXaA?lUhVCeo9oxd1zi1y;wiPE-!erdb7UNO*<&AgnZT!o1xSx_p0uPkZSeUG_nyzL z3Jlh;Yxr)$lQfK8?J0`Jne~9@Qz0w0{_WcjT)igZ)Si_O%=*(JF&jsRdrQ9svY7L; zv$v+8CqZ6oqJ0w0e|ow-Pbb5IC=vK7))kWRYxa;?O8sD_!l;?_Ra_PYUq$(O(ge#_ zhr9(%9#kG*CjJ_u_2pA0Ld}*PFd!~JDZ~#JEY?kMu)H_YCg<&FC;61qmxQshs8pVc zv+!#O;}og5*nD%DuQ){!4-iNyY~mNy%QfbSF4veK*40PSevZbzMD$M~rkCUv4-%|4U#l+v3-H-~n(v zkl|Ps$j6F2*GJ3fiOO@H`C=yc#M8^N{@fqD6-f+Y5iyHt5AH!-^l$xKkv5BYl?D5I z7JjseJ^HVWtqb<@E>Q7y%1)YfPWd2Rb`{em0mkFMs|o%W1r+O!6Axmj84FS<0`bbK zOm%^LIyG8F<3!^~o1fX#JPlq!-oiv~)SkhbEzJ}X9!-h;u;wE_rIF!eWycOd%vkPq z_}Zwx{4C(HuPs!Xo$bv1+m|}-X((wNXKIbRmU(YS+Mm9&@W|ISUhN*|{&iM|&*N;) zf)1)L%~lSo;xqM7?+)kJJ`1ao4QDe}7C@Xyi@GOB)77|JG~#FS#CoUr38x72glra~ zME-7CI;pDRlW}aw8(ct)lv>68Y;RWO@L&|w)K!5vqWUYbzaeK-uMk{m&YI+vnCrrK ztKPaNOQq_Xe{hR$ZNGQw1pM1GMYk`( zYIas8`Z2~a?x1eg;b7HwH1Fu^1l?xiER)EUNbsXnJdr+qW)H5g%Cze3)*o)#W}Zy! z(_>WqgVAJaL&~pVL)@m_&CVNQ_vEh?6NkU3m-Db2`gO$+{o7>Uqf5#dWqT4@9$ruo zR^ix$r(D>{q^(R#ZypszrqjzeXCIV(Gn4IR$ZL4IQdy~(xbZb`cn5!wYdTf z`jSssmU_q^u3LnxU3etI&62w0NTS4{Mh69gyd}sZkJ~J9hcRq%+ks4J@-9J9LBltL z6o)aCr&I?l{4>Lwat&`*>O4!E*YX@y*%b*tOVj@c!0<(BnX^w^H54Rta=dbrT+JHe zc=|p1jz8vPQ~749i==4g)Iu7!`j*7UbnkM@=BBstrddIFZVEc3_j)VbcH$(YAHABo z>5A@YORfy<=}M>sgBXP-BriWXQFEl`CZyRSF%-e8~>hBu*SNW!cO^%#tNH`K4GsfsX7OCa_BRlCafiyC))RR+v|Ia7gTQ%*+A;}4a?T; zMFa;rK|l}P6%&jSyveA+heKCTK|EGMUv**mmS!rq*=k7_>?Eztd#agTxV!&Imfly1 zg3FWgjPyR&YIj8Ii1OIUCTs-(EUtZbdypj{RTp}s{KY)x`m+Xkd#_l~PF=<>s zv6w#J)Cmslvwlk`T8DX4kEeKBePhi_%kVzw_X>3sN@f<4PMPVriP|w28)O=#L|@eT z0QY69x>D^KHXdrJHz6j$SMkAcvA6Fz!m8~oB9d&wVsp)S36a0vOk-N6D8TQ9mVW}*W=Ok0bGT;&ptGhcsT5x+qO2Wvw3Lj_*uXbGwEg-rT)A17Dt z`U1}DKl+*F^wrFH-8A<2tk$FOI^{iEZv92S7ydoI(M>-xy9f!sKsP+c(#`}NXwsc+ z3Ms|;Wu(hU$EZ27C7bDdZX8{-!0E$wJG%*&FNf3^G0&(dNVs=tL)Wz_h#MzkC-3Y^ z;t0(T_uI7+3wj=!Ao)&jO1ERGZe0QSmp`rfV>dVIdyk_iHXgA0YtijLjD>H!v!|Ep zEh^vsYNJrmhDtNp+|)VUcznfVY>m};ab2^wdHFWw@?0yfs4V}^Pr$1Cwj)jMB@UtK z7$vX7&fyOkpMWb8x1F3erjI;aChv50*#UoI7#A6&Sg$WlukZl!>wZ1*2ndgW*FAYr z#;9A1q<=B{eJo(x`^PG+>VXnKT=(Ls^u0{E3`-k5xtEkm_lim|vnhxH_EIwPnpkzbs!+O8xhlO=QHS6=@t2kdf@ft#sa{ z_N*T3(b@B+WU6{hMG0$ae{p!&yxem%iTR`Fc^#qkWBF(9xFUw>t5E z)TQ=O4QQ@Q^Qro6&3SbW1L1sB6ay7>QpZVzr_ybPg^b3g zD>Do2mvBknPg;5#N@TE<`{yQPbT-)`FgGqtl(q^&?aE8eS`^vaA>o|Q9E7~0`K&A= zwJIgu0Z^RSW5`Ukbu3=0zK@p~jczVCtZa~#cXm9w^yO-g0yt&Qnr4HWVW`uM+lWW& z*dw>V?j0S4;MdK44{;&;c1ho{nw==*@;B)J0Bs-o$647o@Cx9J*z`tqVv0$v<-Iqt z)T-5Isk-FdA!KV42<#fYj|(ri+4DlDI(xO}yuS)#G+f||;&=N(eifp;F9=sj$!*&b zS~}^f+{7a?rw3%1ssR+6teVH2`Du=&oe`ITmUTjeJe1lBAtg#5}>!T)+jX zU!QVA4LTro;LD5U))D@&5Q&HCYA+-nxl`AY!e*u0n`AV6nAJs(brln+^O6Rz)-gJO z5u}#Q2*XifKu5kY)niK+q*oXy1qlIJ>}s3B&f-7GvbZ{6kiQiL1z#g_B)giw_1rXh zNsgIaoUYlH$lmpDXVp22&K-+d3-iCb%@oB?+(tN@f;BCJ*9EBCAS>?Al>ptijE?O> zdoGS{_-V^_p192tX_6mpd%KpYrG{`ystGzj^4Ip8RgvQd9M1 zY%EH1k2ZD98Df8<>A;p$vnoYbgnpsnY;^Fx5>{!a-`lwMK=|_9hO6J| zxMbog?nT9PHKwls8ZkFDiDN14dH+wC5fsm=B}~byT9#@eG`Pw<&LL9Kq6`)&su_nh z!pkjJb*3M9xpfbxb(ALpLd+J_Y!3jm!jEXvfPuY~+Fh049MX zElSihk9kC(1~A4kBKgT_TUVF3Jbe<0x9|4{RbNYgAM!*b={>$YzSweWZZO^ZYsD|H zJO2B~_4snpt?5#K?k$hEgfZn_YYfdee4C6qfdPaC4cTRnwFm+Y)lBxJQti+d_nvWt zmZD`Qoxv5fzhaZ0WOI{UnY8k}CraFQW;i|ZyNHirdV^qM88Na4D*&MWA15h-z`hcB%>sSmeiJpa75n~Ovr`-m8I z?stT|{&SbAZET4DwR|_;c7ihPLMTju%90vRfuvz2p|KcH3SP}2YGlbr;6mwU{rx)| z$^;^3NGP>sP6by>u}NC4J);(s6S-G`MfW(FcALIjTAsh!AWnW zS>w^jc<$Iyi}_ftI%^uuqhYvGZcsVUOu13uCZR3w&^*Xz%m5C;@M+dCgDKF5K-tGkPl z5^p@;ern$|yCgyYJVkZE_GsRKQN1I1;=~pz{QZ74It`_cY1I?MVq>6F8j$+>t%4 z;CY|}v=FA8Xk*L&@13+K8^a9CFg(Rdrrs`i5y>MPJ$qd24_3}HiP6*g8(JCvWx@-_ zN6EnN0@i7#&*cUTnU|8D7uv^kFJ_g|K@Rt_o6dWd54ZLkmoIhm8GD1V7e6|n{b(;6 zB*v!p3_`GH%!_N&KNoG#6}jr&rK`VxZ%kuKirt&qJ&JuF4Yg}iDtH%lbGk+o^KQpa zVKxrW3zatc6%xq6iyzDf$w0#E72|<6%I`x!4 z9OQAmOaCx;#@s3%lY51RUqKP-Xcb1gdu>)T7vnBq7RwQTZu5=Hk0i3hM5cqY9Myi~ zVS1J6-&?T}XhURhM}`rn#2A@{v+(*VfR<(Sw3oo9FcE)Z*s(~QD^8km<>Cygo7~NA@seC7AFtV`T}b{;SOSzLivy=N5LfBO+Lf*du| zjlIUaOCx=fogVL9>yzb#yh_!jGUY?%vd3D9jB(bQr~J6773+=hC1*Y%vLM6zJ>KzX ze(5gG+=AGt%zT5RjT^;}ZwtS{kNWnW8?D=^$u;Nru1|4aWoPTsRvh#6E~b>B|J|y- z0d(!Vi1P_+Z!FsNYy@w>!o?q{nY>A4g8eoi#*D1K`JYx>GxVJ+4n$X`NE3B zb#N=7%Sp~cH+bkFB=9u4SbcQAT__PW^!QSlbdyXz^H6NQnigH(?no$ zMuB0mwTK3(V2li#U>F~*We5Yt7my44mVdEe0b8Egj#DYsI$Pg9lkO>$rJt6O+VLf# z@S9PDXi2`ETCuG=I=4-&VAu4YTv_ot8qiO?IPA+YiC-Io$;b`j>bt`jMXO9))hplZ zMx)*RhWOlwAnf;xzs;`m7qo5Y!JA-0s?6G5=qcp)u3!%T5n(}UHC~9^t`}YYA$*ZG zlL=ZuRT!o%qCguk#-TLEJ|wsCY+>VMf6*4J#05*NYD{h&>tnhjdi`+r4tg#$0irMW zgUmF^y-hlEMw~~!Zm<(Ne+-25;l5+H29Re4Pa7M4m>Z5YLZw2or~P~M5iT3RIWdM1 zQIcM|`byvXk6j5SiLB0f*s;kjO@tD{)Yx1?p9sW5^X1^cnkb$uouhz?k1frMQIvfo zA?DXCNlY`P$btjZr;a=0+@uYwj;OGD+T|6#4$Rj&PmYpj^QBo)JYW}3wig2GEgqyO z+f@EjXj}8Vu(CNl?pe2H_P%H*7-&|PteCK|@sz!vN-(>yv{I~+pN{*=83VowVH`Zb z)2#lQ4q2@F?JgjoSeiEhF=j0-+1EE`+OP{;_{GwiM{`3&&w89U9`(T>;K3BgKxRwKdxCkKh$5Jh6fm~WNzDG* z)DeZtovEB`og7sTM(EXX3-vnO5#szeAQXa#=|OD%?Y)Pq>mEP$*9x^zt81JY-Pvow zZPJSTD5RnyA3&JJ7+6Wa4sBeK9UT7%bPYlX6O`1t;6wbW6w_~pE-67v;APKW1LD`X zmG&#ktu`JB_s(`yv4Pj?yZD$aybXN9JlF;yHME8@yGf7N57tt6z<+QGP8ChfQf`Y= zF&@n1&whI@M)_rmQe@MN{nT%QYAP)+o>@T^{CTxgw-#P1PJ18J?40oC6K?`|y*txC zX0ZM@2RL9(Nx_Hol6?NpeUZBdMWxD#EC`7~OLCK0yzT9sfZ`MJKf0eq1pZn6%YRT| z)BD>svyLvb^Y3f%-&;*Bu#Rpkr3X9T63P+ES{XVzTO;dd5hLeFWy=qI#Z=r6YCz71 zi*j|0(>9ZoIsa@!ZVz>CO68e_{}?~}*~AxiAyfhWriz@M-`FRU0F3xZ5Xxv6hq{kq zl%p+CZR-XOwYn-P%zpRzk#1)3+G8ZjAp-aV?o^t>roOZ^uYM~=os~Nd~b_lJ|r)gIn-DG^X_33)|s|nxG>a_&31%+$H zsEY9@4fylWD2Gpe1i=}eDc_V{#!%jj)NkzK3k#>_2~#S@<@DEpJyw!^(~b0Z@G06I zZH2SI{{aM?SbB|>+)3dt_?_LTxqw~Vr`!nDA-~uda@SW$|JJjS=oE+Rx!knKNz>S`TZ2q1bc_1Urv)rTQ^Oke$NS;;he@1fa?sE(%85avDU zmq2!gIsT=SHjk7GkEI8Yn6?QIzLCK472(i1hefZtD3TKyX06IHz5rq^9=^$GD(-)rXwmSf)JQVdX+riPd}QR1_YHeBuU( zaK$qT7sj=w-W2Aa@ADo!P97ka%jEFAD-Rf3*gQp}h%)@uTW+5Y@;xa0(#QWx?i2$> z>mI_)mXAvJ%FFAXn#1#zF_LDTJy$k2PadSStYQr|^jd)^ z9??rJ1(%R)5NFQd^|w4(b$E7Z#;Nx*kC(qw1{==@k&(Jt0Q=S@N@@rm(4JYxZ=kpX9TY zq2%tKRfQGaRC3Z$Gm4J!$oPw6{}+>k@vF&gB=%X^`R3mk6|iqklWi4^KM;0tSeIzwNu zUQP2&9!tJuN=k2TL~7F02HL&q!Lu$=o+*aQR2g$T#eSl)m&bm0CkEP}v`kD8gP|+| zqkN`=f;<72`jM}&O%8ek>n3p|+F|NgVL6VrW<9jn%6r-)gadweFB-SJO6;$-E# z2m2YK@3VEIa-4tCywQ2x9*w@8WLUSh+E9_6;B+UYJq4@72N3F@gM7t$4eDk!L2D(iQP`;#FY*NUEW#%(fcud zQDZNcc!Fqkzd1iW5pD08nMb5)daYBuA%zNx@=UKL60E$5^K6Pd()Ee_$B?x<5hi*T zj>DjE*rS^IcKFgorLm3nQ%oGxU#0ZhJ;Ld~LqqxK4~t=ddz0FFgK@e*3AsPPV48&9 zeexF$y!CB&sQytIKS?)!93F4>wN%_4A^*JISmR>ylwb|9dK@w@ zc(h-l-FGR3!)^`M@koozrU|a?sP8k*zf)}-O!t+7mdD6uTbpL}$?%Q!h+mO!T}C=1 z>iF;(^1e7f3?0>_(I+`pi+2HAv&+7G>fZ&>tJ7v0v$87;pU+*QfTOC4`TOCb(D&C+ z2*;>@0vzLCKjtN|ylF6AZR{1gi1K6bWqZOdEl zZN?54baDnfTD_)n$ak>nwX6&s64bjSXQEtH(Z4}I+g3l&;56KejZYzI#Y0t`wO2ZA zcl#ei+?iYTnMUg1m7ci61nzkL49k@)GVRl^zLQnzQBg!l2ZTBQ^59g!g$2?zYW;oA ziRAStvE~Nh{XlPaCS(h0bJTF?_19~_*iB$@AC1{q3kgBBoT6>A zvzAe4dR)i1W%qp^drMpEQG~?#IL)O=yh`oatgPX>9x3C{lNtYvAthNb=zQ0=wz1Jk zIl-5nEU|;dCJ2%xDdvm1W{k z{e>jZ$5B4vbTQ=IlS$WF-^0{-aYf$ZL~9~C`qNG|wepSol$1rQ`)rynF-`EB$rT0l zS5NWBb^J}86Oea_#HRu>mhxre5+_-lg$G|=mHPQai0X-ju(oO8NGMzfsoxBi5o#iLLhoN|f zU1;lSq&KVN?E1~Ri@tuu3sJgQVZg* z80s2(5GMUiKr%Mo`^^aAnd2Ju`g50S)BYl#5b<$sR1dxc2N=G*M|lIeP<0+vc30$IEA*;djjX>+BU}g_;->X^`=~iN-4*P`{=Nq)Ih5%8{?4iR^^y7y z#IY%bK1!2+?F-2^daC6r6qO#k1R^nL)bb|I4)wRK%*Tm;@@xfNy==Djc+u-vxNmXF zfNyiM*#GKFo`7^LU#m^I=D2P5?$drpr5bCkg&)YAd;B{+2!E$8gPzH_)Fn-_$?ZD6 zMLxxEHAsTxUgBTHi%oCcpe#PQijcfPpWK{L;#OR7cx-MN28qA7D8|<0Zv6DLz&F;1 zPsD_WI?|OJ@ZaqHXd_e{#3JnKXa(bRu2%QK~koM~M@UI~SWx{&L*{XZnBWLOpw z`YU(i<&WhRCV`)ggfK|T=Ok$|h~@+-qc7C&xim!Y_X@eJuOLd>qjbIg%Q`bi;_kH= zLjF{NjZct;*-gjDuMXaCn^eEJ*OO*p1$lgTzD2ZQ{|hiCDoC)B^ z=or@JX7BlEao;rjJAE_lMuS|4nZ4U0)lHz^K!Th0ywpHn8ddk|lH-PiOe8`NC=#D{ zY`~kU6Y(yD)!CPm3HzM&zhY$kRY}vO2WOQMX`s;1>`J7Hx~m8vgpU;URYZ%xjPJf4 zX;q?7!uorhTi&_3R`mMc#xCg`)$>N_iX9R$sj!zzxNrxmBTDu+2x)WonqlE z!OXI1>I=TobpmoH9x~_LiO~3uFD~Z9yx)~VeWelW;|fwvg(w3e|H1rT)V1Un3-A{; z+G6{l6z&XQ9@x|%OUiy~aP&<%c?iHkT0?~qM~d-^X`&6VeiQ3HQVKU_^5gfPmd?`; zm@#n?LUSz=Iy=xVZsm)A!@3pGf}kSgvf11BHl3$A{+~y5&$1E=D;>L27;tTeQQC1l! zD;{BY^yFl6<~LS}iwHl^2KFiyDgF7^#lx{>Ao@UJHsa<3%El!k?v&d|msm8P>UWFYj z?trdlsK-QsT+DE%++Dbu>AI@-Hp1f1>^5I=4=Ac$)yIj)!2--MP6ynOX~@J&8G^T+ zC`jc-38H$M`Ad72_OR+reD7QbOpreowNVdy0?IQ<&JJ>bka~|-wX-V2pUrzP z2SA4ryawFS(WG3pI!IM=w)ZMRKqdAscN%pd|9u!^0$L(pIb)y-#{`LIU~Td+C z!CcXcdMrb286dQj*0MQcKj@_nw?^+TVUcE&`c$Y2;{LR>r>NzNboE6!R2_hHD4r0V z>7mM*?yo>gd|wQhK-06@AIfd1*$Sc2iUDVvnb@kjazMzy-hdFN+6jU71^G@i;Z0dM z*PuPW|6;7XX$B>QFCS?RA8;*WMQIBgdFZ})TW}T9k6FWz)S2qY6nf*`;7Pidi<&Do zVrrq6Q~v1|I+ynqZAN9IN_sPCu0QXwnqZ}*q=#mA%Y54`Rx^C~ui}MhU+K`oLkQOv zJc7Y?lvINf`X+=))Bge5P4zzM@;AD^Vh>~H1JcPD@O+$#vLGprBI5;`Hg&SQhoRvOEu11IB*D`R8Pq;VjLP4J{~eb&IvMaVT~E8>YQXlIIytD?Vz@ep=awPi4% zc`xO2ZwjcvA89n|2lDHYZRa*~Z2B16Z5po#e3`E5WVG?gC8R?Aq97a3>u3`@!7EcM zqC;B?OQyP_^X~!z8Cej^%N`FsUU=Q6iOOYgh83B#|4N@(d=#SeBij@E#b(vc3gRP- zoH=K1fw9`el85dN$63pp`#)?+mrtp%YNk5ZJPiD^h}Le{aR0ky!}VEIU;poMQStKi zX*uR+f7vEI_;L`31<4c$qQgnUwwnz2T$HuaR;5yoL% z?YRhoGwxNKXfo$1mN>D;TS4jvg+X`)?IVTp|9qrrc5(9@&Q_jJlT2Qx;4P{Y& z{-&GxIzQoaM^yhy#(~}#vx)QEH`SUPA1Fk}xn#m7Mi{W3OqO#j^+YNQxeIr~>IG?{ z*-av4^)7DO>{%+Mn$D595duxL*BkA8b6;a-2MsrOT17#nQNOrP4Ax*+OVd7~s+%`J z3{8dcnw&3;X+=c09nLBpP4Fo$7~$K^GY`G4{{2&kkSsIho6uj@gL=7VZf^L%uv_IY zcYIHOqJf0a4r6b>wIM|0{W^(H#)KkmvE^t$ax{YIv13H5#htiUVR!`lI&7&~qt#SgNQIY5MT3Bd((~A|G62=>0*gQP-JkHyi+l^(M1EW?l|9JD>9t zahlDW-|b6}s?6EDOZRgW$Ca=;2jrc}D;2R$zzrMsf5GQc4MHY}6h#jgH!qvZ z>%V;u>+U9BKAct+Tx)#;ii>~MDmHmJWcszk{z*2G#hZcW?ADxn4^sW}pVfIVGwKE3 z^g1djAPanP1cvC@-SnB_@H(M2;ZpI;P=X_Bn)RpNdf!BU{zN! zf#>mWHreskS+0Cb)j&ah^>T@67QoFI>0ZRCC+~V=OHhQ^-;>2T8#{PUL0wk|u4hTU zB(g`a9J3V^v>H33JLcunE+*4~yrlh5M$GEVzd1#SQ+kzr-i4r`=BoJ|0J>;msai+I z?;f{N8R^D^2n9P*-Mj31s>Ycd`yxZl#Ty+W%xEo=yFR`qLr;J7{IA&+K727prdTaDvFq#b8-m7#kiVWSIo88(aVcCbfF$?V1*7iSYcTWIniW!=iA z?6?tgb6!?*eUTlshR-2Q`{ z?i!mnf45A%Jm4om{1EoOrRil4*vJAO?sD&K)gye*p}odIo}bZS_&J=zN@($5XTzqr?7ttR*o9f(A+QsccI;Ni_So_7-$uq^Px5Klx4^!A(-KG7YTyj@c@v4o1+SWJS^tQGD<@v zQB=EvPhym7-$iN`kEKd&8peQcqBEw*j)s4`&P~%TFf&<#(C9p@q6@kM{F5+ht)Mx{ z?BdO-0AjqGk>E~D-tiFnZ4(~Zv-vkW?5?wriIZSR?FoJr>yYPF=HS)?_ry(Aj*}q_G~D7k#}YL3fNI zMdklkE9;a7A5UC_?l9CErOZ+eDGH`n7W%$U2ZCf1d?s;ConO<~mCLSM1dN=NEpEE# zWJ-mGRNM-8N$bNCC-|!i`Zh}*ub@t-8D7DIY&vWdV0|qsFA8LL+7>%y{ad4{(ls9B z7J@%ujG(kYhFBMW3MI;h%;7CbpGI4#8jao4N)A@x^mbwI6P09cE`0(Ux{ie`OPIOC^X7l1_T$eUlup3YMFilYXnG`MA;Nn}`eC4N-O$&WJe_pJZ?K6ZpaXKU$>6m_b zPkXF(?*U@4Ok<0al|+$}aq`CoPpX>#L&eem=3Qa!kp1kd9Qba!dQ83Q#E*L~E;FsQ zE-n^YrG=sx*pnt1D!avGV^b!H#&MYBhc7Q>Kf8sHmcLSZ^eaEP+H z6_AmK#02lCZmb-zgC6B8B6jXpEn_5GvwRsLl>9y&wRAl55CcA;=rc?+@;KZFs z^A1?zeiGT>iaB&~y9^6_ns1I-{QU#{dgG+O0iN4B%4+D+ofsg$-$BD9nUzF95h;>P z@eKR>6~exCNIJ5PA2+d}%)&tV^2(M(iYh1*{O%;7GM$cGR-GmdHH?~}oeENx`*o6` ze<(xXlM7vohm-Hap(M&`tS)&cRSALDavi+OIItXjH4Xi3NT`)}->i#)C8M^7N8!)P3tbJ9K?uKPK;0`)4JTFrc&_Du{0 z_nF^Hek)gY-@3opHD4~I#~t=T^Tm$S%Q~f3eI{-#f`WJ!TY|-IPdFMCwK=|{`*QNu z{y7F4cVAI#kSI192Zuzs=kR*KzRIjgX@yzpzHB4oW?XgQE*dJ~3;q`HMZ#2B{d97p z$D{2maA0z@H(Tt0W9<;V3s2HsF+WG8yChpT9+7=Xl+SL$YJqPC*8o*=2-odFL-+G>dcEOE8XO#1fk7YK zh!`O3#_Ytt5FfEr_U+GM5A(cegI5ew@O+LE#Nhf6z}_kD+?fR*Ph`5w>87KC$_$B& zI2)*6x;bv7Pxg7_`WyOO(l-AJXWNJv+=Pzv2sk(s)IW}YkmXE5j%7Im4Vj&rw^^id zzVEtp?&|D&8(}Kr(21;d$DF5# zlm5g0I)UY$tpPojxz-8x_-~sK2?-QZrb6cD0Ix&z6x--A?WxH9mkSY9CkEKPYe}rD zhH_(zfRq0doo;1SE{l&8fJ=e$2l-EL;#i^UfP)tPt^+dix$fyG+IM zu64CHia|OH0|x$DYK@^X1i&y+;<3Z$@srjkk{(s4Nb}cF!oscUpG2RDO9)hJr3z&Q zD}RyG^tc+se33#w1&UV=k4wMTWn3B-hU={*`hb6Zz%BpONbtUhhjKYXV-mZoF`uFX zkKbKxMOfv)>}6OjuW`(vV(<$K7Q5iy-8{OLIe;6x%-xi}^(Ar341Z<%z=dyXzH_?x zND`x#RHSpVoKy8T^7*en{hzEulg$zW7xkV#viZG5V5-v7pFL?h#lX-uuzxn7o;OBT zHtQRjoT!NpwBW|saVQ>(|2W&!W$&U-?l-RVO*tH4hB99BHW2Xeax+#1q-?W+Yg2xNwRq(lUXs{T?)DV3T{Dw;wa1SOlY0FL!O-6)n{&{c{{blce$M%kt=`Rt zqV)m}z=xvUP6S$JnhwU?^+v6L^yfVuADd7mVPt=5J@TBxc(aEq5xh|fZ%SDoy(vuX z8`{#9(_x(5-_nz@k;M$-UF~GVOQqbj$2l2GGo!kc96cfFrRY3oUC|eIG73`Q?<4fV zLW2{A_1!KMV&KW&o4>5*Q)i$n->l%bCx2dxBD%kIy9hK@^@J?hbT*@U`kMA}vQ{bq zFHii4FfosXkdq&~ZW(Fr7jEw{M0cwxkt8n5Gp$0NJab9BhiLqe&>cNNuC~uL2(xz1 zh(;LDm`U?DZS?iF+=-C1Mi5|YQBGDGZVm$wtM9;bO_C4zS9e>n+}$#&PUz6(?c9B> zB=^ZTjWx)@T|77EDRBi>QRnbbyao1G*(TTn_Mi3IR!O$|{{VEe|Kt`MI&Y6i|Bs^c z{-^Ts`}nall`DLb&io48HMw3J`>-kJcQSVTq-%Z|}-YNGLdW6MK+yb2?Fmwm?8f;J`=Z)StOaz7| zS zYb5+fLjH>=EVRQFo1MG!Z`M^Yxu!9gjR`esDdt9$`^Qe?+#SOguv!bv;c4=G$~)uT zm4W)Fcz&HYRLC431J~WyU+{zk)8L5E$`$VJhGg>U*-;BnurjXeubhVIdSn?a5nwFyN+fLaQEpRgo)={HZdG_*(aYV6^9o}fv zFH}|Wdv!VI&LoNFkh3uQlt20F>l#lBPJUbt;P=`fG+139?5=cv#fQzqg-5h|C#&9E zqcdjk*`udRPtXX}>DBsaOApS)OuN4K9$2FY=|tW$awGN$Vs|?EbMN56>jBJ@g-kzr zE+OdP-@N6wW8hp}RhGS%-85ViW`Na1@V1(1?saXH^G|HdAG>IL2^BaSyJ(sd))@X5 z;et(LOL(m!r+q&YWMnak-2Sk}fYp0&zkqdo1J6bq%mZz(a_df0>a(aw~17?STxxj~c%4T~2fj0-5Ize&+8c6bX9)q6__5`WHYtcwM>|>DO zYh26GJ^7<)EkN9)eC)4madc)NjP&}MxThkF6B5Bk~s?h`ycb1V{I?4{z@`<7~CR|M6;afKq#8Dyq=uYGk|w&Qz~7q z7%&JJAtH|~75i**dv3bl(tS`iL^2exbiBgm>Sp+YRnXL8xleYM+Pt56tPPpzvBeKR zQ@tY4aN+Vk9`7txuD{v9V6#VV;ZQKl@7V`|2S=4@7mGIEpPFCr=5Xxv5Teq|cPX{( z=s03F!ublARj;}z@=qNIl4&y140EDE@$wlhJG54I4roA95L;0BT{X4J`Kg`y>%XKE zBF~I+l!#TAn*rnLPrV(BVh(`V2Gt;XcON{fc&(QW*QiYL#-@3>3F=8CQautSE^nW` zdYM2eL?1Gi} zNq8LJv{n2OvglJmoS%Pp_m}tFz8QxdNFueJ#v&DtjvLj5oYQSh$rh(29A#b)X22W3*u}! zHjwEMm8t2YibW^_Yh7RYM~WTm6_Tn#gNLc_FCx}zRtPhdmjM^)t55!f=RsQy(q8gj zzSj!cRecYmBO|Kt4rC4Tz06;{j}$6W$D1~fcX%WS3T9h{@v(yhed%fjWckh(%DN3! zf@61Wy5e_+p%7NF3F3sz@t}HKIBba$&Bxkd={H~IXqRG8@6r9tZJkHdl3XjAYIXRY z3_s@BMQb~*%fh1Ys$t#aGB74$zv~9>;8)?(x$-xE;lyj$j#00NRjlW=9>Y{kl-O*%{)El_IPbkMS$OWP^mS9`TlB}KMo@Z(rOQWt^6UctPd>_WAR-QgO%ndzAUAb)nWshPPK;+(=FqJZ`A$d zE>K;DtIm3TCl7A=++PzaPNYuSPx?|S)2?4cGy_{Fs4B{9g%=Wx@O}{aY8S3Byz;sN zSUU7`^ag$6{@u~d^SVChM;p2F$TVSO%mztV{k&|IQuGyq$@`*x2ckSu zq;oa6SIv(}w`;@mn$6VaZ3;xD_j^om5qdx##64uOh;iakBS$T)yS1$q34 zzEcOqBe4I)>yG%jT~1GH;C8Kiow_ZS8+X>mtan>5*wX;c*Xh&n5aAvFuuUkhI(ykf z(<1aV5v;7U;i9Z3D{*kYqGyr5Z`9V!Ru&v~OPERdBe57J!obPJMUxl(F4yV46v3&` zDet%8+-Bv!u)X}?^JZj6b6 z%}V!9iM3OS8cKLB+Jr5}oI{V=>!WgveOccget`g09ms31=TEyu=hos$-y$R^EyjF>=fD}5$Ua-4zK~DI0_I#UR}^gkw}evX zOI{%5W3x^2mm(RqJnCIMJm=H58I~nOU%z$oKP+-EBM`);*~Ff0j;&_X@@h{9ac2i) z6B(5qA*IP|$IBM8zIc<{l$Jz4(QYS#)5Xfl@_zs*rJAQi zh)Vn6Z-UWP?`nJdNsADsEpcdVSq6rZ`0nT?b$hB~ys_P(kNPVDP78iH{w+&rWK#>* zNdGrY%r^4Zp}di(u6jP@h0VkmEPF!-8WGi8z}d-8-wDsU~o*&NOm2xtW+S2#vx_&bJN|d`?M3S3Q!V|J$`_qOURxJ}%kF^hb?qAy1wS1F=yebn? z@vQF>X`BhP*)AGx;O}@E8X|f2fbjlt4v=lKxJ?UAA%sv+0mx18?E##+Q0lv!Lu=f% z;T9*fQUZ@Z(GaOOkIA{R8kE0v-cqo-dB5hFX3&rat!?3v5T}~)<*K$|BfpIw=I5He zRditMetQ1HsFHLBS;wcqHSgh$7k@L`C$7onhu-YbVI=r9ml>0#TZV_f zuSdJwx?MtU#-mR|N6vSfhsvr!w!K~IbKe~R^H=XU(QcaF)6iWy!iDNIwSm>teKok|FvFXtcO6cO1oh;qGqB^6 zCQQoB43Q6Ihkn=QoK5XYNcH%8y>6RP3@(0_nH z^+0{^joR=-rezh(Nz;YO`T(Ky)-fIl$#?v6;|Z?{PzSN+$k$DDWHdWNdGoorC8O}b zNXgneaB5v)1FLito1D7#gh>#@Q$3BXUD{Sx%@4Ik_%U_qPk>>hWl;qV?kAB_U5zGr zzZZ3aeb5o;%>(oITwb=hc0zQxS$=R3?cUCI`f#yu^$Pbqc<#Z-)ZDR^9eDVJPM(WT zP*5;P>RlH4JFye)@`!@O&5h^h*3y`1r+70|r8a2xi5xNj-e5LX!J*Zi!)#$mC`5Q`Yg8KOnx%r(@=kOPx=IJEl65pU+*Maix2MJ zP2-kY%)y5M%+(JU3aIYPKOWL)1h*b$^h~Nv=B_~L5b>7pvOVzV3x6Citp@+> z$2u=*>ArzD^q&@%ypN-AvJCibhRE{R-jd4h`o8FY`SlO{WKNK)Qx)fQQU(8)kH3N1 zw!ZL~g@mL}XL6uCS@i`!zI!IDqmyyQ%#%)|Re$<=#XbI8*6JJPDfq_;n8YV5%W*?j zhxlh2?1B`M)PQBHjd$)7As0Ea_KFERc00GeBM-V|l`~P0?f(FDrvzJieX~S^-Q(MP?ZlA{0h4>7>fHKbklZr)63rQ0eVomqoz~vyt@}R?|Ab4 za^1KcpZ%?)D`sJKCZHBre(IW=p1L&L@`kSYUB1RmA=3t;D(&Q2*Zxa;Q?k-01M*z* zF4U7-(IYIcmX23~36kT-P8&}er0?F7%Cml(+H(J8D-|MPi-}9*XAxw)rrMKtx-fz_ zzI-8Lip6$ ztn1=962v_8w3UX)6J>IG0)u7nc=pi%vcC5>?Xdq&#u}!yPKQ zIis_kWB~|4(6?lrdL`5ZOi@O{lqGYLGE20)@{oKRPUAbHZV3pU&dL#H^r>$s5(Khk zXvK^8vhD&Bq&oF3td`SFv>kh@F_l$!FA1J7QXZwM{TM6vTDUl?`lPlu2i+|dSedV0 zQ7d|4TUV;v{^E#?TcKhl62-Y=Sita8A){M0DCpCjytU)Gom^|$SArvK;*$-eLWv7T zA^!T5lkPs#r~bnXUj#Z`U!z(7UarPVu!Y&NSxP&5R<+l^?}LLASN`|8uao%DqMW!e z8>!t|%!TE!_>~7mYB`Xa+7wqTUiUO5E3&vPChB~#_PVW~n6>{U^J#nbVI!g*k=GLO z+_|4jIYV@B`Pjxb%TAI2KaOlUf7Fcc5m7sxfsRwIN6RxPlaiT@2X<}f2Tck`@t!3 zRGE{ZRCIO%{hKP|x=2@N_N32N}{K%~*TB41%}sxS9ETC)m|MBe`> zXl?ksbWfHt2+q-UgfaFd6vM0`TpW%N)NmALdYtg^C_gr*p%U;`Qx+d!Fod7L^3%*vlbL+D{=+S(f z!tm?Ub)H;CkB9Iv?GVyiqj`@VBVY*GkMdt_{INKF&0J2*Zp+53x6Ipo{hk--x$^Zl z?MjqyNhinaL!X=R%`w#V`)aX*(_;5n&-T7a6;pPtIU!y2!yv7Ji<%wc5u?i(GmmXyL)LOSdz^+kvneCxC zwm?sc=n9a=K{H$TM6f2m^HQGfJl~Ik_3Jm{OZ)z0la{a%f1J-&*PV(yETF@bcuIJm zB}QnZYg{W9gn@%oyK^3f1!*7JjtgLn=#8Rk?L?A(t_5{ukaGyC(v-O1i#U%HCnSS* zQXA?w)4@$czj6VdUf68&EO*}-oDCn-x8`x8cys1oY1Oi{Zw-|NpnH?2|EzVFLT|)g z6>+&sk^x>m7U*QFKXJTzAk&UnARM2=hpVp_xrUAx7gib{aBi-3A0L{sN)0H_C-KJ( z{AxT=kZ9F|NPar%Jm&M+J}r~r zGA@7t04}juS&3Tbv3u0+;vLvYKKHV%*PH90%}l@&|p_yVhHH+lVFCR;IufA zPmQvHA&%m6C`#+LT=EmBgHfc$B8x%QG>SSOw^6K~rqKsgA7wI!s8uevXMYLwW$!d< z56kd&E_{_bh|8jIUF|p@5ZRd+tTL|C_q}7x03=KZUzHbPk)p;+E)l$+?@XB-1`eYrZXl-|TYOlCZYijC*KG{8~9P#vTRt#RkEh-pnn6ll@$f5C* zjZWaZ%S;}t^GU{B{wfGMjA^PA z^G_Cj$=_^@^DvtjfOiaB6xIWAVy*ZH_Nx5Q;9#NCCWGBX4htKf5LE~EiE%Yzzl`}{ z67I4}qA|bkUBhaOB4*R!*st*Gc4MQnl>NE=_o9@tnW$HRjCtHy23vM?`rUX&kXvex z$NA~9AsIegExM4N(eTvKDllAi`=zp|?`L%yDkfQVcglFfcON1JcNrGt9as2Uiq@Gj zCN$PuQM$;#L&FcBmGdbdQ-^-Y-)RipCPMLd>m$VzZHyv!!&cHJ+7gg|2T5W}wY|8( zcWZa13)M{UN6tG~UC%(gzITZdk2qmP1WRg0)oc5mi%KDErB;yAmSoZQpY&JE0HF0f zSTT8pj*L2#OVEY-t+ZErQDkY7TaP?Vs+PV^arAfP$}a#-(lxO%DRbdr5YG5^r?~A_ zpQS$l3`D^rbRt|}K@m)dYCidRzmYEsv|yEVJ00B~R6?#4VSCADSCsP-)*ClLxxp04 z;AM6#GvmDvga5gh$JFf9z-7?pU5>!(^{A7Z6~9b>d*$?XM>ro;sI?lUOnaUGCIGW| z07WfXC=3jK_rq(!0`3qATHPrIrE~Iw+P6S)&L_J9#mN5vxA@YH4qvpbi5+dCL|UIA zD#q_I@&^0XiPN~UyoDBW2ewpKWoQ_0cwZh=`5X4L)%3mL5|YJm&yXW>t>K#mL-X% zJwV_Ulqw?&W$D7$F+pZF<{Q(c){w!wo&8vD;Npj1rPk0#s_04eySJ^L##39B!k@lw zyf+g&vwQeg>TAnNLB0bfd!gw03+wLQ-=CW@2+&)F8c42BP4AayEj{EIY6m9xHeSrg zwSDL`Tb^!hTl1=r)qVT#2%e=;*Fk*u5{q;pA4Y~2q@No|qkVzDp@+j$3W}=MY77(I zdxM^^WiB`8gmG~UY9V57(K_lG`gARtp+U=nVE&Q<%sBD~p|&Afx8E@|0m1&yAJ#?+ zpRPZpE-jspxH(_Gguvi6Ed#NOO4-fK4CnUJzScE(4 zRNF9dB?9wh!iW&$jnV?kwi(_1HobWhZ^soqg9UGd<^}H-uvOIe*&oIHwmI+3i?CLV zPqn_PoBI*-8@~9%dwUJHlCi2UG{j}SoIN*ZBi#ZL@Pe?hbP^ak-hrD0cV@07cvM1# zdUr;??CiMFjl!mSGR5!;$_mIh)2UIjQ)*oZZ$&};^fqUjFL|4SF{=pe>EBI!;;oc< z%I>s~5OGsjsL86VqS zFM~0zaV4p}$_3L$THUh((zF(cXUL z;zFspQx5h=<%;scO04CHMHqpxNA`+zNae9&1fpxq`5@y9?q+P90I>gAL`zUa@%F@j zm3htR!yyZb0M1r($2m0otWXJmplP04v^h7m#Z2^LZDIdAdb#vc`|36*07OfZ@YbBUgkHT@xSSKnodE}G;*v2tp&NK8L2Ks z&XL8JH;+Hii|$jQ82XU4O>XUv@8!Yf3zsN5>3nK26)-T$K*w^+#oON1+q4BfqxxNQ^NWy<0<;S%>9;c@kg+?9kFTF27RMI~Y?QVdRkBZ_ z{EGG)t@^H$!Obs?>>l8STII0MCS&chol(Y1g0YA=;x_zgk#@7cr-s{=RQ=8O6&1Y? z5xJ~vUeg-{BdTF{?zg~(NAA0?%B)<2)k>RX!mt-$RJ=T z$F!}j9lbWUH-WOf#HC`xsxK2+-n#&7PszqD9De1^K|Z!$9TffR!>Uyrg#{V|38V;d zWfuW=Vr`vrWVQ^(LU;n<3=zz>CxpPqSoafvk0sJbTesu|h#cdD<)v{|2ZA-cSN-&rU}{lJj$iX?9`c*()u~Hf}}8Q>I9RBB!w1)`MvtWbw|lM z&Ew5mR3k&KR3&Rx!+LZLlMFOhIsVL?8a#2fz*2R835om<;ybRVCEixn^y$iR_vCIm zEnQ!s=~fL-ebEcbpPlqCdNsM<1pQLX#JQV&hw$5qayh0J-3Et``q@fO<&TAUqj3e|SbPU-`F@S!mDQKJKl_B^-v0o1N9ktg?uS3W?6-<&gekQ; zzHK;?94QbAkh1Uy0!l!g<%)DX=(sr6Al%wE4tso=mxgOomED3P*1doMxpcg{6i&Rv zTHSCQPZ}kZnBr(P;w`yKOnj?rRl5=0Q4iIBMD@zRtbU76n;O~wIDTgEm~_DvWGq8LymD?=uV9?I>g8Wc2@<9K?})`^A4f4 zr8g}ZfaHl+)2l?Rg#^SzJ|erG`+U&`cKsv#bm4e6tqS_1brsx*_-!8vyDaxx-$!TI z_UJ_BsXh8A4Y3VzRl-O7&jIe!3ihj)&U-d=Q{ z$>BE>CX!S;=xqgLQ~8o5AqCdbW|RRU!xVVm>j}6n(%Kkt-x~~D(F(3>+xq7Z8msme5nsFg+8Nt|*EE9d z9&u*XV1ZB~)tPLE`Vg8ph|Jv+0VoHL#2B)8GRNE1??OD0WrLk=$XYTqeF0G0tT#d)=~#O6AZm+ehp?)#<~Ei`QmN&dd)TDa)NN39v!a|x7pWvV;#FK-Bl^L z%io!xB%(wcfZvt6@5zgW?uuBL?3h7TGm9WJ8VvA&5F@Gb963DZ1ph)_TUK4A6*EV{ zFxHy{Kv6H;Q#MgB&&#xU{vSZ;XnoOHf4x>O2qIc}n7{7-qvh+$NkcDd-_hscXNx}_ z#$=PBnRnjATV+oV=%0{p=iG8MB1_|bRV0WU6;AuMf$7wYU3%RLOJ1+Evil-`WVYd5 zsSS|7(~Nl73KukJ7b`lT4|5!3F9*@xktRE^c_?nQP0su>5mZ%V4CHmA?U8 zfTMbEL_GQ%St=VJ6O9G60k0|LFO<=(lYua!{R&LaV`SZY;M4mGHzRL6uJooqactbq zku%KCH&JPsbqp-E`Kz?=AKo*h(B|$Pa~I=y|7iW!kBa(AlD@55=ZU7B7a6gQ8^c4r z5)}D1PQ`_JjPK*8-GgP1ZuB=922EGA?YNCurPlX>Zq|66B0X)@lVs1GPZw$EckrRv zBj?FTAqzUtEwXJQ09W?vO!`!9!>xn`?^B|=OjX*m=7DZ~%#duvYm}~s*-wG=kWh;R!W%m_O7Iszlda^kCfkJZhN?n;=ks`pr zXgmWC@=Us!$?;tZ9o1>tM&1PgtXtzCQ4#d36B?|n^izR?dOC61)=X+%RD;u6>XK}0!Sv&>Dpe8iywTgr-nBoaTyx{R$CGP;KfCmcYHHDx6!E5vY2XwaJN`rk&v3Sg zV{a@A{@#p8Zm*Hcc7dpy8f7z)o29R8s|F+fm|<5 z@CKV%B$F;qjK7}1{-2+@rju2{EfUxdwf_$wy|}!perzQZ=yma3YE9r&o|sfVKp9Vd z3>KR? zemol&eTd=qk>qmKId(IU1c2jJt0I#k{DrxiG(wL9Cym-q# zIAO@->vR2&7aF5+bDZ05t5@5$Q#ofia~>!1_k_UPBOw$3m}E z7~&E@4=vbiv0YJi#m(GH8s91z=5EVn#Syf*KpAhq$Es@Mf^1Sn|As9JCsTMnt7a@b zo@`NI2H`xKaX%49*Ha%1`vux3a$(-Sv)~CVm@7L%c8~~JVHr^68fRQg%rOaP%Y30s zSTuV0!zM&T^-M*njHDs!H#$Bc^p&rB>t4qU(n`%{DSy3U(N>j9T|Iq;vSoM~Ur@0B zmA~VI1&SJF0RQLD%FJI58=#_~^o*}|zij<{G-cD&)zZ>o+3v}r@fO*_AtliD?`N6V z&>nnl?A-^!z%`f^qGNX9p}goZYzG1_*E7uoMw67Is07QK?PC$u!Fmysu({AHXCtWx zPJeuBzRy7N9tsYBQ-KioJhtpIq`sgb*>cuBRb1RhYM$s3Tb^L+0OOv8*$JK_$$Z|j1b)fZ zhgr;kcW2pGh`3@H?P6y&<4F13)dD9cP&LC#2g-oE7l-fETKTB_aVsu}`quXx35Rxm zl1JKq-6ScTv@?`qLR?jyG-{Iif)CFB^Y_X1H}y!dh|14hcN1643`F1#ms1ZEh_VMd z>4f}X0e`9c|3p_fTGQ05MTZmlqX)R*Opn;O<5dq#oyf>s^Ol{Wd@DPX*T?lx&&op% z-O2QJ&d3Yg#pyj`AH}O|boiyUv^7Z0ZhH-}frt%3FVmP1=P{{mX_c*+nG5$dYi3q| z_*b2ga7^ubrzy{UFgCuvFr}NoiiV3$N2lH2o6n>iT?=p#(2P6XK;l-!8vgCY9sv7` zgS?V;(C???qgbFs2C3FNp6Sxwl^c{C_K{30Npe5K3Pmzuy6+=L$z<&qs5ish#wm*V zP3EUn?(Z0QQF?grXmBduX8&JaBtJ}(EQuG4ct%0(=_Y^pk&H1T-Dn$bo}3Hb@ymmH zps@?<57308z7(S6Trhlpzl-4b?{{T)Ls{K1ox^0sZ;;e>he4UCv;rGYj)7V;T26${ z)P>r>HK}kX(FaI-2Sg4|o{aU`*yb9RN2Yh{X`0bvRT=!m$}pS=Qap}JPk>l#cG}|r z=&;a7%TiDcJYX`mv%7i8xFw#Sv}ThgmZy|aXcw1eT+GA%1kNMtH8G%GQ1FC=9_9$F36cPOM;McG=m z{MoZO97=#phOZI|>od)R$wBp&;V8lt4lu!*f`)Z3o(vu zt^Kg>Wfm?K5l}7WBqrv0tTZ9qU)%nlQO9!80f)Bc?`Ml}VQv%cjQ3=nS`G`kpi2Ngv^2Y$?2w^s41Kmz<@N)@WeIN8>-OTO`@CqmBHW*Q)vlH63 zmyXy2Fw@<7)T3gqyGvjDESok1A<)t!l+!@0xqGukp}{`GU0W;BsD6YYG@2Sq{_L?N zBy>9c>k90Sl7j7G=E{r(dd4qPr9S<7{}{#pS`XlUkCFj-xKR*I7zIozuUrwVp|gO#n$sw9Mu(YzaP;<7+zR zn*U83S}l?*^!e|1pY1f&c>V0PWIL0&!L(^DGg)~G-mzh?Jd}N-R8LR4BmB54%{C?U zsqsN$JsNxDUAtcK+sXILF=?YOS0F6}`EgFip#&$*qSJ^^(&(crC}0ZlMLGDf*tfT3 zo@5_sRJ-SrA>)!&&;d6!x2Fo2gKr*+h7L!-Gz3H|q)+4}2Na=Yp5Uujl?r#7H$14# zZYwdBiLZ%u8C111s@L{SfVh}A6S-(YFKj1mUk=Sa7aryOaqS zpV3>vPslgwR>>HEi1;fcP#&;bicOMH+)P}MNU}_rg!59?zv91oUxT`}%|ddQn+o)P zX>vYbS<7;U4@pkOpp4z_hMPrxGzvtTs2cm!GdV`Wp+4nGOy0UyI-2zv*joiZSseCf|B?%@wll0K7b8Uf=c32_lZ~hx>;U$S4W|H)>x>_ z@|JaaUxc*bx35jL%7G~<+Tp#Gh`h~ZWuvy!w=5x$?*`gPcL2G4z1@-# z)kh=sPB)(oUzvK8sj%9TgoM1&DKfe|^YR(bue`6S;44D!pZ)ryML2Hd5c9>xyppIK zvFcnq_b>l>{RP;w=%20k2i0WdbwG!8HW)zx?jR;dQ0)mtxW3%BRNguaGLCsLH8d8| z@5l7d)@CV`n*}vSp~fYWd5U)Y*2qdIzn^fF`sv+{gwCA2;cFGiL(7}xm38S5m7?w@ zMI}y0V*zZ=WS-Dne^dd?H9G`Hj9)_-q5vRV>foDOm^sME_dLhK>&x>+qcn>hq`lKo zd)Is|3>t-9TfI&vj>=jk=(aMGU+!E#7W3Tex(Wsf zbjp9Y4^feb9z>j-F{>~cQKaGqi1(2%yfg8vM#1EiR80rVLv1g6`irzI$zuav(DHn- zZY2s;WVgk-01d`JqEye)$n@lyLeXxVJT*uZfS~BJ94h2R5^%t*Zq84Ce4fkPt1K_W zU7WuuBS$GKj=KoAp2O# zEg>AVrc$^Ff248=aKmZ4&{IpQ2_eouSmYng5~i+?kV$gMi(xIy^gC~fG)xC-GjatD z#JqP;v-f(t=If6Ha2PKjKOObYYs3*x^i+Y%HsBekj@VF|7v)3{ijulRBKfF0a)pen zoqlz?`06_kb-L_2b38Nz`g40Ap)1&YfnhdIlM~(^I7RdrUvLv(0DwSG;FT;bE>0cr zzLD@2gSv#t>0BgAOBWOkQ;(;nNKjM#cBboDqJh55WX~3`Qz4$qi;Zes{u07^pxAfAl1rOjvqExG4`akx!du#a&&Wa z^NW~<)f0kc*VW23B<13nZcC(L+9puticTIp@x)DDYl84DeedJ;mkWWI3}ZT-rqvwr zS)J%nUpg74xL+MW?QuRP?*BP8uQ(04wRBpw2EjM1OtW<;S30P)1%*s>n&8?(A>vg8 zyS^z>73IpETh3kf?)h=X|G>bp!a5o!E$5v3aMlpt`n%n+4x!N#8!WkM!CJD zSWuW$9j$N~`)4%$$D(bpzre1h-e-~svZ*+T^PYV6{^Q%B0b_^VN6Y2(dshKJ6d&R1IHQ*xBn}Uass_Nf%Q8jj^|VL9 z-weUb!=VjP8w*WumJ)F?sz@3a?mqMJ*I)8OzdrYc^=iIIQrR(G#vQFnFP#7Q55P!7 z?XD^1?LhGv?pc9A_X(COQd#FcdW;~MqS*Ta3VZH&1mQ!&16!i>k~{>=koFJy)w`Mq z>1?utd;o|zH56!E`PS7aS{DW!-+;XqO_!yY8*s-`1koS#G*YE<%Z2<(u@lnf`8e#> z%{7}OLS}jVbQ-bEDHKHOIBZB?BxSm`UeS*EaZ^N43iF*ZAI|NU?u{2a#D{5v9BzelF`Sx?)vPu+z{k22zp1H!)l!cXD@=KijbcxsL{ z(XlXs0nW#Q%tVH$s2Zzd!*Uw(RjQ)SXEn<^9Yc;T&oOwLwHagSKgwV^b?T%F1L$E1 zP$9rb;za9?<|Iiu_VTQ#@Kco`2d5lu{bszt-m`d9$*q8bU7d=Y=t%W4o zyHuXxNmwGRp@E^D^7i`RrVUYxVbF9?gqPL4TaR33*us!9b8n5&%-Tv#Bq5#)&TI13 zfx+wb!Y>qbHM*{=U;TlRIntw(x1ZiWwmQt{=8vlVr|pRf;%IZ>gks~p0(RcqS6tIv zyd{9E*Z|fzfUx;9xVyqSfai%|OFXB<`(J2Aex4HJd?+Ba!=-6>r>`aPK=h7L6xIqj z%$4dS>zwce=INYmSv`u_CPQf(6f<6s1Vn`{$t;qnRRcE?aY_8+Gc9>7zHQ!rZ+6}< zOZz@@)|_(ub3w2w{en{|B(|XryB^3?{)0v;RcsfYvNZ5xo>JCG&3tI4=N)u~jo2BM zc2Z~*mhn;uA&bHbG6&Fyp|UpoZ*>{Z60OK=_$O#c4AmzPhup`HE>55##+bTfn%t=)MGwr~`olf6FG!&OjJz1H$oR9tE2rFAVfaJ-}uQNX2YY#&ylbR)E2h<~$SIu!; zR}`Kd#8~^V%FSt?ldz?#Q_mN&f+hO7bpHWr!J{GCBCqzdcd{%=Q{d5D4WFjB*$a9s z4es$)jJ>jt8dPXVF?%RzLb%#qJJkP05FGBZ4$`$1D1>BN9t~~q)^=hFHYn0Hztfl) zV5g)UF}=rq(BJ1+V_XT_e#n?r)r^s5(TmcRtpY#R_koXRA-=Gdpjn1)YIUw~1~L~L zz&T?S@tq6o9YeG+x-lOZZWuuJyL8HOQ>{{YSpv)yL9#D_ls-={oe4q{Y>w zAmd^-We|=6U?Wm83Yh%4>|}2bP01#EqK5Xb(t|A<&!AnKj`+3tW35S5V|2U6Rb6R4 z@AdP)Cv$}GZn3F3hhGHTF{!I+z_yqMLen~+>xZo|l9g)q8|>%YtJ9~rUV zSbAVoP@|vQO=x_aQ+cFLLg_qJ)eV}0OQp8KkrtrIs7q9W)m@Jj_7>u{yI}R6*x&jW zkyPcGddbuOOJ@IjP?{k2p-1lBd)|%#QpCAMmu0+p!mmj`logs(mL;y9K*@GO{K&A% z%@=3lS+tKq+adbuVmYY&khQTG zIYIQTcn;#@wdcrXTUm%*3)*B*0?;2&Kb2N4$6HV z@BcpU&{;byHF}OB$jObj^f%?-uOKs>V?pka26=zwBO%b*TD|7pApx-u=8?4as8$M8D4Yg1~Vu>+GpvOdol=+W|iVz z{Y~4fKj!7~s{u9jV2u{NyJ30$DH&b^9X?y4ze-j@is4!z-%|C=jz!m;j_|mvXMji$dLWroIXQrCz`Nw7;(p-JAQk3^8`)W#ffP#|&hGl;6r*QRaT$lH=?e z3FrmBDFKB&pDrkF z-n9MTuYZCwCqW6VkKeK#tC=7q--^iLh7m#t-`pPkVI>0tcBY~O;5 zs1;WAR*tyW6Pwc5GhkoDdxB)28KzU(ZSlRCrtqfxxPb}i34)XlxOSX+WPeI|*2%x+ z?N(_MI%|2ae_Cr}!&^-X7Ht1ET{E0 zZ72zL-E~&U795^Oiq*g!giHux(o-RIY!0bxtTAY{1v}vUGo6KB0(B)Rj1~0k_mpoD zEqVIEyh8^%FZ}H!ew-0v*Q|fEcFyHUAnXkfj{ik96vo1@muz?9KhtiyQ%DU@(Bz6g z>Nw`tBC7I{jw;h{;~pkqb2vsgz8j@@%Hd95zCuMLI@XX8-Jcef6>P+dogL5#zYssv zg)^9e^C#3bZ9q>_kJ@089Rm*|TXf!AxBD+!Kgr)w($u{d>xfbHCBeB6KO*+RyWk5p^Q0KBc3>#CtyHUIE6BNouKBwwLLcnVt0jKFUlc@x}g>ALPlC**Km~`I<@v#@%P=W!h(k+z- zzv|OfP*ss9r@-*cxv!=(3Kxz<$t5z9HqNTLKe&?SJ>KbFn z$C3R7lHhUpdp7#-#iH#{p<4Bf|7E`;rw#mg(bPYQvR3JQ<|0KHL3(~0RnN3X!xZu4 z;{|A!3QKAcltI;Mrmy5b*lK6;D!HISwF> zs7Fui&)T2Nc*mZNe*tf_&0A+KnjHUG4ru#rEmc2NznxIO?2>Gh4*RSPvQ9?=JFYZ_ zZb=o~G<2IdhugfDASz5qmH*Obf$X%I-H9o#2Eh0d#Y+Y#c2*2u;`PZ_S|%8N1pWtL zrT2{FZ)4#-J#zaW&nX>1wqt^lV#1%rfj}IIsFn;)FtDR}|Lv=X*q6*3^};mE4CEdc zL%3yL_QrYZzs!@VemFK@foBEFu~u7gOuCml{&OWV*S{i=JTbL>qBx61xh7AMks+SHN)?ebPC-Fy7F?M@Kq13gk-Bhh9_ z^y>Wc;CI)R{l4OV0B7aXs0Q5V&VG6zE>Ywx#l8PXNfK|XFDxdRms}iC9_ceQm*_% zkH3C~#e?zgI5&d9R?r|H@c$S(??Ne+ze~H zeS1)xGU^4N-98mibF%(@&Lgby&ETg|qvBR!NClF&dJh_WOR>;E0Cs3R&P6UyPlSfD zs0Y|W=pFQ9_h*R?L(EgdZ)2@nEVv`7X@gcoZc7_h7FdS{XNq`d7^?dY-4f+XJ31G& zcqq@29!jlTr>{?PvmUHC&dVStez}@uXPHI*Hn2bL={UsTO2Y!_GGXru-Eh{4tEG%q z%5Ia4;+d#CD(tOEV_n4YH`r5?32rs2_aZEJpL-@h>neCpsXl5lq5`T&bs? zsJ{L)sj9Zw^zYww1J0p&G{b`bme=Bo_4d}#g`1Aw5r1!NQS38J!*xuJZQA!39#jL( zyJCs4Y;xD8E?wPtB}xu#^lyK14(y%uohDSgtUU~N%HsN9`N9%|YuL}KCQvUJtt{F6n!o-ySGo$@&9zjRmRdF!a?R`1-mpm^ zg~q@fohpZwo&Yk9YGS>iI`W?{W}6=7WZ~N?9~V2ULrP%>w_2&toR0GMeCjZJn*Pl6 z6MnZn2u|TjQS?JqY>?sd6?uivGUCcX7}ebpmO3MoF_<6Y!T;}UnjzsZrAXSp(!92F zD*pw-r`}cyO%e|W)(@MMCZ_v%Lwm0suEW(}D=qNTeM9z{P-hM_`$fxRW_m;ef zDs$`f`mr3mogk;=i*0Tn=W0@|;{6x+hQ!cMTNRvFF+jfaG5kS-N1ev>iZA@F0cg=U zAxKa!0c_UioN;Q>YhEo!9F>cS^O|7JebIu$Wz!#)<>x<+*X43&v~ISFIy0T6?-0yvG;r6G9_p>d{$-QW(%lUzJ5S%A1fJR(T^hkp5$kD4D)5-IKv= zO+}I6KNNefk;&}WfeH7n1vIwbGaJ%22S|l!YPlFt58~+n_nIkCQW{I&^40%yb>S3x z2BZY>p95oeV_C#=xJX^oo1dj^91eZUvW~aq7rMdU_GVDozE?q+o!x)8AhViBHV?Rl zW|aDhA6OYUetrpfL2s->lG(rC^+W?w-D_PC4xLS&c>Yfl%=Lrd6O)2Lf2rlQ-RxHsgP;)9HpS z)RZb}nQ(|tf2FQqjzD&_wf1y(-)`+OIz4$oJtj4oBW}d`Ed5T|p>;Hq{Kx`p3T{Ha zrMnWLgV6zkyMNtE(E37_wnkA@{XqzInk7_J1?noUk}QmE9Lys@hW29zPFA~^qWz7w z3-$}_T}bwkhE4naZ`aJfx%H*Q9;&xTayG#ppb5@0Czgjkto#Kk3qp~^dhy11Bm61g z(}Zv8mNQX#Jn7|H1u6j>=AhdnBa>dQTHK<;GS@3G~1l^J@*NNIP|;k8+@%cn7~6 z|0!$nq|=iD0eJtPxpN!ReoZ9hHoLv9VxM~>dwV@mPb4lvbw|V6DJrV;PKe-5*Vr`S zML@4`V@b-KqrD6+LZok@pn`T%-geV?ETX!pITwq~aq03AbN)083k|8l?g1?!&w4E= zZqISM(NJ_ta|L2s5AS<ifCLi&&5%L90(%X&c69n90h{KJ{Atz-1h@6s(fi;#K1zCDzZtt2 zKHEMVRN}SBojp9WH~WoW2uj16SVd~sUi77{z5bI`ndbB6!D` zM0x=-zI9BYGM;=L`QvZ7CM5DR)@qg$#0VCSx}k_RkIPV2(^fNyfgZnSoa4z97Zczu z8y#SD9~a5sNY?~>dbs+`SN-pMrozUfmlU!w#t1F}1t3RE24@&|z+!rtG5+;U*dgX% z1G{Olvg>ZBnvK;zB18mV^*(SYT&?m*5y0gdt7JD^4U=V%gnSb^ta(hQr_Xu9y2%D! z8*~zJac~zXAjS1Qj(+&L_!N*a<;zR;P+p_Au`4%^A(Cb`@w1Quk8?J81GuH@6l46L zT81)VK)sLn?X7{vhuXmhjry)Lzvd|g&-f+%d6Q$lMlw_gqnu|*c{!4_;C+T}?ganjWxBEp*tAv6f9Jq&q^ zCmV1ezl|Hct0&mm*_~}YrU?Dgx1y6vYE|OZvKUY^p+Lwo1Gw-HECecEHGsP};Wv&; zCzs5IGloJno=1_VO%cMsq?$jFQ2$zjJrSVDWum%xGGr?ttzJjdE>>mMSlr72H{r=v z)}1c8-O0u-tIOJoiSBca<~lKxE&VyCGTDG?alQ|6fy}I@JtF`iF}z(cY7a(UwJmqR#owB5mW?t=dXTBXuE8d%$y6m zUHNnJhtBbQ{X3Ae7NwkHH|9N4w2~MAAS;5aNtsGtFY7?av`+bEGQ1I5t;MpyNrh1e z{K+hY%L6tou)F~*QxYf1YdkqOWuck(*SW~_113+TVu~vl-b;q z4`#%KYC;#Tf49w7qR@u+%@3s0(r{Ng#u{Pd5I^zOhRn2ab6H7EKFV>2Ub+WTzm#qt zd-l_3eUg7rIn!k&y8c8=!la&8D&wDSZAc^9kc27gL{u3G9&s4by;A zxSZ>ZR8yQUIN2$uxP%j-KDyj&y67qgT+iK-lMKqjFEohGs^cPWaBcQdn9b@@cq6}s z@Yy`BFPC^uL`Sr4-J!dahi-z&PhGS~Vr*5I8KeA=Mtb74PnZS}dLrtJeJF|QQKr=U zUZG@y>C~C~|)V5#M!p*43qnWNmj zi=k{;r(vBQ{W8ZB%BH9$D1&hoTWfC-2h8yyclo3evlk$}J-PNF=PM2!mJ8E~W?BBZ zhTLz$ZLd!{J{np~h92?8g6<$0B6us~Vc*!=>q*^W4EHfHS%_Nzmd2 z;q?XD0~fwNRk2Pw!WS&#=8#y?lLEFM)2)8%3JETx=8efJ(3Gc0+}|A|;yTrOdv40D z58RaI14wURx{wU<7_a|watH9Uc+nnfGu-;PqmEr;&m&cwRQzQ^8zMQWIX+?cy2t6R zsaEZrDl}y!R+&RmVH_Mr*bGn}3Z;|k-hz#w z{EWK~<04Rf1@UBGLT8{E7%*!UyEd!4P^Q16>S(XHj6K7g^&?Yv5x4$?<6GV7;vm#_ET36kuX zIfq=l4P$(Eynq-I{Ocr6hSvcM*eG*Dn#7cKOp{+IVK6JY%X5F)t$Ij;^sQ;cx`1 zyLK2EO73MMnc=`1Sg+ojhRQ~Sy6p(kd|Dm;wS`?o4GV1RHRmUhs}QH}Fb<+$cX{B( zfo5GEg4C6oX|?zb(cH;<}nM!Iefn=5U$Yr8x zu1jfa7gLcioeLdfsmw#Vr68@(e^r%M`qmYVtNsShA?R$G7y7P4S`8gN%Z;t7o3WXc zm_^qhIH$90J_wUoW^wCWw|C;6>%CiM8Wvm(=VqTib#_=cEAS2q`Xzs9&SgrY& zC^>`bPRV%$GGxza9A+9c=FS!GVG9JB1L$VP+gGKmNdrifo4j7~<6BbPj3?1Xx{?+5 zWppMelUt;Ar|O_sjr0CV2t)SyDqx?=jdrUo?`MfQp_7JM8T%P>!ZLdA)mDafR9V6) zgFL@kMTfFDVB;1a%8x!rrMyZQvC`Kca%3+RaTwc>l|_JGw8EMc=lehWp{Z{j&75m6 z2xY=N`vj3Fd)un!{P@rC8&;mo_xS-ams$;eKAsOqK4ZAF8}Y;sM9-(jvg2?6zBq4v z_E?lG@sPu;QOT+WZ(p@OGwsc2mf=eF*6>pQSn{##VHXher(blcG35oLv)l=SOrqVN zq|9e1WD>W0?e6JW(r{JH_z->GIAFc|{y}?x@%lUrS&QEvm2odY&hrhA`QMwMe4Z z8Xu)L@SluBT)IYff5+JTGe+I8GGKzBj%pW2O(jV9gzcp&N5QKmn5lpTD&dW|Pt3iQ zq?|s-9Q-Mzrw`lXtzTQxl5Ar)0cQ)9 z?`@pl7Gs=1K!#YCjb!z2kgq~?>7l-@pv#MXcSw~lYI_5Wam)q3xSeL~r)6J&|E`f9| z45flESXy5`R_!=Oyr}E&$>aBy!Y=K+!z0`r%FY%P zsxyLIqRV`}!laOq-090(sldW<9&_1en_`GO+%!!;W>|)N8F1H8P-7)3hyvn+MOB=% z09*caF9{ETHk-d9{^G)4V0(u$|44%m{@X+&mv3^K3jLxe86s-+O$KUa6O}Dv2E}D( z=3fEGmBr)y$5iixRv3KhNmK`43gNqlF-qd`4AEDQ{OiW-G9ag}UAKZc@rA;E(p1lB zCC6q(2)IK7S;l&@D_LETMOHA+h-c3_4{}kjr7_Hiv(s!2&6Lma7OTVgN!3+9`&U{; zMJVcb-$g{|5h3LItJyBh_ZO--cXC~+x8sTfB>_2hP3^(#UomRispz>11t7VG-shD( zkw-+G*cSMsz3BxF*p99L&9lIwsh>_z{_IQzXmJhx`)zJ;&?qg~wrDHsl4kox$V|2= zKB$u<(th4AfBy}zA5W@~nR;e$7kfGhOvF?6j0wtt-ZDhr;yX2?K0}u9{yTV57_}LE zjGDK__S>)Om|U8#^!kMT`h|w}zU()HoZzO5o800vcg-11_|tOzubx0U*%tIVxpqqN zDn6A+v#TRtM81a`1i3r9&;k&hFk{x8wn?0&--+wy=2~ieZqnr?_4W=!-|n!LTxw1# z;EAUSzj-t**Pv~&jFR_?X=eyAL@|+b-b6mVXiM7slFG<tqE_h_eq>y{82AiF_^GxgU4q`Ch^FBJfoZ^EOX6U1O|}`&aC6PaK)Xw=E<{I zs8DA*jHvJ7)N1hl;WhvJ)4IR`W>s=(>OdRNc0`(n7cz|h*_cSdJ2xNUJA2~#TM2y3 zbQ=)3Hu=U{rj)A;kHOt3O8E*gkF`BElKw`yCFUzF_o2VIirdftj<)*sCJK1~v$A|9 z@Y}3k-YR5y!qK9}2<>fu07sk24&w|!Z#>6SD?=(~^U7d}wqPnF{OeAhXd@-oXAC~P zsLD@5Tb33cj~{pG?9+r!TSsd$5O;SNSz_3s-e@9pbouj|pykVC>z?C%gA|M_J;QJF)nON{Zp=J;0I8hkx`UCdD^BJR@>GZw%&Z4 zwmxvf{$2nm%vy2B;<3Crn{}J$`q#LUu6p7`5WSW=Z+jU(>xNeFMa}2YVF^)7EbZqX zs<7i;aSP!|8dRrG4*k!(Q$DnceV)*N$|^*hP)7%VOlA8D5qTE6aM54AygWHqlQk-v zy#CP9aviw>QM&@2o<%4o)==|WorEmdVYL(Ot;Q|WS@2H#2n*XMnR6mVNTI~s=AQ?N z3^q^4Yg=;S^R?)CCMHyLpJ!rDpXxOyv-A2+>`W!TzFY9CSb6e0%97^I%n$!_w)oY#)7l2D zWSirlPt>~AZN)$PHRN*DT!LIlT5sI>^`C|k?{n$(H@s(8zflvfGm=xY41#eMkr7gF zMk4RFb|Xein@YkO`O^7jco4#iDp#TRb8j%FiF|An#TZQv>=4Bk3)3H^w-)QBh;oIR@tkwY|p3EY%s=ib-AAzs78Oz9|f6wNY#M zTZZtl1`0wNO^X!8btju0h3L37g4It=(RK9;kQ(d;YC4{LLMH1XUfavnwWVjk4nfM( z{hCwe9+Yc3X0aZ6%^zzpCk9F!C9G%fAlU}$nJCa2w@lE1AineU-AzV4}8mRK%?&9z` zqn}ToWsXvIB_?iT6>!1n8pVR0$D@%Zh)$G+{MCwVMUGsKr06oJI~brs^^3 zEc3;do?W4XVreO~JE?3YOZf9EV$z)JG+lL<^Wn=NK6fPt@l_4G;VTgSXpv3AB5rnyOK zYNi`j?>f>rN>o3QYRP%}F_S-#f_G|M88_k5FU$9Ob;LB$HFFu_#y%#9Kl?jTyh=IQ z6Jr|Y=7G=OFN#;LHx2mE)`z<|{cv!-mZvjmB;UQ**G#Iz(Ox168a7CSVw==E`jTax ze^0(T-R4@mRN_6wwVb;az1%4n?@Um>HydG4=O$jRavyv$LBq?A_z!Ra?|&$Rtstqi z97(&MT}>4O?(aOd|H|pk6Kl>}THg_XfUi{s2+GstXNBf_(S=I$)DFI<)mtyoI55#s zx{LA}mCH4n{TK0jv!V4(+8h>|bG8QSNMAX+ma>?BnfxSzElt$Un6VG;nma?pW8mDO z_Y3{C`504+s7;Yw;X~d%UC$L?yZT1cDRk)&CFnu< zVIyPH(;_7%i{K8GN(`gWy~d33>&ED3porR3uSIyKkW>DKc?}=68(ub~xRh+LR+)G% zVGzl_Se+8#<9XzlJz0Rr$!)@&mS8S7=Ni8B5N)xJ>qEDXyqg&Wo2gx`(o*~ZyBs12 zf7(+>aBngxC}7~JW)Em%Rd>N z+GRq$%lCIDGvgJf<+fgB56Rzx1}D_QaII z_umIqyT97Q>Uf4#q)UQ0pPK*dhCm>o}#sW?D2vyTdR6) z;-36`?c+Wt3vpU?MnrhdIJLy8`BaL+G z*Cg~adGrL^+%}GpMsZF2jFd3vpMt;!y%PBI@?pzmUU{y4v>gqB{>THW&$q zxu*VHKU1N_owt8Wyg^XQm~ziB3n8}f@;XXR?1u<03Q8IMI_GzgGZK(?m;#jWoHg{E zJwQ6j*sOxb$xos!fJRfBQGwxYSM+d+{d7i{=ATkGPZ5N|cP&XHipX(4ze+I6;P8SH zvL4M^>APl0mZ_uANf%?b4!v3TMhlIj)-IMe!fm7+4Hb2MD2G_%DrtHXcun<+rsk{AeXg4@Ka8WxK4e8FCLvNRJE8LU*sEi@*uF zz{GSdH3;`BCveN;Zgg2-!0_audgl6rrQnX@2;sA9u1RUj_x6e_@Ragj-J~z?UV`Ny ziuXNwA@FDK6h6*~2ZlC;s8Ee6b9Yn38l~Y^z9%SKO(rd21%YJW0t4%3hqH>uRCala zhyi)`zKo~`2|jIJNW^Zw7)Cq-GKK-ipRf9JbM5FG*bofzs=oS`;`$5{w{j=Bw$eK1 zL<8x@>iG45BPx$AeYE@f2@A70zr)Fb-GaW7-fM5d8_@p%RK>eZ)3-R#Eyh%hhLQ$6 z2t{35NB-xMhow1Em;O2`S1-IO4OCda4$?k5P0(}GwO)mLxOQ%lWUVQ(zvs))x6a-Y zX$Xkqw%j?3UfRk_8?wL}zG&M0+N&0mF_=ngq`aD<)8tL%tNT-3#@YV#xAA&Fx>YfX zI$mZ&5$cmr+?k`q^=~L}DSN%jRJnce#kC`}_l@CiOOACrX!jVq{-Rz0t&K(Do_(K_ z9pbrwaZhQ{hg_86oX`T)L5y@5vy_#mr5K|%XF_olALCLr$%}B^LHCRczMWAcB)NldV0<>15||cM)6;3 zyT(dP*S#j;1S6oiTLMpx*)0Iqzc~Rr?`80w#6g*5 zviB-+9QTFFdEox*SkhQcq_PRyIG7@pWpov1PY9mcXlhcB#kzeSw`Q)_ou)74Cw7hE zu@99t|J|E;jb05jOWsd7#6FVvd->wVClcrJ`ire(uG1#`(WEE~V|7}U2n#BFy-h{1yFQ*i=)Z3a(S>AOQ zt>6Frc1XTlap=9+Lnasg*Vr9?VSjvzKJouxo_yg`i1;K3VWrz^18?1$wQ{Di5@)9M zoIQ3$GJ!*fu_G$QO{7QOE*lT<{kz3%z9i63YwL&7=aX*)El?lM+vc5qU#fTwAP;hAU2VV#G!!*th$QVRO^b zQS}3H<;|R{Ki5))mggahONb{mNql+^7s_SHA;Ih9ihCyWf8KeiG5<-8>x}NnVAAAE zj<6A0Q!t>^Pa^!=1y4(~(cm$;LHCPu69m1)wY37gVJ<@1YsRwF4?H_y&GvK?%31fN zzw9Xc#${-^@J7_~no)Ok=C~OIL&~G^lFmx~GqJ0da7}FG%lji}cJO-hzrQ7zrc1xx z_V0;3?mup3uaEm>MmDQ2TLTr9*w_0CUrq$`C+Z6NX8x&?9{z2LI-O5W%4%2Np=Qc&(C?X7`FXB1a3R}0tPe&;zvQ`m+9Q4@)(dyjZ#XijMMR>~c ziXvnsxfQecX5lZ}x&-`fXD8hQ)W%eWN&vlH-OXyz?I#&qb@%|{pJ?{8;WP1vO`kf4 ze^mi!23_B0cW5|UNW;IO3nWnOSGS)lHm?*0^wMwTU^Qv`N8&P! z`B+W=mWWlfp{$D_3dfe6L4u>91Z&q9k$cW_PJP1%s^K~{LMs#fO2po#{zz?l)5Sta z=!plmHDps8{&7!rL(kK)8{L8MY9iQRBisQ!5!Q8|eoBx4 zztuAJwhY~0JRgNOAkEsC>IRL zuLkCdN4G^6gs)m>!DvB%$T*!K4U%#RGKW>(IJv^XP&o9|{DUR?LE)yC8~q8_$BTYM z+e$MT$>e`f{ApUgpxze%g~#HO*^^W!Rx>R|)%e|$U=VZ138fE%n2C+jZMSiefC*9RpjaiIyTPf;zKfI3f?!%4 z!rx;T?&k*n67-s^-6@-8R$}5mn}Pf@@bqyswruKaI&NIBf*qr9Q@TDI8{s(nC&x#- zBa$6+oSmA(iNFQ93={O4S=hP!TpRD)bHgA}Qr zWz-QKm|T+^&T?Gyv6_A!1nj1qm*SEe!kg?!z>KY3v2RW(acM z*_~3p==gJnRqCQ`SNlnXbcxY26z9+ct9#p^vSwiQUsc4~@0%O=g0`}I@&-d*ldDe+ zpM+x8iJa&TAlHfXrn}aekzdPX1@m{c_V*a32(ZeX^9D{4)3%~ty}mf4;g5o{s~d(3 zVq70iS4=-`)3D|*B2MaPVdYSjJ3#(L)FsAlP1p}b74^!_rQT<1~CHOTVqi9zd zH>+$X9cVa8bmaWFJh^-paYBgt80cMH0q=I0Bla~c9lSA=fP}j*!Pyu9+uwj7v<*Gv zLmmS6MTU%K@@UBa)})*Z!*cRTV^z!IDqdGI&Nl|Rr8I4GK6D#P>p{-|;|VRWjAdA3 z_5GH$@2{^{U&)CtaWfhaonn#IrHl=X@;Vg+e=!=3m}*GOuSl>_kUQsK$(!8DctXB) z+o95Z{IO9{bNR21E<}x)i=*ybj%s!u2-y{Vn|7t2n>>rD8(;l1(2@k;M1N zTPV4W%KATQS;$a&r!p5W;*b_77s_EK)Jgj%ZA~o{JxGq0Hl6G5GLF8yANEL`z>Jc6o@TDy4JQOwnvjSxwD-9>%gWiv=W zqRDF|WCVTSvN#J7c;uSD+G@-p7iLsg$w-rXk9l)_7cK~dhtZ4oSbryu~Y zbRR<6UI=w{vh+M3+t*Rg%b+HHWqeDZyEz%1iINq`-=pJQiK7B4WJ#cVF`sH)T}4xpS8o5UukS5C^DJNN zyS_S)VhB$YPdi5k`&nH-T7phwI@m%Z6M`dDM6&f-?2QWld0jW$5lfAW?Hb&nEGnAw z3-X~_o#|9i^Z_uT1EWX4bQ=FTRREX=5Yyp(Zb-G_r#ST}7yYL#S7u?=^17#kThzj& zkqsp4m`kWX%<3!2y~FmzcYf~Phy&TPSWa_yYcc*;W*jVJIUaY4%dRUwzS6!MjFeZ@eZaB~!A zu7tq{dnE^1FE;H`j;vp=vDwISG`b<(?T}ctSgrL`7*p{M_hIA5<`s1D-|)lJ@WwqPBjQ#jvvCR>81i$RU08}%k zsI24MjHgZtq`fX~|^AWuY;sD_*V~!Q^*J79+jk%?IFaQvP5OKHjaLD}h-G z3tg+){P-b~wQA3ys<`Rb&&EeSX;Jfx&8}3l)Ygnn@mPOmBWA6K@_nxlU^dMHP7E02ZkQPB!d{}9*9%V2j)S(gY1=6D+#7?F z?v==HhCqRW-7Mb0%urQo9fZ_61!F7vIa)|l{7zQ;-N>s~Ti`VKV!Uxq1^ucKxHA3f z^M~T-Egor3J%Z&fbX`~NMHcv=IMtZNWRik*&=Z!#MLTO0MC^c^Jik#hlMgc+QPt+r z2B0-(TZ&7+xn6PAZM~j0dSw*-02s6pb!0W)+#k8nZM*4jv14S2u|Y@t-Z=bcG-uHM z!N3~9=28%$hD)NvQKoWIC_a5>C%+F$o3v(4P$SZcSINMg3bw;g5GmX7cIoBq#gMVd zx~r8=KF?4S!0|xu=}ty_%Mb(9PtUc(PHj>jDaB3A<$Jmx%e=T!rG^=8Sr_W<9ma7I zXDQ8?b=G^_SCia4JSe}l1|aU%jY8^zj~Y-*q>PInEL`pPm%tZze8Ji^JXg8_U2=_C zM;cb9RJ44hWQX#*u_Yq4EybTtk8J?4nh!Nh0&fYZK@B8)M5`HB_?w^4W(?5r>8+m2 z`pe?(05S?m3bISOT8x(*->bVE7Oz-pbr@4^h?<%qU^-rf^h%3^+sc4+Ws8yND7T+8 zY?Mi%R8D~BsaJwGV}kKpEV~VXU=*f5Hxk`nyx3QmKWi&wN<1XF0Z$@e^^tzN*K#hd zj2pI^b~^d52(SvbLc!-i7aji^@gXj*3Da&)(KGgg%llqXE9;s`xRY$DwB4!o*UV^+ zXdNTH>KCT%+X{?L4=Mz12~*>b>4fMp(sZ-m(D#FLi#o(S<#kppr^Fj)g18B#{I1z8 z^y8HjXe$58*y@K}unKXXg1ZehAx%p{D@G{>zD>Pnp|m;{c48RR63%^H19t#Z&v#DE zaG0P(k{^V`TdX3$tch4^fJ+%v@|JxKfN8XMlGfSB+qcq3=0Rnin2MUUM^juq888kCrmz10PW)~{f!ZyrX>ml^9 zRmBr{27c2~^R$X;ImC!U4K1Wgi&W5pYB)MtTl?-!C8j3jdL;!u&i%OF)P6U}(^C*T z>52@q)XsaNfDt~+j@+7loZw_kQ$y)~UK-<;zgO^^&!w_=jAU(XvyPFkvqF^EC#H)}^pX;aFJ2zcZkxh%}tiz7)r-|Q?*QcCF>-@Xy zdr!A<-sefqNM)&>9C{1pPr)s!NDmVKvbQJYmeCU$l$Z93OuPg0rTrfw4eV0k?p z&ydc+HkR3$Tu;9kfUxZ)9aPYBsLLxBVjLwlzs|b((L!kJ7vYm|@+4qaDH+@U zyH%Uw^Pwe4@HsrbIrii;qvGp*{!sE)miI@Z0WS-R{OCuI*p)LuSd87S8`vrU|2SSE zb^lD-3dxbpyMX&?UwqyEF)F2MX);{^R%d%;fF`a?;O3fEvfK>DVU_;_Xx;D_uk!Wu>-or9<~$g_Yc8Qejhok&Qw=?LZ1N1xrzY>%bJGnKmAyeGv) zUsc_EC3;d*82Tn^9cSPER9oAYE&W=;$9>ZFy^xE zX=6LHVm}4&sHlybF&mQw2Bei*P?kg(=&@}KyF(Gwd~|$+ILfxe#%>^ z8rAet)foCNUk9I}h+uEg+Od~SO~>6Ip?!0n^Jl?+cW+F<8F=Uj!}Lh$ODggKt7Zk{&R+_dU$+jZ@@Gk=FXhYuA0U4l^G%FEh;A1yzw#a{Phk+`8h(?&`*ao zrwjbO1iiUv|N75$N8oywh)*wX&Svm`N7esAw2vTGCV55+qlnjWkGx%#OzuQ&J#Lxh zXda3iqyS*O;ZSSx4Jfowc8W_=KbFYa#&lU2z_sBYsDKD$@)pV(Z*PMvY;ZgGd~OD% z<)??l;uV;+p4|(K6xS``59Wy>VBYxv%@lz0de*gMEl9>mt7;Az?)b+QO1x&vH;2a; zR|lI8#Sk+i=$RYxvLgjF-)+c|xY?8XwBF=+iv8Sw7#ym= zhL1D6y}{+$iSF)5)6ZXG)5-f}2@)oE<&Kb3C&vY4S~adW@4VGRE}LRA0$3Q&XvN1{ zir8?vkE!WYKnXKKosj~wYLb{n%y^OuRbI!B!b&*pRpRHLmxDC1FyoE3&5rl52meSz zffxLCby-`aTKEC1$#db-tEKyN_NP=+w%o=UyD_Up$&*WSfV`Gz#>5<7ET^nMJJLO! z+w+lI`tG}IZJ+j_t|3t;t5@R8cldaRC~DDecOkbW@~aY|4dUW-cMW4SuVzE)7)R1& z$wq_P>6@x2vo=tKU4%Y^_?W+-RhUu&w;-)errrck5JuhE(9=r@ZE%QQo|iMnUi3zjl-(v2EV0;r%N%v5HsS*CYb*X4(WE4=u%Ttck_>l2iH>ZyX+t|tC+81a=)lLBy%!51Oyl;N!AnUhLtD5?W@C(d$;8J z=KLRX`W8`nGEP1RY#{(nbYq+$+T;|~xtTTIf(fs_^tk!j{|_MHCh@xB6X^CnnO@`) zhM^ssjNAfJ20$CaBn7x~q94;T9%Q8z?N>ZEeOp%9i0+s{E%t9>=X{zF4Ms=#P{Z(F z^9PF-&D~JwY7k!ZEBsrV-o=Jm-b!U=H`AoN6?eWuJ6tea+Gsx67&XXP$I(&F(GFpB zudqx44i0khC%2gWkpqpJpBH?5>1c}X_}ZRVngA(cnP+n&9Y)lyTrtHrP%pkS6P1Q2 z_o8wggtFPN9FpZ{Y(qQM5ZmLr4J*FrGR^za+K!SHl~Bk1gM*J(zxUfYwLwiauwVx( zpEr&Ve1O|r*!yVhkVL5^C19nKfG}9RvF!a+iO%9c)DsxJ;uJ^n#`Yd{EJWBmr^=Ix zgAGh#h{Z6Q6oouYQtUu0p%=tt@e&i%%ih4d=4OGj(XanGp4|$HP01o|*KUruN)3Zc zUO@)5k#8Q}RdkH`U*|M6%wsLH+rz`wl$x>F z%axBt9f$}&)oZack*Cv5j6qy$6OIs;El(PndGsj=p*eV)Ti76pQ0#f5adZ^4r|25+ za**lo77ZhWYkWjoC_6N?ju#ILQf&Kl%ZuKS#ep*2oaIo{YFw~rr;i~*R|dR$C6ZPd zkK6@wv=$ZTljYkd(s%MwZ$}wcA1Ld{Z2!UZ{cAe=7?Ct<8MyA)dh>fFlmIITS-z=U zz&~7S@v4YbzLb1s4suffFn?CT@o(r6CTJ+(PRSd;-Ps^Pih+8GG3Db3Z=hVO3WKNj zZtDBqSbw8ET@TkKUk63B4OaWr&gPJOA2GCkH$w>wjr)`%)43qjmFl3+CnVT+4maC& zY7>)a7pEG;Yr^+w`2BG$ZxOBL%jnx0QM;g<$ z?!)1;hh3svpVDglq+(pw}a-urBa_74F3O(iTYwX@D61#d#Vyi0& z?wZNdFyV6-@zQ>7ZNiplkL+>?(9^DCvs)=H_b8Qz{0Vi6iy|R`p{BFlr}upH2OZ-q zd9B?YiXygE;8j4(zyoee?`8}xs#`*&y8Ga&n(b1G;V8n!{$*(IB~+>UeJHBgrooY} z8uYhYXBde*=bD%W-?rnSIQOIoWk5u%$c9~9%-idKqK)}Gv#4U}M$b7}JWGQd1N|vI zq5L`g$@>&OxAyudlM%C~J0a=R1|Klv7NpjYD?WUZ0}ujtpk4`B6Je^Sg?2aY;CpK z-}A*fPBGemvp(c~%xmPz#6%ICw86^pCylUMhzp5)g5Hk18()mv#yddl;fVYz(?u`x zeXPU3pUnd2(39lMCmc>^0M== zspR>e|G=%{cc3`SYJ5R}cavY!&DnW)LvOgvCPeYX)celb(cwqf%iceY@d-ri>7LT? z!oMrNB=r@_%ud4K)=tqWZomb?gmlZm&(Qk#&#&Q#4lbEvlBF6;far#}qF4^<^(-R5 z330t*-YuWgQvxstif}^$OB-Yv<(UDXoLMb9s`2DsDb^&mI72>{0mqy~i%{V`m$)JT z>-*_Q`OP7V8E*}k{f&L_xfKaMbgM(5x(;Ou&YhfaJUx4(V;yRc5&#Pz^p1?;lHkLc zw^_H8DQmB^t^zA)ybrZCCfz%GBBoxZIZ5nG-d@P(n&?t-7uN|&P{I9C>&S62SQG5X z!`n6rb9btCi*-`TAC2G(_UrFjMKVZylxH zx@sMq!dHP5Rcg%H{vTj=274o8aaHB8Fe&$TWxt{Ce}Eq{Fv%<|)<7~mHP*^%_*)s* z{VsD#cOFiFxI`$32E_MI(7^L^V&77LD#xqlqMAH_i^=P>b84^oPj`wsXZ0TFTwC83 zo7`LRZA9owGFI*xLHD*qX~_Inl?Q2rb3PVqS{C)zb{V|$5-d_KO!adzPZgW;bizWg z%!aD2bg zA%jQTw;Ou>IYXh)_80TZ{5Dv+6XID%`sJqm@8#W}5ZZ_lNn|LS-S{?FQ-MzKIT_aI z8-hl}o6paZ z5g|p?n_Q#ZHB3#!^$S&OvL56y|q)cY!tXrfBsmS-#pDtvJvbH z7>Twkt^{k^A-R;Kn}41AehRA#6R?{Q$%fzK2USqT00$gD1YEuQ^k~=~;t7j%Lta0I z-cEYm<(Mdn8 z$u(j1P-_cW@?|I>(obgX-}1%J#stw*s!^8svGtoVPp)6u4*`4~9iPq@r*u2BCO$u- zhwQNnoUlgg&Y)fqv~UAgUW?IPK2PqIb%WyWnd{kX`?_c|p?6)2+_uiq=~yW%)sWts z>h8!*Db?im+%|&k(aY=0P3Y^tFTc#cTG%ifF*o2yzs+LMX;v2P)H!Rfg$GCLn$guW z1QSWTI8ff+YgmC|a^-6nV=4hW zzkSi_7GBb9+e;^>E=dT%Hn-%-1vL`ibga0Bf7HCVljmphpa|#q(qFZQ3u_np*fSsO z;kV~&iy_9AACICMPD4-y+-i2Y-WL4bEv$bLhewlaT?}m@Kg@GohA31HzFekDr0n$| zw>%UC_xBQ>@y<8v(px(h8{4t%l2pt_cN2#V6{Xqn%F`>}=aR>Fb@;c3L?`CB+wjhJ zj?qTKPy=f|q%gi;f*fGq$`VKWu`5B(oklQ&dVT4jPI$4eB>b$pVC^OD3F{ zxOxH_ODT#6I6Wk9QIwQA8)|WGehBqpn(2Bsg_a5OQ4`PCaK|_DkV9xfzGY$3^BNF1 zy@HSbr|3K!ss6t|evRye%&y9~!sVJ}WP~y<8QENWuWOIYxS`UuF0Ramz4yJiX7;tR zxz{za5|^U*NcsNm?~l0m^*-l3pN~iW=cg^;fe>G@u+oVrSqZm=>uE$k?bDR0%asDkssW%xM7(&e zD+#zM@zs0v4gB64v`&iBPoRX%nk#+}f(TUKx4aSDTSr3|eB>^;$k**CeMhMK=~FG0Nc{Z!4}d9-yXv zRtHxJs4_Lx|B^a;a?tCKSwDi_1CVMUbz9u;lv0#Mve|bfDRRsvRkRb;80Dp&Tx^w* zL`B|-F;JC4h$e3v2)>E5u)Z>pDFJbf3O1@cn-#ovcIE1ByISBFlZUD#U&J|C?OqNQ z&?`*^23m7hhVhkP^I=uH=pDPRS%Y3YCc_m0hTD;M8Vl*5y{P* zj0Rb6zi}nrdeVOmZWO53qMN1%29{(#`_2=raqw`!SJ0ewM*ixIi=jF=!t=Q0ZsFaW zAFS$TJS*=_N$qN7LqshRijtgAsbN~X-bEuoHqjLe)KEEr!}4uG$E04ztR>4cD>9RLEcQ3ydm&2XPEITd(<5-7t}1C z>OPY9T9B&m?tNr}e3z7Z?f3#L|7~wkum3?poH(2R=c@@s4%(K^GCCazH$B1O~-@S(M zu(QWV=C1`phxd0ziw29#>4#mj%>RLdY5Ko!iO#5;9Oo4cxNL}XC> zEOQseJFlllk{YIGiQoc3ixtSD({%4cGYnlN1E{<@;B^I&!Ad%0KcKpHohN8$oMVNjJex7mI7ib+bprgyw^E)(i? zX%Dejx6f%?hy&%@E#3RQ1A~nb_;o-L7vtiopp9d@qP{=|C(b>A`P%a@1fv$HT5m>E z1wr_A^XO(sZy?pho;M3li>lCRKH{!dPfPx+xPQ^#?nGl`dr0d`9Dq#ZCA5=TGf3LD zmy#~lhyLcPqqV1kIu=y#6d3T-+eIT2eN*)3+U}8tqFl!+;WLmZ$_-%QTf@if58b6X z7XFTgwYl(z*z%>$?iu@|(zce=e1Z@rQx!B}9!J(#W!MWM=&ngp*$>1~$qj(48sNe} zW6|JbLE<@zNmQkGx)rRSZh6Z|uGN4vu{k-`6#zvBn*c70#(J;?wO zo!yh-c}23_`TpyFnn7C_lK*1m=wr)?@X5A}Y61TCTQqBl7lT42i^djxkg8L{?bX>8 z>&a-p4CV|cuc5Ymho=P~k%VoUHmE=2Hk-yLDJi2x@87?D_ZGsAmuDN~t=H6RlGgZ*Ur%D|NkU*x%x)GpZK7xhv#FCLt8R+S2`?@X# zUJfp!oy)Qx=BFkjY7K;4AN@X1b27ggDn4aU99IP|h zDhTm0FwTNEoPg61Rb^6Zo6ENNca!u;J@q?|rpjz(E$SYr_PAvBJeFsNs8!jOcD6PQ z{BLGcoA@ftq+oa=gNf5^nTqO$O2UOMv|jQn3;-%V{EWmmKm907OgQ-<03YkepY{y^ zjD!&{8mIZAQl|ZH?s@f_fDGe5g`_SkZAdMg0w}=pK`GDX!bz?ESQ1+K96q-pFSk7a5^wOr+ijgi%r&317& z1Fuk1;%yb3Td=+8(98b;xWPHVs&xvjz?S@CY1;;q!W{hpMY=W{a)$!gjh)G%MY^&m z3eboKj(r2jjR*-13TA$KgHag1^F+^v?eB+QF~3jmeZQF?n>7O35w zguzp^nY|F*%WF?tA4o|-`QaYWqt&50F(l%Go7A_y`%O`Qn8FUo;AA%hLv z&F|(;@X_jrg5HT7!*x{M->orUul*vYmF^TD)u;VM=4P29Sjk z2ev!f&h5IiLq+Smv^DR$G*~M9sRRJ(r~dh~YjC-v+ggqap*EI6+*+!?RjkI5dNdmx zH)%jS12X&17%Z(zyi0RRpQAVHw*+G<2Rfvz-IS*%M}lSa^EUZ}qz zy%O0d6peOtJn@E|cGtF%COMbT&BET5n5)p{vHPR002WqYJf(9kimEw4V9{CH<=VbU zooOz#nwgfyk~do=h;KwKUzN+c5>P4TKMdpuXSN8+zyb9-%9y3 z=;;O1BhKEs&q5&NuX0)&Afr^yx^}1XU0&h1IJ+j^yU{>%Bfqb%oj>d(PxBWxJfwZTQ%@^my}UgYim1 zF9o^E=UA!X2zPL4$dZ$E``#+YD^4WOvw>j5>SLjYi7&(+!VWcpXA3b4^ZmJM|^(V;+2Nc64USOhma^UYK#CHt;ec#2D ziLTL7f_qqcBDUUb8jFq6ew5LV|G+^iBm9u}7usLoWj$ z1P$lBGjg`ts^mnE$;K~`5|e%wa);9`$i~9dI0qNv9fvuld}$N*u`%^!(w;osf6p(% zW@;dJ&L6uy_GrHE9hi&$P$ecLx+gs@AM#hfq8Z6y^u=d@H$WhHZFXa|HR%baX;qh_ z`FgMVb6Pm)cDwR0o#r42(jN^g<4`!Po&Y`1v{~GimUl{uxQVn~Wp;@%bNMb7Am%h{CM>BQ-_r-|Ny7 z2x|wppg=y4yzVP_1H#JNf_1`1E?h?B6^M2O&(P+}WccC+YH`NFcI0kW$XSGCzHYxn zJMu=3&f+jY06-;=T&B@v@7uXp>H3CdTZ!Luk^hw(5@OrJ4={BSuur}9 zWr0h!nWuF@OvEa^Pd1q_`lD+hv)b1)zj7Q?0{`1SYx2=^G+L6oz4ajf%Yv zgLgrUGgR>W*!^=AlMLPSWn~s~JIA*0($xL1F+x~Lt-S&s3uq@Zme{!$|O&yVaCz*R21)QJeL6moE_ zRrKwl&L44>#oIm+rqD1oBs;h6NuB`54ah5L0T!m;qo8;|$Ty#yq2DFE@QU^pEX*|s zs0(JnDY9?@jB^=e+kew?fdE3%)pNSB(J=+SC%DyTfo5IK4F-`7kf?BqC&5cIY6b=w z}q$v&1m$^-pN=||x`uflA3F?TGl`mBxV;4Op+Z!>1 zO;dfrTrRJN_^ivIE*!~8Tc_iHDG&9t)DI{trLjP{k|Zhk*Pb88`8`iR>vau)KMCk% zD~v2h6<3#dUGaxfmyNXt8|cQ8_sV`z5JcqNr=}5SmgjXdkbrWmZ5($_*O=5EwzsJx zas0CH`-uUlt}EV}9$b4<*@Z!rg((XO{~!c~d*2?sRMg9?)@y3K26FeqeEruL$f%NW zbL8z&Cp0WEmX5UN%IzoX&B{{bxJ7$4kz{nkAD0$~=IQJN`P0dE2) zFw59gX;s%0rsup8$R}WrO2*rM?}uLcMvIW5$dwzb@6{JuqUrga;v}h@vpm7=MT7v+t-v<-T-9r@xpxog{xs!Ka zT<4`Ky>RmbgrD%fV5|K=L_M3cs+rlx{0|Te>xRdU=6H3~!=(cZRSbNuEayyXzaE*n zdIPnu7uA50Zke#JM+#yRRj63Z*^yiH8HUD?PwNJfzJa*`KWBfc_ zco$OK4ruFW@^X;p?JSU!lo?})yxQf90qiHtSA$zCd6Lj4 z7D8%dKKt$Fipld}+%pVWB(LOp&gX^ag>9`%H8+LFg$p!U_3;yQ)E~EY;M4I@je9h2 zecpdNRu$hJWl8LWaSY|bf&{bka-|qS)E#^tdt7Mm!y_r%foJ(*%&NY^pHn&nx#CsY z$K^7hS`2)k4W^0*t+(I9E*zUF7vuwfWNXod^YQ)sIsbyP3Q964D(&UDBF9n6k@B{& zcmxq#nUw@+Bq9Ao3v2w9?uu2BQC2+Enw+Zv%+^=rCuLpod{Xo<(q zS%TYex}a3fi&VXV79phHSlB3ls()Ci8k64)BRQ^3)c`*|c_Y$Vu4s+D{?VePO-Fgp zdpOST6fBo*H@k!(s)boyVtWmEEzjOK$o6#ed@cHG#APd zymt?aGH-i>vbkm9Mp=D)#wZ2kqW5-&)ph7@-&wq}iXL~~eNtcZ_nwBGKYM@+z@+S9 zB@hu$MgP!HQ^zW2?yr@>d2hs=rBzF75jr2DcK;Wq*EiPx?XAQToi=Z>KgZwKmf{<3 zVp&f96q=*z(WB5}SrjYyt4Q0bH+@# zo-FHq=Mqc4w{QDIOjw|2!z43DnC7x8!{lkP6Q;W@2U6sqG}ySIU+^w7BXUX^)aXE+ zEPSozkESyq)ueNhr2Y;Y;7Y69DLl{-bD!rNM(<9u6!~>XX-2!d50z3EkfA zautqA>CE9J$h6Hs-#0oc>|0LVUc|ZqQ#X(Ad{hm&?9$;F=G$`NH+nV{=FUIs^F|W$ zJ#M-q699JQ%X#zNfP+VM1DTbN`1%zpT`%$u=v!_nHK(vA0R%UimMDvTU{iF9Sy8Tf zz^G-twwkab*OJF6&#;4KvS5CQ7>EFWD7>QqF#F&~teY$^5$w!{a%=G`U9*(gNsG84 z!==(fAyPSiQmt42cM%nUNiQeb_u^nVwl!tO=)znnqHq0KGfDld``1ZNmBiXmkYs>r z(8%qs2)lGy)&*9c!V^7JL|~xooRYmE*C12s4PyO#c8YzKq=xe-FxQEedt@?slu2Pe zOFJmUkNcOr-RndN&XV&8+J}wA*6n0G#~_WY*QCmmIV5KGyh4*|nyvcTrIdyJG(oGYqW=0=;nmN8{WR)C?yHn7j9z_@%X%KzAc&0eX^cBVu{Ua_tc{$1LIHrZ)L{dbxV)G{ST zyfbwO8U+r-H0#q}OI{T1SsmBuNd9wz=L5B4WDIb7c@Y5BPwLJcuRkMY`uex?=bH8E zq$;ppn2TJda+PpxFp0Uko&nxmxlzSD#Z%O{IxTtf82+FuQ1IW^5w5=1BVvJ$l1kTc zESlRKwKsg)#o9o%zm5acbB~V3jSR7qy|A-ii|Zk4mx2Ph@M6iNi4>l&j?|wDoR!kW zQ2YKsIjUhDJ;Gg1vmsFO$q7MX_=OT=`*P7jpvkPg30$kWP3JF1PnY9gznSdiL_I4l z?QO)Bmt(6{rvG<^Wx{xJKI-qhV0SZ&nU_}|gQ_V-4UbWb$9-cLP~{uRD+f8j2el9_!2;L$+?c&-S<9jU3CU6qn1<(qVRz1+aY} zUj*T|QTI>jI#7!WL5-1qs-vH@sDeM{^uAP$>EJlUeRxZz9_~kmpFC(A$pIHMp(Q7b zc)&&ssYX#O#TBTPxEate!7Xj{O};RNoOP_{Zk>S$WV+jpM6$k2nVbuD`2IgY@_spH z-G6GWYUaJD>J?PlD3=)U78d^H%e-`sxi@B+dR#A5jYx z)LCs*{^p$522=*%Fqr)18*5y|PT}^VH~kW5KLnjVWwoorpti4jb$kDl?ZMpA?knEQ zYT`;%_an-;P;ZJP2t_aeLA^N}?S#~7A-t?KdTczYoB+AiXfeB!z{`raP>J9fLr8N6 zn_Uyj&@G`-;ownjG`iV%~c0O&Yy(){lry3{5*?uw>ItE{ohs)B$>vaE+olhj=biGJLos zL}b1R7MZEak;$G8YKY;76Sm3DCtLB*^6o;BT2$gIA#DWiKReWRMs`4RaQH-k-(q2kr~FzXm_WGYOiE zE)RdE=LG{u3GHJ6E*SE=RS+Y8`k4*3tL&vem5Gy6xe+6;hg0lyGh_jG@es-EEi*Cc z-8=3l<#=1ZzfYwp$waea;!{CR*ZI;f%l+B26Zx`9<)ebXISu}2-5pj}+2)O2af^Ra ztFXSn-~)bP4wY*9)sMz&U(H6JQ4p(vzg-{=643j(&xWB-vUav6&<`?34Q`%mZ7cT? zdZm@v2}A4{T(V{Uv_*;xj!5eU4_zyLag&9G+G^*{r`olF&@)-E%|Jo4m3c>7=1T>r zimQErk)He3?!Zysz0_Z}YM8m>Uw?s?^J_Pg6v`wj!m)A2$H(2Rb5)K;5Y>=X@zlDl zH$C{K9>!2w@tt!{YMH{hE4I@R5Huz!n@nU)sF+$6Bt@qap*~y*pQ9uBuiRyOWfLwD^zet|>Z(1Eafj|t8 zX&bF0lrwzd=L7sOE|c$Njksu#Sq_ab`dXSi>QfmKHS+rJLczG$ECV7R~atutkzwd|8XyQXfNt)iUWut+y` zbB@56Sx&_%*iw@A-p;6@)~p8glzAVmF;G`%*g`AgHfN*K#R#7B?zo#Mc zCDh=o!ZMGGp!S#wtNMYzed)%QlIz0#Z-lE;6RZ=y#nz8qC%(G6p68#npx!+>z->l) zk+j2E^PS0ib>#)WZR@9Ffwt$yy3o>NwLf)N21cE*I`n9hD%(&H*Cc5&Uv+|Z{FcHB zH96;fj))!;5X|)V!?r8!TQgs4v>3e3ow3ualmpNUuBN%$Faa1~af+9bq2)GV_wc|{ z7_4DHYs`u>FdP9Tr5D+?hzv0}_640K} ztQR+z=bnMl6bpI(j_SOnSSR?pu^I)JH_h@)Yl*v7Hgq-sZPuhk%X}J6gOcrl^3nU7 z*6`L_AIwc%dRP0pwfg+4e_LW&Xoj1?63cR@Vf%3e3A8lCl zOz))<^y)5{I%)&K>eLo2AVd3%kpQxhOreF0OyPL|*Y^tP1UNJ%ZS%#-(sWSJz2fc2 zAdy<1H8WoPu~TI72((w!NdxKOPuhH$8yOG$Vk@x17!VIMxmObVIC`} znzS#GQXlQUax=+#+k3D1Mab;YB4Ml?wYCU7T$ZXQk>T6g_GqFFNE2u)OB-+S!S$hkL0NyPW}xUU9fCEXJh zwT;<_bvF=UlwdDbH};QQb#)p~F$uE*q8x=o7hpz6w86NIqahaC7M4Mmc2mON!cQ{k z43*SQQ$>3^2e9s*6`mJGoP3dc_rpJ?jBxv=3YMSz80p47N?)u|`8D~7&UZyH1oA8I ziOBgY=C(qFX3%B-ITUbn~;nFqCN5>K_TFl`+VS!17L*0%NCa7Wo}`KY~sX6)c4IJDF}MkdVJ!- zUA8%c$ozNx*eQc%Ct>&t?H}+{WaG;HI|13VM1Awof|q*Dyk~#gnF|4t*O!DJWV@A! z+;7ENG*Et%Q%l$h{bxpIy$`T!W4LwQu$IMf$MzUFNag~Q8?^eJ?s~C3{&0BNZKbgV z5Er|Rq9VXD$0{KuEh(3mW=5?T+DbZMdEqL@%#d|&QPubFcPSz6stQdn^xQ*_XbP!V z6mzpR6Aft}ogluZX9__cNl_hv^?#_LrUS5}ne46^&F;>SZ~Dz!Xi;94I>f-yytudo z1XAAn<1QS%Xg`+S;qmbf=I>ea!FkNlxFRMJmr1ETf4_OvhBASstrkcD&dW&e-J&q{)6eI^k~?E9T{ z|Avr1R}u#NC#iT=1BBSd4BYhdBQt5q zu1(N5cYp!HbVZ(Odjtq<$}{mAi7_nV$&^J06-CSi{!n=%D~ZMup!z ziaSz*vum`dCzO?-twKzVb7ppqNePI*=mRnxU8eYC_^16Ci??61&~dwm*Sq-smo!C# zEvMhsrOe5-HLvpjhzTu%u-YpUUEhSA!pmp%Xvs9*!#5@~+Ap>!*uPbFUkGI6~&)xck0SF~TgNQL}c z*r_g3Sp4mh&!B%d8z-aVoa6-aI%Rx%r$}=cT$eN1W%jPlu2c>@%Fb>Y_VoJ8-K#r1 zN`7K|f-hy^Dk?e6$xui@&rtYxEg~J?;{yb9*3?O`_3fHI+uKUoq+#-}@8q4morCB? zRHF14&x`NlMs;f8ze#e;H7j9y^~{_YDRaF9OYI;5Nk(vK2Ga>AfLPG+fYbOv>oXjb z(K<2U^0%B{IxXEr3i|=y8=!So71wcH;>0aXJ0Kn(A!`2KCVKUY7=FJgL8pt zI>A0UUB*EbkyUT}w%&s4PD^g!8mZ0^8-;i51pBAFhh>qGw{`*y)ap)+ts=@W;rhg1 z`rv10aNB36zoSS?B84_d_s5B~(+R5{UH9xbrBMpBC6Q#Vbd|-YkC7h~0`6B4de$_aaqrWSvFf=hR&mDi(Swt=6A*k%2<} zx1KJc6!CiFaK`rPACoz9@K7}%D9sQ&R&{@=2v>^jnKtZ&t&Zf+2Kg^hSlMfTD1^-w z$oSdOvDAncgiH_@c)x8aC&7W=#2`2~oVF1>04Ej;4B`yts7|T_p`N1-@oClwx&`JF zM7`eaSIYD?&O%+7WO4N`h_5T-D!F0AcpZ5}^?>C6CrL*80C54C4Sz^P=%>v#6b$wRdvbKx+8f(?2G_Uez)q1OF1j7#$}UrtYs-D} zX2r`5vf-7c3a)v)G&mZSDH*z14SFEOnUNQ{I~JJ!uaNg5v-^Xvo6mDoIae?*K;6FX zIKK=`q@elVP1D&((-ZhKENI{1?*wimmI)6tk<^citv~d3yt98-;)$X+9o@A)p$uyL z2^lLJ8^Yl#-ZqssbuqKCIc?qfYiqu+n_Le>p6`>Qjw>9FwR>On(Vkefzk zN?1LLvu<4}iMFpW)DNS`TEc=09ssgD`z}(qH&>FWzHt9(ZdMO34;1LX`SQdWFdQY% zkC1aG1RF|x@2q>cMzWoe&OsE3i;7d=d_?1ulc=7$`g75TH89wBx#hg_SNk z$y_|QZyNovC50craSeuV|8s1fB{!JY_F9GE<~lw;7$Qynd-rd)kYhKT4zb! z+U6NsZfXqXacZ9uFXi9=Q7DUc1(X_+3C$DK`p>J+shy!?P49xi(tr7oH82GGFgRQC z5S{fnQ3^{MS4|4*VpR9Np1 zuR9~lvR#=RefX6r!E)N|nBiL6)jYr6)?FLJRiC z9};lz&)2`Zjej4{W|BiHIB6lqq8nZaeN4OQb9U#6kaFQe;B(ldl-rcB9|wc!*mC8g zz?cO`!xPkgogl8*qCG3(UENOohDC}_)RCzU74y;#QxsI9@;UkFD9;roB>0lO0abhP zNO|=!rAXwW?u69;ncGvsHF$pWbMx>Wj_Ff{xCk4$YAEC9SIXnm%&s)~;cxwymU>Cw zyB}>-zLF0wSiJ06FCqILS2^q|<&zzzgE_rHC!Exo!vN+^)th)!1@Q7jMI?YT`kZOMe9K`Zw1b(!s~)&|Lu9bu50 zlnQEb=bdpSro=m3N+k1(CRfj~Y-V1ymkY&!gcpuTWCtiGcrV`L>k1n+v_= zc%H(34(mK3$6@TTYm>eq;M2o0vSDSK;PuamK{t*SS+*P3DyiHbw+9uy1O-`g3)0n` zRjo{pCT-u{H@sMl-J|lr8`X{#82MsXWW7z6PcmU+fJjj|eL7vglhM=@_RN4|G=&dQp)O1%7tat4!vyJ@k z_vD<;7g@$2U!DR%r7_97yUHvK(}IB;tc)_ty8Mj;(#viL0^jw)=N{o&O-Q-MF@C=d zH)Q2jICvg?UJmk{+m_S4RfW{Ca5a3zf5fWi;kUsizsyaqAB1#lDK6@aIxblWUHlUs zJtf!u@o4R#M4j}?{{Y1VwH1d&6&t=apkllpZ$SI?{~s3D!8`39_UnqKbaKq<5-N#f2LF8>|si&xW+;ZDmfU(trWB{l-El==I z23RE!8$MHQ&14xeV{zUjwMOGu5LXFdJzaLs@N-Q1hwlMqHPdiTOQ&>+Y zPp`j=ShO)D?x}O;IoqHo&3qwE^jMmNI1>5EJj=~#6czbg#e(M=)M>JHOwvDT?^}fOxzfya;aq)W1>!dQcu`DoFzJp!&SnMq?511NZ+t0 zv5JS?VJp}nzjO55P2aOO*1T)aG9L`M$SFLGFR+$U|LIbz=O5@x@Hy31{bWbL3l5Bl zg*`(LMnfgloh+s_uOmhZvt0Nj*LeLGDQ#FelX2Qo{AFIC+xU;i?#nmZGj0HAl6@Mx z6vz>rEj<;&51~vg$$`Z9ojiPoVsYeV0HZOoU!k^VXH58@7oBNqcA_9H(pjiUdY=yWG^fBXV`g=P9)`#j~|h6-$hbY)U@8b>5}yMz)YkYHzg^ypFBulig?)@`(ly|zZ>D}<*DNn78&Wk_FlbP+PAm! z3Oc&z=p(Q3gONObn=Ei`2hg*HW8dr=ej2k3OfzO-G4uX<1$2S%j>wg!HE-V@@e;|t z`z~W_m~qO0eUv#OERwDzaLD3g3ZrwjPQ`g*!=p#`13vBjw8%RhoTrjAk??D)$*Y7T zT4-ubQ6G&ki!5zcW}z582ysv8l_+n;QLn?F@T;!Y2Sv)2%GxW$^7H`8qQ!9%^d?bi zQU3Auc_y0G$XxlyyInP>J?a`%e9K0RtB)g0TYso-s!j&aO!FH{K;ePM{5Xx(D6%P# zhB8{yB6nNjrUdd8+Rb63S@<*y`zf47BfbR88R`5Q%s?8a8SUjaP1b&+Wpq;3mrqy^ zng)(}2(z+LVfjYsy`~J8D!59bRN59C{{3m_jm^AEW7j-qA9~!qWR!nV`WWthpUwX1 ziLO9o;C_JY4MwHZ0-xl6UtbN+T8AltP_BrhGoqz#+AbgYh>=Yz0@{)YE(MtD0hBhj zXwSJK6c$e--DwS7xE9D?u76bk-v!ifrV^WqRvUPq*<9)0e}_;pNZsz6TmC$oXfha2 zNvA3G{YeYA7{gmRipKKn5@0c;&ZKJX%aotrj*4@Q4)BD!VuFu$#kx*|D*K4I+-Ozq z<4%r|Xt8Gr;~>`;zVVYqz94Ffe#8>|5?+89btmJYI5t+P})fa6<0FN67Mo zQ|E?c7kV2I9H{R6<$)F<$^p`iv{aY z)G{?ix-B`JSs7h*I*c1)x+mQEE{(kKXG_hgxcW(0eHE!r?-M3%L;v;HwJ)Fhq}Ijk zc%C*AA}Q1us&fk~iD5&Vvar|2fpM$Wk?SIxUL|JtzHElGCtK2ZU2&8)+eP+R*dAick(XAd#v?fgRn|3wFR@OYBu3uz*t@o{->UIPR z&AcyDbvbHreQ(`x9sWXe29!+}i4I%vxm;0`Y*_o_{n6FEAxGr*wv4vOXS7Y!o1!z6 zt3$91Bfqt+e=1`W;4Lb1i(PajX} zq~EbN9lV>tjwp58W}nm|1Q6PoYb4UuzcITz3p`nK6d*rCD!?+|zq@wPa_SF>_XW-b z7kFgfS17-~HbeDAKuQrVfC`Low)W)X5t{$-ykx;4!s~-nbEILqWTWA>(@~J<^r@mS z^Kdv~i_u2O`>s)EIML#N02X$5k6w@GZPL=1!@soxi8T^B#$`pS1b6C`>28%n;IkA> ze{Nwq#cIVm8#9PL3=XD!Q$E~O3k|#VU~LG=YmvWczU4*CYXBHd# z)nT0HAYm96KBtYHHiXQXu9lbvn-SY;7dmbJ)c966yNA-+{li>0dMtW_4|VvVD2<$e z#TX!}=}EU4zZ)81FY8Lyl8J{YuI5^4m)0jt;*!q;gI<0<;^Q5}guGQ7#j0ClWj64K zL!9toa}K~AXMAeS@C;(5+_>UFPH$#ZW0jaI+KUe>Bk3BGl}w8c_zDhhyoet3shm8m ziOv4qQ`CAI{`XC)%6g`aT7dgs>^|Yl_es~E=1_yTCO_D>wUOCxV(N&E@zR3E!eV!U z$&t#{A*=S15l_5dV}6d`MSpc_>f`)dUu0q2p<=9wOt5n8D%Ow|X*4fh)RdRK=;zH4 zk6{se)`ie~)b)*#u9OGL34le;lc=<$?ABo+|FF@=h70kpb zUaU{ISk4P7;fv-B9j!ufu*u`H^(vhoqnH;S2x_4h5>X_5cKuSqhGSMZ71%W|OA}fH4qEwNP6Os=E3>mVqa#SaFxc6vq8ywe zXE)KSIRD$idBp7b3#o2cj06X9BJDeeHJV+*t`a3SWz#YS*7^NilYdJ5CI)Qp7Y__>A;{!mR9sM3%6t6Cf&8KwW|0%2-m-PO97~Yg! zwRQq6|8;`NZOiIOn~cQyR%wYXk9&rAg92_(@Szm6#hOPanB}f*U;B&-I)BWqnsEQa z*nvYc-4ZL?*CU*-{CoJ-^n|R?XV?;y*z{sX~zi#fCB%^`3 zOJxETN<633BxazH+5o-rGxL-~-olbVn2NyV6FlZKduB126AIi?R#8YAEZSe~W+ngR zojlcYV7EFI91IGwmz?QrZth{!4TO#X3Y~xQj)-#Z@OI|<6OLwoV|U=hwVfUd37n2f zXxs6*?|*S-KFtqxL(@4#`mKxag73-c)D)s6Khyv^F4E**zC=9sYjADH6yINv((fuL ze>EPw&FOM8=oxy=CXi?VXr~e2-9^F($}W(vnHHb;P7JaE8|D84K%h6$sDMTq-wkRv zgy_&`cxxwx1_{g;VRy5Rmo;n?eca>H6Pg=HXH&$Pe_=Q=S{k?H8@bZW*HxkazBk&@ zzM;I>F$_&!4~89Zcg%UxU6h(Tv1HnpCK|lTs4j7$nVKmk1~U33!909nN{e4f%%gV< zoP!ckMmoQj@ylRbQ1UmKjtZMWH1`h&JC#Q{Ck5KPp0VGJkLGiFJ-O(l)Jz;DRi2E#AB!!R8*7`G&k~6HJ-A?b4v-SsUkA&Ka zOC5>2zpkHu{QSF~%V-2N38X}&2g;zXf9O;`o32{s{pen+gMq#uLlHilD%V_v_A zy2L4hZ}dTR$E~ls2k@S9tl3tu9+D`Wlnd^Y6|wf$vP(;#ryvUIT1VZwr6GS&ku3oz zQ+)}J!6Qk{yZ4w9+5%T)X$7mbzSIWdB@sgYzFq?ovQBX!x31a9_Z_sqxEbnl`^>U+ zfxKjt&Tgm0F17Q9pRHuq#i|2db4qu04+LN^;;~G`l5pYJ>6x(ab>+Fpf?)qK32*F_ zB4!Em=YfBG8lfn4EX6<b&V#5_t)|AmE)_UFZxszA%Kjz0$6JuQp zGI~1S>Sa?qF09j@m!?I$G)7xX&g;Bu!!~J;qsQ5SQZ!Cf50L;VHVisP4z}`~P*FAxC586BO>pIVhlP6ajC##P3yjxzB zKB)VOP&Nx|oW~j98to58{E5%(ZXp;J&|7|BUl$u^`Gs?_As0Pv-_Qw*{M~j&wD(!X zSMdiA`Jn8&v~(c%`ejCFDGShW9^O$97mY|l9fL?3Y^b%(%T1QIgigWjLLX7r&5orW zO4jV#jGx+Yy<}-7@wzu*(!7g?u!K+cs4antQJ(NKfjt%#vPqypk&f>Q1XLu&P|QM6 zo1I*Rhgp!2SiCF$9#t+vKMC)!Sn6_aaBPng#kygQoAQlwqrdf#)z4>q&mHyPE@{Gn zm>&|!OvWqZ&wydsZmOr$K-xsskm;YdhuMMfHd*f;H<{Ejsre2TRtuZwAP;W4G|DTc z*B|pHA}Gw4J>^N{4!gLwtFW_)ES3+fNX?m5Y`uZ{%p7};&D3>sVL4=aIEcf1;x?S% zygoXrdwq0+pEat$UoPkBww2?!!LQ@;hL4>F#rvi2U#Jz6R4>0fw9j!m=tlEH!`+xt z%CvHFBPqZN7hZlg`C(pJ5OV0n4m1dYUm<9ebJGj6A6kkry(3a{N}~4~i1RR*F)4Cib3*Q52P! zB~%G5EsvtloA+bB|BoZT`@XO1JPp~}XvobBWxd@+?%Q6WlLF~@%QLn9NMNyC{MSH> z)T%C)E0e%lEzh9*^(v-xG%lQA3M%1YPWL-Q%ja~U1_o>vuce$Fdn;-4|LIxGFYakG z)E#`kY6I?Y)4x*)BbS?GqqMuwXESG(6@hIEr9M&mMa(wpPfw5I%2#XxsokHQ?K=Lj zWkRzlEuPSFP;v4wjFlE8Ky!N(VfB#hljp0hOwE`uyk|ob`lW=QNOsTktd30Vs z?)$%+$R{q*V*+HKeX!sElt0^D4_(S@0Q*5D@)8!gDP>f z#(!O~#_z3300q5^cuK$by`~R{F{S2X^!r+Vva}K5?|BL9Irp#q@L6zf3%^z!-Mmy2 z(|`zhT#X!tKHB19_3Oad>S-U4a~M^fyv2Nm4et|L@ z8nJ|{uDz-Cv|w#j9U6*q?T(7C4i3^-75WrUcpt%sN}R$85|?um+A`?iArJapZ>QLn zcF#g-2%zZsKL<$fEIh4uRO_jvA)^+RB}Z_ldMa(cB}K{>}`r{ql!%l+GLBy?|po(uu0Ql0EluSz^mt{8&zx ze)ZG1wh@u4sQn)xB4S4Vfm4Dj7z|?R+LB z+vBYRNt9W}uQ3>xWrrAGubh9JM;^r>MM^_d$5BEDun(vOoY7s6cfpu(a4OWTYvhis zxX83X3$f~@!~NHM7BQ{%lAK9gPlihn5$UW7XRwOP17k8Dr#|Jns zQF^H|#It@{ycPZ*AZ}?6r?eRPO(LC6C8$K&4m|de)&<{B0UPc=cKK5cs}1Rddnm*N97JXD!Q{>qw zo}IQDgN4p#zm}R-^`XQXQEAWCQ~OM*bA6u;#9>5rT5{GUxdJDABYIi^GznRP+C@!F(8|}$Xa=-EY(>{&eh3w`v7}Za zMDt?i+K6I@{m@YQ1U@kbJ~BYiFc29m-uDwXbp|xA9xrm+y=+NR`$EzOS8u72$}Eos zpi3SG2VZAmq5h9S`T6tKIPCcIVc|U7vZK?oI>h$LHH2RrscR;7U!_M{a-viY9-GX_E%|JJzRNs!KI;`Q zWAS2J>>TT`U57N8p**kw%F&L+2U<7jv~NC@&Jd&qX7$C|O{NBiw@==v(CewFy&}Od z4?0;zYGA9cw)~HpEmuyyLXDv^BK@79-_Ed2_+M;9CtZPOe@d0otkCG)aJfQOSN=P* zU4$nxKYr+nyrEn$+FC2mZltyfSJ1vw&>KE5G}|G=2bK=x$qE^2u1)anm%=KZG1}nI zfk6306Hyib1@f0FH>H*;*Q2~nd}$w&c@m<`mHaL4EL*!Nux3BdSoRBsoi}7^=Va2V z`r7_)vC~#kBP26EEu#v4X1!{1eofeDw|N0yWLRwY`M51kVnm2M%V0ZKyjR14MNb`Y z7vR<4{9IC@yerL)cHqY-4T5hkdYwQn^L$DoJ*a4YSqOe|fmSJhIc-_E@f9FyrRr7@ zsI#>*#AVI7zRUVX0ZM|5r+fQIZ&Sj`7$} zB2gFd`kraBSuGo^V(F9np4zqCI^pHgBmr4LoAvB>kt04(yYluz&}Fq-hMg|jV?uvo zZ%TwujIPUKTSjo^Pmcce?=zH&j$!sY*@9a=hM{jyN#-#b?gm5@!{w?qZqS3Uu>m`E z+ohVT#)Q3==?~qUUkz=n$~P;nsCAf|1}0URi#WC^>{W=*QBu35)oGk!^3?yKhq6~K zfo|UzGYRUzagzwgZuMB@Btaf{gyY)O&ZQh^n0-Vm%C%Q}R_Q>{hwhTnapdpMw?BhJ z;zRxc=b8-%-7VyCwGQ{S9jVQy{bH^ACW|W%*(&(mcp-_rRN45p$G7b&)KK@#XF@)z z=@Q?&dAU&S*&MkQzVxrY*GA0hJ*#zoJ_l9s#N+bXb3LK>+AkH|2gz3Gn~~Cz@U69; z2KaOkbRXCC0aMdv#P@^uyT159RZfRcw)O72lAW0tqn^Xgj^)1|XHR@N z?XR6?Z+_Wto%q7O-|F4I4-t$ARuKNOsiOB*Do&7Nj+!L)=v~2CH(QVSRJ{-e(cQyJ z$UNjt)8pRr5~PT6{K>@4H6_9mi)u53@*c}>O?+b6-)^i!XI`g@&l&4ubf>1dwo6Do zu02P-dwo|S2E?d+vLf`B0|PY~Q0}+eT3nn% z>}>gqTehXzq8wOoN)x>9?Iu}A*mq6K31jUOK5xz@UA4_=jsAy6ABt(u@ali zU#sfJ4;JIVJzM975dz7eMB<314^k=3)3V!27wG1^0#M!N3F5I1q$g(w`WX<~brtI@ zwlp6!Vu~afFh{u`w(Sj+6(c5_dX^tjL!}n6ccf>%ZoehJ=vHyKoCpM{Oz0?`Q52uU z9v8?(Bt?C5gt6;)g#~N3>;ylc@1B!~d406JtmG#8%$=WCE^G8a-H7S{PHtntt0 ziplit9se^0xRs60{h-G7AA=i$- zm09R9d_91H1$q8<;}>8@OGvgs-bA(G{Z&L$|!ShiACT?j-c=wMw994U@#lLZiaIO6bZNvab*N-nSg| z6QIt5r*=DJ(k67J)2B0Pt&eAZloCQV_jh7fS2k9*n(AJC#1a~%QA?{$J&mE!RF1gz zO+U&_EpfG8YAcE!)!cGPn{X1*N!4K$C#@`6q`(>8?xr7k<{hNqDIdtK7V~2+kXjNH z^yLxxdD5c8dM|LUqCy-wUS$JDk7?M@2bCNo2-MPLdfIe;w%@m94>ttp{ixs+SHIrF*J*ysG4>1}G;k4sSxkL^pO66v`w0KhdLlH+G_)5|` zywyQhsD6?w8P7Pq+vSzXf0prHn`X)k-8y7yyeT_f_dVOhS9xtah4hFAfnpg;4X94N zdhh&|JFSf7C`+!pIJ^eE@gB8mTVxjFE$Deabh>Sp>`3*ECq zu3Zdz;flYSQCYA)y_TL6xlMAF@|eM@vnSKGkz#pd(i<1Q*o$XY^lKv; z(m`~swt|4sc*9t(*dabqA}&vp{e*V1ZDljkCK(o%1(Cu>U5_BBByA*RgL_|H%q4y@ zsdmI+Q)#D!%*_miXSb8ww!c1Bh_6xM)Z8HG}J+q%;)Xsidyq5(VSGR-4xlNROOXYo z`nWy`;3eilO>@A}&4W%`OFF@cLhKaLiuwaRz9=J(>u|6=oriKy?u%J&KMuk{?~vsb zo=elTVdHN=_PIyN?n9yBR(#~uG()y{KbDP2B)Nd9`~k(ng*f*=iV^-BXU^_;k&jR$5OOuJnw z&HloRuO>Z@7JLUkT>E9P6-A|qJ7Anj8Y{3>`wdpdBu74P0NO6 z{(K+oX~{41I_{H?SAl{0ZeBfz$Ir%A4UjpHDzD1yBKQFe!>LaMJ#=)n$1;PJa+j;` zF1<#SN;SQCEfrNgB5z`}a(MsmR2AcDQgy!P(-8KtGeUR4#8Jw5Mvi_) zgC22q#scQ~Tu^2Lqmbcy)E-!~H~cKJ7~Y{hIcGU7Acd%w`a_H2#G--Jqv$8xTZ`A< zRzaV)+7_#zc(6IL%-W>cv%i69-DzH~6X`3^C4_ZuuS*yPuo%Cj>kZtNa7D)JctsWi z2XZF{eL?}#N}oKFN|LN7bYhO^Gxkbc?sFDFp_2DV#~)X znn3UHV?YqkU1eIMu^_k4I*UT<&+Xok>&WX}4IEy@IN(zO*#?ToOK<#jU;orh(&Zj~u4suX;aMAY*z z8fF8tP1@(T&eHV&UsRoJ)IUzDO@DlV<(Q*ef)?vl+m+e9&><2I zi7lguNUc%TIr*9=kiG0jcIj}xzAvUucR0Y?_=Kl7KOZz(&r!#u$Gr^8Vh{d9(6VM`+I#+{F*5cp&WinANh=y zBwKxCOm>RsK*Wd$lF@5&3BM0Ts1+A`fA&bT7LhblV3udma2d@>XXtZPI^mSCmTtaJ zoaWgI`EBj&K6q_5z~YVR6qtrQ_Rim)QZ{_Y2KN<1_d7Z-kIpy3gxTtzB-8*w4p2?+GLY1mJ-F>E{!XdTD-^tDUdyEGpTAN~VND z9evzUM&hfp;78ppoq+!`^pWoNec;2jZ{iow;2e*etq4u7DyjQ*%h$ zhO!86GxCLXiRbhgDZ5XTO(e^-99o1FlDKGZb>cVd^+VmLOyckB!R@&2#*O?O8dd=y zX=Qm+8tPqz8ix5$ZuFUii^w3(fSrFt0;z6YM;Lc`n8bH7nwswUPu&N`<-J3vLrUp* zt{|!+DMP`Q%%9KG!lXZYib*_fJU|vjHE&H7Wvxh?5zC~r5;!B+YU9SDFnsR7wC!c3 zg)7HmZH|O!;i#(Vbt>PU9Kbs+J!2?&(mRY|{1YP=9!aj)(&r}0&|nbZE^fPBW0J)@ zZN+%)wci&z8>#qWks5C*w9);4VISjetmXqjhHi4}u-TrGySLJ+9vLeNGyu!+>M$BMJA~t_^)h>y%heojniXI;ln2nwLN)0DQ>p)80U8haq%QXQC;-V z7L2AV^DEZ^+nCf^->iDh) z$~n2C;5jZm3r>mB#`G{1#$M!gbf0p7NQSlin|u1SQ!@)4ae2wI!4+McqbSb3Uub>o zwBPdS)`yJn)5;_83(Ixi&ROr}t){IzF>fIsk2O}NE6Zd>o~KuEz}daLtcT|&0|etr zI0W(ED!J%9BO=_%w1lvRLNlrI>xL3yniWpE8yzJS7Fn9ZE)>t`9B#`Z3m2(R*lLRS z#8?$VTJ$e%KB{u@+$5-J=FhTjtT{wD`-Wvxy<_WUVHzLx)Vdyfv<(>g*_}G>YJoMd zDQmnq1iyo%{q3g>Zuc|37`Rrh)DJk_-1PPBB-S_<*=z|HQ_}ZDcf1+Lv-p9^Gej&q zm)>tauyEC&EsNAsV-&SfPrJZWroB|_j*bC=`D0dkPCQr2_uy}x-2N#wUEN(~otaW! zlV#5aeg&>cu8~D-c0&tO`O+;>Mc|YqujKJYj}#Gv(CzwkCui%LV6aV~I*WWa5OA;L z0T5T){JOWsu^+sAvl3Lb&YISHO{jUd2?zN%B`G5B(g^)PB4iUJ#YKU5j($O}Wmg7--HOQW8#swz!2Sw4nJDz7sK6j}-1w;e0q9q~@h zs0Ox)jrxOY`qO{nYhd0iT-&?Vi2T;vLs;DOH&bj>sm`d3`48Zyjeg)Y?m-^&_aRBj z)OxVW0gTGk&$rg-)e0f7OZT1JG#z!G+kXH=ouT&!yjztw*Dk|cWtBgPA_MIwneV$36&IlqVDH$7LUmv+L-xK1^UtyF{HIvCQZ-q^6fx#vq0U57iXf(zpE z4)Aj^7oSq+%K5uw^?9uKMaxqN7F;gI8+Cz!-2j4H@FaH`>CIDCSf1 z#;#Ia>;r|UnJS9!G`u9r<(-2qcc;OMJ5|n7?q_#HD`S+qskna_B&282?2@q&R3_Av z|5d|P6YH7BD|LS?tF3>cdU_)Cf4@?OYOH>#hw;d>wW2a?CA;(m)WTb7L2)B+b5l-$ z4$WgasltZDg`TJW5j3$PrH{~-@8QSHX-(TIW=mvoO9-8eMI}_)%t&#NuvL0MuVGz# z;vBW+WUQ7a)KoZ)pQp`)97ZEP#{PVSFD#pkfad28WKBm9ohi9145;Ud$9*Tg{{GnO z+gl4-i&7OL8oIOe(&lifXCZWRl@Eomb1TEneSQA25I0)@cQ;4N%Hj&OtfuL=UhcUU zg%Q#}`Q*`|IcwK6aGKR_rCj{=j9L44m{LIGVt(t_bm2>lU2k72Z~Nkyjn zDLv^>{Opq8l62DIUD<|NZ=R%T)=ADhx}J1EvBO7tJ?_lYKK(rlxZnD~S^Htk{ON{o z=pWj5K;HMAOaAqn*xT=ior{={g7d)ex||uOO=rmrJW8`^FCC$tFtjc^HU{8=hzS?c z%j5K^`%ri9@+oHb#N9jiXhpu)o(BNvse4nY*+*y0Rr}=t^)He}87H1+YUQOPqNu2RTt@#bAl1j}eBE30#lbFdP4ktw z$i~yak5pVC&8bI=8?SH}8}D<`s|z`uRat6$I&n?EG6j)tLD5 zZsfv3p0c-Cx*UmNGrC*Q0%k*3vW$0qpH7c5zx;Bzf_TII>>c>NM@kElH2|JRyUj7p zN3Notgeys;%q-7W3O?;wgd0Y7&%BYx^s*=b#&bP++uWxBV&X>$p9%AXd-#mTI`GBy zuNd%n<8*p%orKiMpc*c6^Y5VIi!S9`^xlUL<}tAVOTXBD`e=i@>OwzJ3D}q*&u2%2p?>R zOs7vM9V(fv(rz{MXGkj4Fa#`(kvUlB5`ex`L5Uo>IW?K;y98E{1PH~c*n z`HU^u5v~R{N`YhEm>4z^yqg=-cX>BDXHc?w?(Oj`GvBUt^C!{uT_xK9Jy$py{e`aA z=GdQC9u}9FJy89Joe#rTd6wcVSA2xpaW|nU=Ot^ZJ2?!^oTjn z0@Uy;wT}BkVeCmeX*O^u{8P|7&88(4f4gW9}(BC_S*YFhIo z{JL`$bcL$}InS5N`zax0MBX8vs$uqZ83ofJ0P#F2F0**t9lmZH;-QTdxum|p6#sc* zYeDhsb8ZK5)hMw&RiR|fbcR}haO7(~Nx8=z7a8i=`o5BJ_D|2)C&@U;?}ub@0}p(v zGM76USYMWqP2d`YI9jQ0{Q0G zmX*JBrqa=BuZ$!IQbN*YLp}V`EgZ>%T>ml5O;K_)c7r*WG0A7pCqQJRzKnfxcpAh) z!15yVRPRGwhROi}UP781!wtx^v^Ad5md9O1y>+c*skZD*9d6dy+jHv~s;xKj#R+cz zZZ`m_nDy+Fx z5Fe1hZ{=nw9RoHi*wKs=uqwc=Xj>>JKddN9@tSprGx?4o z8hKg?I2Sj=NUfS4m+QGJo&Bb+pf;{aSk4iYU#8tgpruip(8thkyT_{REh?I;!qJa%B+%)S2=+q$ymTQYRf}yxemd)RjYpZ`SKC22RSwoRRr8<#-XpD z;a#(7_EQpty!-y?y`XE91wJupBSTH5y(JOTD34LGycQv*Xv|}fq8P2 zhJ?^7M4VAGq1{<@ECXInbFO8P@TDgZMUL>kcikR?&4}VP=cT7AO+M)FkDz-4(MoOh zei?Kh!5_LZot0QLmVj_=At%z-+iw;(E@Eb-LjG6qFrvI*!P|uKXxAhK~sx<5=wRV1;h6;|; z&+=WXV70tnjwJ7$fLoi6DT$FtoKe@|Y@opn-4Q(U|5AKHk3*m9;d@R%65JVhFQ@@bkpSXaHhJd;`( zel|wRZvBkpOxrW>M`gHDNmfv3d0lN&Z`kj%)0m?@bi%>Mnbp1?>~%OXP4YV8$vcWF ze}9*iaj$a@TNf`ifG4-s6mU;wOI-e6l?n0wW2Hgw4yJjCM`ZB6W<&9=)=0&brkX8W zYnm4s`vC_$t+uDB@=;$rs~*w2uQKYUNuY+CCWNHkVncTxa`fE0+3=4PG}_45vI7*{d$e+Sb8>TnbEGKKmjYkf z&xSgP7E;W(ya{wCp{3zUI|y=%jc4a1i46gUqvSCcR{j#GOvNs-2Pe2l7UiZ-$~)S@ zl{_V2;Me|KuQM*e=h@}mz5-UVwYHeJE>VCu60gby$loPTZ;vwc6zWCi2T@zX1*cHc z{9}pl@=zWh)sW0tItKa22dZcKe!s3y8yC+yd%~h$T)qEX0a;zrJqlYwl^xYLG$O3z z+n#nS;IEwf&Y8K@kS$kdV=ffD7V4S8gK=eRopHWAo0h&oT^iHcRF1VgwO5)`bJWCk zZ?X`yvcMh$Tq1{yTC2ErPO;E)Ub2*L|M%&Dq*H(4M#t?8)R7ehAL$$(GdLsqU{B%E<@P>@WYELE8q z@(i-8+~8i= zb!Bc7$Q{eh{e_nP+u0W>W!Sv^u{so{}8{c#!UcVC?ej=byt+Fe_>HtOQe)-uQs$>tbKY z@cLj~txUxl?ltUmXj^aRML(+l^xW6S=s~!#IF<-VG`?M#cRpkUwOlBup53=fRo_b` zMbjrvXgXH4%*hWSfQ(%Z8r0UZzzAR=L>d@cN=A_-x6z$e8*ma3sADO|%btEWNObXi zeDp)3iN6gb&(mF!-vTMfBE+UR!CgR>7W9IqmsR$n<_BP==n^_5=11OSac92N5LLYK zUA#b!@8pK{su6M_)a2PsV-M?_yy8Wf$X}>;(-99VI~Uyc?S$g+bPUa9%8!JGRQR=q zJoz%Jlkp<>xoxz~LrI$utnVACu}IU8J}wB@Q&EyaGt?74YMgk`B&Js=K|o*q9(XSc zoAv#R9be}a>puEshO|9yUY#RQYZ|z&bdLKTtSAbJnWaw?=~^GBQT3A4f6Sgodbu9d z*`{D1S2NuPYIsb#Plwy2kS+&>^ItUxoIj!58Zv*AZKx(FyId+fbr#ultDOqn( z;zWjtB(57vJ*O}!lkBKw#){E^fW7U6w40PE%>z^Fv|PIbD~XK`5i)#csBcYA%V5g7 z*J}ZO_7tNBVXus{qCS-V~i zE8>;vf>GXkm$%T@i&%$^5S7u!U$@ps&}qz@#wOc5dlr;3OBKyM3VEEQoA!8G(&ly& zX8>zfSB9p_P^}lLEK(`iGBjHf-#@0irr)Z)5=p6MF6lu1vKYp1VPaq6=N| ziSRrF-FCGU^Kc1L7=5sgO|!sU9uJD+SSkaBSW-sp>!fBsB}vL)ba$J6SK!VkO@nWB z*kXa%Tz4Ec8xl8ujCmMoV^1*b3R-vOzq0D66Tzelqedcq-xv9kr1`oQ?n=BXB;y#K zTTzcUe)PMZIlsHREiCGJwi(9d$cq+Mr`c9C8!(~ zxG#UK)V&9AV8=}glIJ9o)`P11igGam7^XCxEPdX^c4J1HK(#o0j@pl|qfCE2L#%|; zP2KH(0B1Fr33?U6?!Xq+a9ZF0tADd6s<*;)eHzS0N=t?rf^A1GRi1D=nZ+eclQ?ul(>e zcOd`mE>q!k9pk17E-p*cvJiynTEW{1NbtB?Cd%~{rzNLH-_#`^aGJjgc~*@P67|Q` zzrrsqzgnXKw2yv#i<0V1_OQL;iFfY(TTL+gRQljd@N3^;m4}Gf;PyN118V^?fz2JM+|9jQ& zQX#!4$MD&yf1?Eq2YTWNTQaQS3EIMHz07iYPOsF)Ioh#1Lw6P@NRs5HTswT!4GR^h zpS(xsUoYHcCRT{n^yb*~8JZ&@C(E)?B}@1A5pE`Vg^f73nK>`%sI#*d^^ykM>EL-g zpc*b7j(^E+2wC>8ot^CylM_uV?}*G|C?xl$mB0HWG-{MDt3fy-+|al}C5Z)xd*HXM4)#QWYDbETKjqP)D)uW(5cLGsr9etp6fP@Uk`Xwg)~vUXc}OtS@|S-TZtFbqSGat7t}tG> z^!LJHQ_m9#M^M}H7q>2387bjx`$heY-JsNWlHbSWy4bj1vBbD=_3&6O#lr0D^n(S? z$wStrxxuNNNtOX6fm#k3V?3t-`6Na9x49-Ql@8~a z)awhw6gO11BZvV{@ES?_#oa;^<~R)E8c9~S&Nj6+g}v&RT;9lmb)LX)6r)Ma6K^Na z&X=k?aW+k-kV?V!u6{l9i7(IDo;!IWXUZIkm09#3NXxT%c3~1eeO=zDpBE{aVVT85 z9!OPFu--ro%W~#o;6CX44WSlQl$_geiAI60<(x?mr@KX2o z3G_&1(<%GMYZ;tqHs&&JAj^8m4;8h4W_+48Ok(V%E(ZGl9Pa-L?+%KGry+_Ooq7(d zPx~8I^Zxi)A~QF?VzCrSBJ#ku^7&ic%Waz8S9)0O6H{2gB#Dt7@ws3b2chV*;cps! zeqX&H)+C=hMf}-IlGk4j{J}M&K|J}-W`px3Lc8Ef!QB`I(J%6&K3WshF2Q%q7wpIB z9A<@Xn5M-_ze?)GI^ffkTt2Bi}UK2D>gmRA1UbiV4uZggz$W3roGVpkVe zuk)G@FSjBETkn9$b4Zcxbq1AW^U+=F+R0Mf3SdsN*vjl#ds&=fu!*lwRp8vF2dfFS z1#5vplU+;zv#^#pwaj{~cwd{HLv$VI27xg4mRUtY(m$S4;l6Fopm?BR)TmUpQJ!%H z9ky|`*oVi2DK00JBJmBrNpE{w%v_!kPhFL^cxhM+HS}3Ed3d?`?VZ;Z>-VyO{(#0+ z-Q^psNCez60pmd|5P4;Ob?1L3KGnqubpLl^#CTDUN~N~KHnW(ab4@SB<%K2^Eo_@qRgxHjGo_Gl5>?`I3BxE=f2AM4KL|VIGpx#ltMrA%LaJZC~%! z4YWRe;H{$$9=R!rsz<5vpQ(`6Wb0b~w!c}efAz#f#c z8s{a<*ciTkV=G_mCXsB5QSk(2iZs^Bf;XO;?gv1u(C|rVR1% z4C6s%lnQCc1qWyQM*NJ1-wY21Kg*Mm&{dGUNPU&L9+%zSrunrVn0aNl*Do!ob4Pqy zd`i>G^qUs&K$VYVmFWI3$$z%I(O9>LS^9u&?sfaF;*5}pKK>JN(}M%^cJj+Ur+D9A zdaf%8=h3qaC=Mj*T;n6?DPNLzZI@6vJ2*UjT%Gj(nUs4mCVpMReUnN;08UMk9IVTE zBmTSe_wUBv>+-XBobZ00LB3bum0PRFAh}CIY{U@HzMwa>#)$@UGIVy+wUb+4J57fc zuKCwPL3UEG^ZVYJ%lo}) z%Sym1Lh4K`fi{?O+>1YX#KdZ=_atyhk;#wv&m}IJO`a?l7}Zs^V;UP?-o41PMc#s9k9c43xDS1>aa-M%F_YZzX?kUiO80RzIg#??~Ip zn`scts?2{m&$lNpD~?NE4C_9ESsB4?OLz1?n@*~b)CA%Qcsf8BIsl~eOE4RPT6l~m=uGrmf9!M zcof6}*Y)l$t)`O^f3&UKEFw`hpqu6}{D=I`f5cSmO~8@^m%>Xp%SK{2BKt``Owa*) z+hjOd>STS+?y+OZA+C_6jKrJM{{3-Z$I#;?tI0EDN1A^`ctv*ZDIu3p|LhaNi`I5W z52E^zypdb7z0D5&b5v-Qt8`kG#DK^Ld77E<>V)q>F2kz*is4;|9` z54})l2i|gs3|K&+9_?7|)WoKC+-7rzfm(epCWuo$;`q9e*cNN66=zN?Bn#;t#(1>z zFpAT;vR!7=5TsrCF%_wF2DEa{QijaKbf`r0FT=y;zNz#R-vbmSyv+8sco^F7sa9?z z;$&dcNgpT15V)<8o{sM$VuZEm*H!Py;*!_ppik`~lCT_bVei*@So!-{4gGgEO$pTH zij29pz0nZA69+DjD-jf%lBs#hi`Bgz<}#<689E8Be#3gsLiLygQ!3K6;E}rE1>e*X)ESL}qXXr? zE#&0t4Daov%)N9qEh)yoYJSqQ=_>@aSyN=&fM+~;Q-?$mnc#yG5r-I_(^~2PLL*{( zb{~-futGK&Z!2_KQc^KsJ_J6uF?uACdf!IZcD==x(7470+an0YVtYOKIM!un09Sqw zQ8vln*8Er%r4)MfC2QY=bT7Z5bkPz=7YLH6Woo*2%KYY1R&Oz4t6p$0QB3Q#e>~ov zgpuC)M#J#-)Ll$EB1CKRPtNJ;(wZAQYURC1evop@T+;Z5uA%crj zgpcro$NN-_6bXHIX;l9>0I}NZWOE2!gne0477xtLF>34;ss8i06XqWclb}AXjQ7gp zQ|P1H#}!_PGx5k-B;<2bPfu>`h1j^KNl4I~mPz;*`(DnC_=L26nVO+_nsrx5UxZ4X z88Iu8rL(FZQKxWq>Ew}(Yw>yUd`O;&ii$32T9tEHT%Tob*HovBJ2g@K8yxf^&eem$ zp$(e&wg2Np4R=gIqU30!`HP!V7?bu za63L02?t&3$&<8!LnaBPH+fc^ClP?y0A1d7=JJ=-nZV94J-xFki?eR0xL_?~L#6G$ z+JG!WaK!#oNyG(#{k`5#+?=M>+Zn7I_&Z}b@~Qmx&FDwUcOMLZ9jOr`a!HLC&MI!J z_)OWla5$xW7hm=qT|H-=I5Ux^-n)2fbA}`|=Ap#2fdr?8gtFeg67% zSJ#@FLt<2}Y7TpOntS(2=o05XtthEZ?^rf|^5I61dMb`9ziHfUKe%d693^;LbO4&EbFYhJE)FJQvbp# zJ76ZrGwA+m*ffAb+<5!0?{HA^gO?6LydOT2l-Rd)XIF!gw5xSUG6J`mg|+u~iuZ>F z1=`D5JNr96$X_zB#4<_ldmGiv*vT4}*`~Vy%7ai6jZ*a>U~92ZzH0>B)i6!d`eZ9433a~ zua06lP15ARsP?kJ(HW6+5B9yHjQeM5Z@fDr3wB>WG@XTJi*ZVi@C85qc{og-Wxr_1 znXXTsABf>hCC?H5WNgXdRuc&b)ch80m88L4e9)iq_wz@!+Vgw60d%q383fGRTmpaE z?MxW?^0b}U^R0s;eA_QjVep#D2$9@h1Hm03^Q{dVQ(WIFp}wvu7wpopnQKa|JmYD( zN*=Px3rQFK%f$IP!L%!RGvXdrty{=U2CKd-IYkZOCueht`S+WenBbjm!>&p@`c+cH zq(Gzv1S%D81=EF% z8Oyap#QgvL&?DiDg8X%CO-p=S?~nM?8_;P|hXP#=R&P*@*goa`EGv28^K&pys3VBe zjgOd61N;X#lAC+VVo;JbvGOtGVRgntC2vb)iZs1E*M^<8CsO|8Dm?yQUR!)_iP)^X zNL!|wkkZI{)Q=AT4q?Z)@&Kih5&|{bM_`sxk;f`%8+mU`j5V3uFp$PdT&_d+tX-CC zG;}v}B8ehQJo7ZXX8^rv80kC@ez{&VdkXA*GI#iDAb$yT6j2j*Pyp)qIUC5UAkZ$$ zB0X*)NI}LYL5u`Ei_HoSA5ifE`9B}@Z9TQ^CMIlGHlj|D?OU@^myXJ2f6=XxnwRpNZ@&5pc+qfL zJIhk==Lc_-9nGslSWXsKzV3sSHUuNw-K9pRY(uAs7Sh*lHJn_~T!}a!Hf($*i*F~kG$7J5t|x-XPU{4Mm-&cHt{fjrIMH(PguwV^ z!*u0hp%;l*y}~tNHF_Mv*tv|VVKCT-d`Hx)i#BAj|4VO2`9Jff%Z0475mwb0QYc6Ue9={wLb^4GA&+bXM%~3bz!gVRB5y}X@8T% zWdAgW9UD=%^YWQcs+U>`y(k&gTbPr>5O{BCuk7r$h#KoEIpEIR%^j}4?YXW*=we*Y zUWAvFe+ChnFqNyi-jK#`D@bAU8qWHyd=sL754-Y9u&cAzANU_Yw`fJwb3V&ZUPXF) zJAO2)91%E5Pa_iR#L)4Ntg|lt2`^4E(a<@tIw3R#JuYaZNdD_J%yK&|Hdw_S4b_cS?Qtv8x7M(-Xj06DKB9{0Y&S%rUr7(C?8sSYZm1)`4?hR1 zL?Te<>{i>Jd@d{rRCzm^g4~0B_Hd9#fkOmdDtSM1Z9k7g#B&{;z~?1*2GFA`Ex)wr z^Veo@wENxz>G{=T^vmFKdw(xvtGlZ=;V{m8TDOoHyC%N_K|G zfbflyT+n&=E11pc>S{FfgM4izGQ;j$^M|5->z-ZTK;O zK9dGKoC-#NQK47Gm41fqbidI2x&1?YH8X( zB*b^GcQZ$vJ9?VyOP$@+0~K+$_`BpN>-}P1HAUv9Aw0^=b*FNw*l$}eF{7@TBa`$x z+!=j5yT&=@=~X9r9lx9()*y}`r+ObTMN+5Bc937+Rx!HK5X5^2;aP^gLI}$`bhdlF zMs@79s5(78N6-ylFyphloY+=5a)3e&vfBVGouA7XIGl3*&SL5HUOE6xqS^!*a zBQ(7wP?v0xg&B?d()^13A$fO8`g~0+(3UV}XKiI1aoWd0G?MX zu6pGC>HHr>XW`f6`?lfHDI$^rN{kNakVb(Gq?=I!0}(b#KtvD`kQg<(mF{MPjqXNT zHb#dap`h?3^!M)l8=mL+JokNF=Xo6XpUY3};OV7#gzjrR!Qr~A?fWG(`}9;|x68{` z6{SF|1_Y%vw}^-v=swzUW+USu&p2Fv?bIA^-En3XNTW)Wk?>*Zq7rROTqFEaz(6J` z0)wXYQ^{F4%_jm(j0=rAyWi!Xq6W0zIItJu$lWm`B=Jz|3;`G&(XJDUu zT*qUpQblo`Lur4e(j`8_u`7VEgUponKMynZ&TB??0VRf7f>+JkggiChBJ?`YAG{o5^aC z*?S@3141kAZ?Z8uViUsWBxd)tobo1Ym7W z+X`3si13)0XzbsOGtmP2lphS3WcGr|rR-G^t`x=2cJ^?7o@nwr0};|OeZ8x!o-A;q z(Nf5pAiYkeAMD}{k$oT7h5Xb=X!V3hL4TN5dfL0pE(3Ib_3Jl7aOVSK8{IpqEzU;riJ(8WVdCYsZAG4*%r?8StIHl(F;>-LkI`XlgVZozt z>BOtVNAB`o^GMv6XNimwG5PYQn+5(>MHx}1zkVZ5kT+^XTWJTp z{S@aA|Cby67>E|_zZ+-iEcW%k3;ju2-hTiKv%1fR+}hexd2PNzWXZ%{D8Mq4(TtHG6I~i6NPpm$KFk0`EvgJSNhz%KJvqZ0Ll53<3dV*5Khc+10AD8 zfqxqC%94X0jiPVb=CCkl-*6)v+hEjF;_$NCDkwq2k6l&U_{hS0L>AD3letrEurGCCrZXBun#gr}2v zww9d!`4j?LbRSZxWsFElv(Bc1d`=|3+73StP=S=xPI3e*us0@Oy18;Z3cWKnTv#E| zo^$N1@XJ1p3+mb|$|`p?uD>IBMq-Siq~2f3dChY+xAFcjtQZStj zN>R5Kh*kY{?#iE>nFqgE^LEtBdW||eK0Ag=eewka3KU*wCjA#ZmSkG#fdEqZu&vcR5=B8ZMZu~c{^LzG6`h|+>$X3BPIF)}^W z>Ds#MdcCKF%b8G$@kG_Z+G;cE>T`p>LU(fu>^oT@(6fLF=PUTMwMeEt^0r4A*vLVb zifJicl@UMXQYhEcMs{oB&t2bm!|6j@)3AQS+Mlo?(mf&GIAyc}Tsuw^ob!~FjnkjZ zRS3yU3L=Yl5*$t;~u9ly^(eikyK4Z3Oreu>+6_-jQR00K& zlhVCww|o(7((IrNM1~v%*&B`Vpsyf){`i4=Uv9XK;EmQCF0E?NTp z7HPDz!@WA-njd$N0M6 ziT`we6ReF6E>RafUVjM>PP-E3w0uM`kz;cp3${de3w}?n{3nRUt5|yW<3sRPr5;DJ zMe4sLzP3o+c!f-oWfcp2Fng}BcXvB|1$LsRCy}FDt(|ndcp&_Wsp(d5xI4zws+q^&3eHF9)M?>;9~loLtw` zJkUBDD1aBBZyoA!2f^qyS>uf3HjNfLJA9l3@a9*fLimx#p7-6&UNb%EMjp3RhYfH8 zQZNp-6&*{IB7j_{AH6i~_(s108{a_O_ScU2!pQOpl%u;A+<#0aG)xWXW7$!Ja`nGw zU?390-0oQq>x=4y(v65^=}2?23fM@#rGE%;w$5bf#38oL3ek>E(V73MC&ss5mpy=u zUstsBOz15UFx^D|O@L07o=9j-hl6XLJIgblR=9Wt?dNEoqHK`?kL8*l*c!F_M7qw{kMQNC1H476k72OY z{5;exPRPq5_kRGAci$#tM=QE*#29TU=M=eE@Q74P)_)>G@oG;9b$`eK&C)j8iTcs^ zq{Z^9GfY!-L;f+n5_{<*-Ugx%`XQ!W@sc;PB_dy%xc-GtyYj`dpfiG!U^jX25 zJackw>kRck&snW40>uO+Mng&tZY9P>f#=wmk-*9q*!k^F89Z)8F?Y5|anxrug$qfd z+AKou>W{czDC(7$0O@0FO@e_ZRTlCnW%$I`S{57aNakZ!N}mxWY??1U4VJUB6=*Lgzn{)@6)^mXhB<*hnk-tF zdFRtAvzQ{k>In12ym1&CehIX)_gEZ&d6I~*55VpU^xuAi=SNr}=dN*EYl6?T=u=0V z4MkO5U^8fmOUn;9T`i7t2AN=5+$((rr^SY|Kd+|V)w)!0jEk%j2zKi#Sc%TlHe}ZV z-^uvJ`V2xC<#BFj(2htX;r$fc*9(Ae7rWKZ=6Fj!SON)V@(ZjCiqHazB@mNttL(dt zyBMTJMjrvwmRL7)4N|<9kb_ia1atCOdYEUvNX-&Tp&4^n*=C91S1L^}n)soitJQN+!?_W95%qTkiA-#fyop zemk|Qj>6DKeEnNb`L*O{McIOk2ss0ADl!gjZDg>s`HlS}{8ASA@aWLSWc;M^n7cD` zWDxV1X!w8hVCz@Yvipb@Q3yP=*31#(3oD8}Q7C!6a zYeJ03R7r1~8s1oRK5v zZKzulTG+xM%N<9Y7nF4Uv&J`??|r5W`BRs;eOS z$=U3EoW*|X`A-?JRbxtqLXZ-P!5DLMi2er#{lpQKCPAAB2H(C9qgCY{_Z?dsarn%R z4$-Y($zmn4W z)>nR0QVUqvK?<)G3dl_#ZPyhN&7a^UDq0!>ag_I?oQ6SfzA%)`` zc`!8%k^#tZh0T$P5i|egu(>?CnyBauf%_mq@hnQ6wT)j$88)RCW2(LsP_`-HvCpUA zdYqF{kHnl+lO?awa+cnEwVj~|oMg3aEz zP>*leplo?DcPR#koMS;u!sE)cQtUw%0@=4YJBa((Xx3|lRk89huM+B^H%C$P5zgu* zzceM0Q=&heXJ6-3U-XesQ8wokl&iafpn<#4a1MSR@sSRz5L*bc$24gJo(Gdw570s! z^g;sej3D>#VZJaullr_-60K}cOx}|Kgg#_V3bde+BNOq5Vj-D&pxJB+Piwk|O9J^E z%zxLP@4peaj+{}i5s!oOEkIu`K9+zptk?+J6ZsdK!UIh_4|`1(7ShY=F5dgM1f&0} z+dDZKqE;HnWa^o|t~ew35Hn)T&aIIKUa6i`*{K}N>G>2Gvo~e|h-gu5>3zQ)3clJg z@w5-Gub=%i6zC~p1j$SY-XfEeDyVgXgE__ob1u9>O?~8oKkK1C%IW_%2>pN?^yYik z2VgWHi5N)~14HM_Y*b@BGHlD-u3QtFn<@crk1#ct#{5)d^7;AQlbMBG3Qj@E2EWhr znGbEj6+wT#AJ^OzK{PDA+(gU4F1 zl_s<&iy-RnS5MNWQLP90((G3D&I~JLRV)QN=H$62(jmU(hxX4DdB^U25)XI9I#V^| zAT2iLOL7Ea7+Y+NefX0la!J>rfA^RvfwF=de*GWv;eA`U__SBnJ(gv)e++)LVFN?= zAHH#Fb+bLRB4UyMJ2;(><3`t^LW&fH1ex?B*)uR9 zHQ$5f)zYd%j!wSr3xUC*t+q^&fHNA!|36Ifp>*d)DjdeA%>7{B2{*9Iuw{IM)0)-kJL%Y0Iku zw5okPKWGm{n!0cgPna1~{D(yzidUpg#P^qU$%widU=(j3Y-e;O)Y@aRUo z7in=@wXS;(nlm9V>k=uGh}Ts%)1?|O_5V*u^i}voaUZ!d zA=d{$Z{6q40G6k$qoO77&4S(sja$cdHb<~bFXP7(FL%+2hw2^l; z7g&8-wSdxg#Xc>!|Lj4Uqj($wI zIhmgL=?MQTe`_u-pV}EE(b+5~@u1N#`gv+e*s2t{u+@vqQP)V2>SDsmZC;0!$2s)1 z>@dBW?pMV#DbAH<>U}`I`I1VsE$fBI^RbKIROIG>u>X%%nBRw9<-QSdExu+4-uca} zcXj((Pc+i}pO3biw?%!RCP%!h0FWywae0g{>ImSL*I&0SDj;zeJ!@>Q^6WdOULTwv zR~Qy1ey?qnhm;4soL#b%lyJ!)9tC#z(?{t?f^1P-+eKMI@_)o)$usxN`9fYO=8(bp zeXU#|`4mcki~%1P23f1isS0Blu-srcLPf-P)Jn#$V_Erd!0$g@}bZ0Y% z>j3@LWFfJCS5RG8Re{Zpn!2e#O z%G!_jjE`AaFD_iPT6MURzJSfNd^=5wQQeJojloP(_@xkQniyL~hVRzF?qA*?|5>uw$|Ek1S?%mcr=G^j9~hy0nF9y_r~ z`jeQT_x@$$>gwb?rjMc5B!c6Lt|&{O(GSg`D$Oi8d2y-me73Eid9Zp3KR=!tF?^p~ zOa`f?JVUE_kk{Pb5W=$1V%XtO5e?h57B$~{pC2s*?ABCu<9d;?>cM=J*gey}Dl}->9XcOzGEzU`|# zG$%OZEG-3Vyo%mNLMkt~n3$d4eTItA3!G+FFQMSLK6fH(KiWjIzoloau~GapNxM{*sV9mvWbR~HfWN)n zL{oGmnr5CBu;I+8IqLlatSjqK_yh9$|hIn zv~^2xmKyGJ>vK`J-%j4-QIKKZag{3;I#aELxgc3BB1Hr{)~ZWeWT>u!-DG7DC!8ex zf65`+nZd}(tO*#@*X^MWCX?X?qawnaF7_x(C|~5bBanS1d1cZZ=mW^q5Fai}BJ_6b zR4Ltb$#jvtif>Hw$~ow9!yMx-=baS4#x7*&wH*~1XEdDs?g@N>G-|9=^!@k=q1$2< zxlu#O;bt{_Ml)?BRnsf~@OvO#fxHhv^1-sGoh{QnhUEzg6M%UeGG~5w`RrC^=TVA_9DDQGy);XA9bH|{GfO8GS@=H#-?$LV3h0>t!<1sAa6C51= zED`%t)Ikwi(+UV_#)Io*923Z+8EJZJCT9Mb@gX=>dk7ZG-jCZbe zA;g0Kbn@(v$xg;h*0p0kG}3!q8X~yv=Y&+p)>g<+AaP1K#|H|$I}^jBPj5Q}nrqH0 z_w%bNOGIu|x;Wvd=$Wwj23XlVNlRb>vG8tFZ*{X@9)oz|8w#`Kfumg{bssg{}UP&ZM;&?+$lba)Y*qS z5fH)@WN7YYs;L;ka!fogoqV$`UWZBil41Bxw=0)(CA+Au%|mMPjr@(sdkpEYZE&g0 zvtq@>f3_iac4a=^Od`cFaI!^^kAFOq1Sb0%jm4psic?n4{=9*G}w%I0{6N&&u8b;#>CH} zqr`|6`Nn3#%M`a5xzM=F_YDQCrriDij$?IFije?it~2m-@Vky>|F)lx1rP~hlvZhP zD1E_!^k2I5I6vUqS$P8H+GzVPO!d(+nc|J0BoMz6{ zl>EL`4|TFQZjh_-AJ@MQTgEhBUA?=pCO+KFjxGgc1fO-H8?U1xEjP-o>FNV*&<@j8cI zpx**%?fp|?L@b~^w<5sfJ>Q`CyTL;F=~3KMt`8QE{v?duX{T>8Kj*4a&CgIa5-Y$F z9<7&OA6|#pOj~#`7hF0mt|L#q!a*&4+*rcOzukzJx$I8B=Y6rRDbj5QC%t{-3|a(h zL7R~kM1r_k6-fq*i>a6b-{;01ti2XM&o+y5Abms2VlhHx9QKJXi6PcUiCy#=?;Jxy9 zGZenpcjM1m!uOONpQY`LZxRZkLZiGxI7XE@Evo8#{E<%+z9WNU*rS#=YAd^Fcb#Zy zOfU7y@2b*eNe=|x|655=n~*%PqkNhP0G(L-QjySZHjO$ev1e{SB1}05?V`i#PwGN# z>mD901>Ub>#uIJGD`CrPYhjAq^28tZTABOLA+Ek(0;RFk?!kaRCcfdOsps#;wSZ|x zJz!pxC#p64_2(JDdx`tJ{Gzu+ptIGgKUm}OjSeF7Y{sXetQPXX)GwWnNk~+YS-V)q zTI6hJhYQ&kSL~K+=>AfutpgCFp(4=-8naI+MN$utq4Q6Epju7*jA%kpH-v&qulEvl zuJQt`Jvr|#y`X!i<(cR({EGlDzoOQ-#IclSX){`md$h9t;bzMF{Lz~cQ^L*YEPa}Q z@J87VI{x93aBGnituY+uJg>#OPAOMA2yRFi1TDNA%>hC=tz@IEEmr|NjDF`_zYih> zb0jwwPE-&0&_ff&kaxvI0g=h=7i;p2;Sl;Gz%-4WKltg=G2R^;UbE|LB>8T_!73Tm z^#_F|c(%p#-~=Ig38B*PLdQ)YQucjKG;P7>V_w}5eh$rm_D9m2u#Hbgr=d?G%gK4} z<_z55leEac?Rz71$~d_3k-ZyvsM`#Q_*FjoY76+vagfkRbYZJ-t&)}(=XVBr2 z$StvbKjc@oAJz~0v z)%$#!Dv6Gm1zQkYLSt^nAfJR6KvwYChuzvg5Vb zpJ1Jx`TqY3#UMKEPr@JSVX}+lgX*8a)WZN* z+G-{L@h<@Oz89$<8y6|EQ-3v(-EUDQE!0XD_9Rh+J)!ISR$8J zJTesxVl@&bV3UVzLekNsQi}PmfOi92!VEDdXM3x)jYu;neH?Gl`oOy{F~=KK-un?1 z6>E$KHOw*76h4==kPo_-<2{mmU&O!Q1(^l431JJt~3P;ZGunDAo z!`^rUKkinUSk{fs2JtVOZnT_lQ!_bwodN^i4cNuISYCqgo2HXyOCh6dC zZPVhasy((ePJLNY*ZL#W>5l=?Mbbely>ePNAB0(&aLiit3~fmH=0m;|kW<%L7Ttk~ zW}8?)E5Iux_DUsF=z{5k0hEH>ok?dWYd9L`T*E1t`WUL*CWe;8W19V$6bVIe` z5rI~tG+mz+{iY;E3lA$7Q14YdV_io^obnitaQV>7UkL;B>Ruz1Lot-AJEFHG<8WI) zw4ePxz+cCEJ0w78JPTGAcI`GW$?hKVBcE`tqL_4XQUFHCQm#$6Uq$c;pXdpdY4YE&jHJnx z2d3_JLrgR`e<*MsOC5&mK1cwR?9tzyL=54f3nJ zdW)1z{d;Dn!Snk+-p_bBSr$v0MP*)wo~{toPaPhv$JM#n4PV*nKRZ)$B^^F1ZlM|} zMpSw5_^ncKP`=C^cQ67Kisk5o99&mxnv!l4JE3vbz{zU5EZg7!&7FB@49+&i52ML3%FVPm*k#lFxF6V*uIW6C_e-=iz zAE9vS8s?iv`B~Jsb56DyKWAf$)a#`t`um*6(p)p>637j{T|AO`s3Ye2_F@C0Kv%noYzKJK=bdq986n<3H~kvb?F9j?l#OR+@VRgYS6<~i zXZUKa`lTj!M5gKyK@2U%H#McHrAZ4i>DnxyM03uhtwBmE4cYtci~Uo=65^@#%p1-z>Q9g?)ybUoH^& zM3DAA_p(PgSA&GAlh0UULeVod$BozK6G_Z251L&_Wo8XXJ0(`5E9Qt%dzg#6eto4j z1^mK3v^MMT7J|xcYhGF3V((ibqWbG zT7*_?TsC`QyYhUhF|*BOJ4>kpp7Ni*?V4D-wlonJ5)t-8HziH!nXA&t+oDqFhCSOE zmQi;w3$ebwsRYsr9#HDvX$(Mjka@uzVFdf>Ewvh;^lk}+rfz@|)}tiuUn@H*;kr zPr4;<(nmc>dDl;Z@8c()%;;1IJdk98@KpsJPv&A!Q4B8lezc{s&=(=q{D_tr`nO3{ zT#vaY{|A639v?3$g@f2hS)$(T`nB!=bgg$uvb<+<11?(5@x_h~?>D?!sE)S&p8h8x zgq_#9l&Gr$GEFd~zL&KXd#b*|-t{%sfi!j%&d)i)xGq8uBI8dlZ}6(|Y@#brpt;9= z*&)OB!9r%|XFy74%rp&<-t~bjp%~+!!*uj4gg<`>Ifq&jyH7d3{qg`yo_~g|u50xl zDqoeDHQ*eC&9*iU*HAK#Jeq1BOCtRQM5OdJE60EBltI5{+@gFamT)ZMiF zyq5KKjd~}^<~DW@5_6v~Y(YD?kj85NzpOVLJB#LZhGilOb#uo8XL~=w!XyYBU+aWg zUR^_=b-#(L^tPKEW=1_OO3tRhw=OH5Fa@Xx>ndutl`F%Uq<8wC*3Crd(XSDzYdNcK z{fMtwImC9~E%ClN2ov(dSs)_S;S{Fv2c@9!iAqt2is{K%(=sD6s@E>H+d;BtKS_j6 z7J58lSkIuHtdn@BfID>$c||pquY%YR2|a@6J2Sy5%Oipw_m`-z5@!Dtw04 zD-4<1et+$T79JY7`bdlv1&cnC0lrzL+Htqv_oZOIo9!CJ#!p&$@Hu{aU!9e9HvxH1h^Kcyu6Jl#?(!@~VG1g8`zZ-DKc{1YHR4n5-@T%t} zR8~%u+Kq|TZFC0s%)i6MrsGQ2&`|lzA~=W$X}pr~L7@(Mf?i(bZ);?Y;k&S2p$hAv z4qqq4t-K@EaY-vjrAgl^xXkQq`{2joDLE~w zKt`EZ$OP9;B@iLZbL0%%SvoQGEc@k!yRJLQf;V*iwoBBO=R2vZfXV-;&r`X8czcDw zb`#Z{Gd_uBxNrFx^UZ*BeTI#Y1j|qd}-NEa@Fx6w&377M;k~R1-f4oMEdvl9I2E{G^uz?)_1$Q8i-6qwz%WLdaTnMa%i{aq0F5ZYh&8kUqaaL6(^X*=pXu2@qDF~cd_ zT^M9oxP3d70@*3hp=tAp0bHQlX7;-Ql6itOF`qTazS)6%4UOZl&3PIPB|24nIe9ie z`Nh}5S8{_fLG36PPVeeM@1`UW`SHG>X0>vkH?j0T#$D?!h{%53n!smkJ#FDSP|rZz z|7i}u1im4VHAbEW>s4h~wI^crYi}VCh;R6;qiH{4T1Q^d%~~1trZ3GEtmhyT^f6-oc6fLBZ`8R{Z3FQblTK)rt7TNmORv!xtO-N-qBXL{89T z^)LGx9T^UKIMXiE>V#~&$-OOaq!w?6gC-CC&3p-_H}l#pUQG5}8=(Fgg35d@dX7}) z3h3o0es_Y~)k_DSq3U_I$MsWUud%uT`R)s8$LDwhF(<25;htKU}`=ZpP^e_N>mZF@(L|??iG29yt?GxfpPA~MQvqzykn%P=P zdwep0g%lq$ow5<513|i9%gkSrrL5eHFAUHuZ8Y9IRjsZQ8&j$OTM|lk-rmQq%lVXk z+iz#9-qQ8h&J5LsTAx*`Em)h9FN>NifA41lPbS*?4?6joO?eerdCLOfL_cFjU1X~#Q5<9XWG4TOIabuDg7xtwnWHfc!D zFbwh-<;-Y09DH}v0YnDGL-IWDxe$|3pmN0*83McPg{0*xx@RCVazutoeB^L^%VL$! z4}yduD$%R^I>_A4ud=@Ntt1-lg?CK0^m4*>(YbODaPl|fzOL%*jP%f&d%60qc7e7F zrp`skqiM5#)|b!`Vg=3x+1+eb#-m+;lqMYZRk|dxGn3iMWUN*%+>ah?DNB45jvJ^R zKGUCcbv-K>6kL-cjr^vX1}3#%54o7tg9;1~Li+V0#+YiOIZ2<{6P+Aq;Z~!-`$qnz z`#qKi?n_;Z^F-$`OJ=5AG(>=1{qmqEO!9QuP!fCK@?D3!wdfTat+y@I*-ab&4X|gP zq`be!3j)N95bEUI?=Wml%VP?!PtM^RHB2=AY;-Xk+r_B`6Dd#3ePHF0xD`ps$vHVt z4fA9~K8;M-VxQYS;HQoLXOAGeq3P-+O6B8Rwyv{5*n{WI^uPGuPZ)0c8^qAd)CFUK4$& zF#nw~V58M$qt26A$E778D)C*bNz2nGNT_9h<=IL$rAt{*y$uHy7b$$LGl@U<;0rqE zF?DXJv;7Ej0eX~?*T`9dj1UZrQ zsJFvP2q}$?KEWEkbi!*N1bAb7(GhHS=qDP>&q$Rt8KhCtXb$B*p^$pCU_o86)sL>D zg5S-zj?3TtKZ%+86WXx<14Ojhy=o2esoMwzom6yr9_yq#I=Wi;LwOivdIK(}wd|`S z7$`+&iXJ>qRTYt6$m1mCVOC&jil^+~RfNJ7(U-DRlp!&1tYeHu^H*E(t!I-t}_YfEoEVUMy( z7F(~6PG48)D+CZlQO%2zr%()j{F*_`kEMEy-OB*-W=nz=e>FxYD>ZOGab;SG%X`BE z@N8~4%kOZCITMk$W0UzzCpDcBq4h{iTB48MxU#QmJ84S-E;b4~dGD{9B(c7kntA?VInW(4aq<1%vX~)zNa{o7CNV?)#FEwS!2wB< zeG=+kHUW%fhmE^C`i*u-f|wL^GlK8M1b4d_cF$p-dkGx{odPz5u@+kOWI-`=gtjV+ zbnenzgEG9G-47xHQXzJ$D^AEQuD%@I&aT3|b7WF&E-pQ0J|+-cuZ_B+C8ifFMLCu$ zrs65-c1J25Zm*JL?MD?vmNqiJ0t`@&SsUBdW5i8mrgFJukMHnO!2w)Sp`&F<%7p&tnr20E5uuFm7xJ48G<=P=y-fGQE=5;z|J_(TYC-WT#@JB#8abG@j5Whf9%) z(?yM%%^|f%pMs7&I6MHa>H5TarDIAH3{bXPu0G zORBb8!K&rm`doQcWXAkyvtqLf8lz~3W!<9(u>FOwxNC^uYS6)k*qES?6qCUw>xaZT zrI~5^Z#$3@YRl2le7)F8l)ppy6GL1fS*Uzpobt z@PQBS|V|bWm<$)Oeha z_>vJnn3stxbJfxW0Uf2TX=l{qk?v(7`?h9FJZ4-C6Ig`Wgr6rop5UjrliK6TMd=Lg zPa@ePtfugwl5WA`WfsFr8{`7{6qe!O{wk^f*TmjP89g&oWCGIj@A2hZ+~2mF1WV3% zgb%djif~Y$=@_R*O4mnm+-a&dSBXBS_i1fSQgJ#79=}4$ClGpdP#g z?Ys!^lb`!}Ui)z>1>tRJ8u$e+nKXf{CPr_ny?=HUADl4cL)*bQ%964VrA+?LM{K;O+1Ow-)bsF9W^OX{ zdp*9PdH#$^D+RA**s)5w!{U9$3{W0iKwNQ||F)k0cPHaFVg0F*ZdGd<27YNpsCkJI z!%zREZCpKMBp$d4wR6Ipg@Mi-c~ldkoPI7O$r59<)_{@eDAcc^;6R{hseg9yoeQNI zfh=Im3R015v`=_nnWPdGhc~~m9Kj73H2@>TB*)xF#lfac7W9!M(_f9$|2*07oL^%_ zdT9l~f7uQGGnE#PZtQKCN{!-IeJI;blDi^ld!u~&7y|5hRJ0!$X3A7&CZZ&!;#Cn; zTib?OUWONFbE_R4d60B&`@m%c`kb<5SXMuwD1GlIG(09ZMrp_C8#ZshD@{fyj zR#Untvg+VUQ;gp^9}kHRUHEb8TWy6{Y@G?E)7VQ_y6Ay+;x=c_$)6m1NF!Mrdl{b(Rx$f5pi$Z*7sy%=Z$0TCv+*h+c1i zWXdz!pyV=BCM^*!s}ZcuG~|9AXr{A|GwXfcetE1`;SS}+5j zWNB##lCQrCQ}3#)c%dCseD8hcyc7-5ka9Kx@bWu+{+QV5mS0z15Gc?GGg^hz+FJSg z?LIl%Io(ErC0e%u44_x=*^1HK@b0dRR%7ha_4R!v#fPs?VS#7(v11RPiBM~W%%2n9 zZf%jaL>?#he)f(f0BO=BL|@KgF|{KEEbpTj3A-pj#|G0l!6F{af{XH% zZFO@ExWJF;5T;REAp0Zu-czJx!*_L$fC{u zZ1SSpc*V8-J)dnRA2?ekIm`9k_~$#HB|&%QLj*(13ROB%?JRQm*p20L4R?*^WJdt0 zX#VO=-Np5M&X$BFh)fs(wYk9Bt)Q@{ptd!PHBk-LRCpIZ z_je&_$)K%~CflRh-F_+hsxrx@7NPMnX$_}MZqjg_CK`4;X~8HH9gu&b_5`YP{TJuH-bPWwyH-gx%FVaG-~E>xf6$=gleYL+pvffgks5EzR!C?1)Bwof=W)tZ zL2vzCt+aKZnU*-s9%mq$Xw_riJ{$k{^OKLrPhGHyu@Wwr4Cyb{+^QI}x9;}59sCf| zJ}%?><@L=zPt1-h&xZT|F?1gMY_MG%j=j|urDoLLyR=1Hg4$bX?OKT))TmL~5=szT zhgEwg_O2CsOJdY)jc6$zRej%li2FY0{LcAbSFLxtF~isU-4nL(N}!&#aoj)4GvHeS z?Z7r7T|-F!wvmC8lUHr`C$xVo7XsONR+1j(m zTJ2HmFZ-?(9r94xdXlfcgw ztU9P2d2uYq@dWgiU!Kz0y;+jREs@N0oraPnV8t274X?@6f}<2_%>7q+aD0BeDdZ$A z3yb!d+6zIqSlV{ej5YlyYkoBte5cXUbv5Ndy90^PKIR>h9}(icp;wdp6YKzqD8|`( zf{#=D8T@pK2zle7pf0u%p%i8vD*qGdQgEo$Y8`Sfy5ty4u1-VA4~*3%g53GkDR+*m zE@qVjBLc^igGz6Ko(^96;1fPKv&J9Hec5DwBlt60Wg-Kjr!}WS>$izAZhRymUKU0U zxkF}gYXV3ETCA5kG{JMDkSA;oAB62#jcQwv)Ww1$ZV{2XIqIIEL>|F- zIwo7#@Bfbdc8oB(&ZiFH39sr?h;M)*D4w-WM$?B9A__VYF5Xc?kL@PK@@|hXdbd@1 zm}t}R5W<>@`qHTb-$VwB-xnaP=bVK06tVY4K0WZ3Y%+ilSN&sh*8Uz>N@uKnpXCL= zp23e&2GwX+SAwa+mew@sbX?@iV^}8fsE;a95YU<3|{`MpD>~xUYXLIYRr5{ z-J8CyVk6d5&*t~{3=Hgl|J{hWxfDU>2PQP1_#Z;P_ulPTiIjj#8d}Gfex97k0Y{&O zV>Ft;pABNhpjxL)SistzYCk(mb^3#|Y@R#A5u?~fM9MKjg{N01ou4p6-gaWV>hAQ7?~8OvbjSOHq3$z99Zo5Lh=0HYxmMF6n)7(7Sox z8d0RrFn&P%_gw1ukfa^hRe%O^_WSw{7b}D`j5nBQ|lfoHIfelrmZe5LgOGuHu|aVZf;3(u1a;8g60poP zj>uEuQG8)$zGfa^N}Q-r2T+A=7VZe{1X~?XgrTNfBO|j>E3esyzEq2iuIddB7RP4y z(UXBBI6dTIb(<4=U#8c|tr|<)1XC)3-5Cczy(cnrizaOs{&~5mS5%N)b93)`WLWQuX#Ny8jS=MU zz4iis_v3@(vbsN)PwQ&YBh!me>sj{8n5YqYy}$;tP7!8!qNu@DoS=1)Ru{FeO|o*E zx04LfI_zzbYT`Qffc=x-iJ2Z9p;`a>I-jdW%Q3- zIgkzdmy+~O#9b{}H+m!IW^s5QAB+B)Wsb&p!Tv1l#AE*0r>iT~$117uOOFiksHL~w z^xbo7sODEnhRVw5*|ETYVQ?MyMEKu2HBQG6PiL~Ll^q0mc$$%P4O9d&HBFPJz{1k} zXqk#^ij! zSB>7WeB_G$aysDB-oYPCD7kCZz~~e>!^sqDK&7VxGuDu)T_0ED1-<@aD;g$A&R6}w zR5DfE%;3sU4)oKxWkJ?ONyPE$3bVmj&%93$c8(?IKuGfao|>T}&?0gF0~|>(t$6CK z-lmgr_2z6s?B__u_m^IUQ}7&T)VSHuseVwz6#cug`C}huvl8&7J9}5^Ftq3T3W9-1 zT?rd_bRuyY^_yH<910zTVUy=!^i!|1Ws<1!v^8~j?g+IPMXfH6Mu@Zv+q$Op3sJRs zdJ3W~5lOB(#6X*=BGIBFn#l1c@Gq`2izn+WjYEbHj8~?p!IqsBoQ&+*srzlGf+?el z$|fGn0s>0HneERJ>^Ji>)`kH_m#{Iv7Ie2v4|*WP7MF^ex3Lf_}F-4oSoRAMA4rk!^3@>J$Y=F8qXk3yEcmu+~Ww$rjgkjd6tE z{bE=d_%g#$3;8xdaUw1K3yuzPqFW;HAYwmQHGqP?{3yr#e#M7CDI*Hwi-Iy7GAfbT z%AF6;Hg=^m1>UL(%)%j!YU?mPA2h?B2R@j*o-r~GJDYuwgZDjwO`x_Y#9PzW=e`jg z$jFkM#wTbcI|~JJTimHB_{8qY!?WA3O}>|<%2afpPCWO{Q#0b>P17U9gI6MV6!-5q z=z!9&5qHLU%%dvO6rTrJOWXHR4GSbB@=%#@YpL54rjRw<3SGmGvogsIo88&lBYJ|Fz&b{1U+%j@s=bE7g7X4fWICWvkl>XaE=PCh@LyCPjhKd_eU(2c`;E?< zp-$6#+oZ%fZ1%o`+vjJAAXz+_`eaL|-RF3_M_|tsnV<<^%yFJkmVZ&2BEdnPWxJb<`< zAad0*Vk@H;x|j-I4t8kQ&GWJuF2M2nxk-ZS1xSN&%$=!Mq}5aWF#Y4;jr&bofC z9>_!Rg1*i1^EVT+ZRrJm0tBxgR^=#ZGSOtrSRr_nxyM=uAOy8(MO7@+z5r@gz4EQ7 zM-@Hf|6^wv|GX>S;5-^)YAiEv+*Mmwigpz%)^JVM)R@(_Lcv3&UyC@a1{H*hQHTNf zV*7stU!;h}M?${9gxisJ{{uuYpJy0kma9>p#LDYiCiN#p>-{;U1krdqbCJ$Oyi^+o z8QnLb*9VHGV|(ZS?oG{ka9=(P`59g2LCE;`9TuXQl1|`D!iEAx(-gtw*&8jHCV2ae z`LmZQ?%1M(XH`Guqkj{-pX(L$bRcWNGkZ%cZ>41jYEuUg%Tak~N`dF049fc3=mXq8l zv<->hZ5@yG!K1919|d5GRw;{Xf;i$5hbNhYcSC_5X$LBK*6yb3U*DRiH%24LO-vSD zOObf@so=Ij=@icX))u5)?Zl=yyQZ_afV}Pl566xZ-Z$7wFpk-=&Tw}9cw@T4Ng9pW zTI<))c(Q&%ygpeygoa|OkV{hP%Clv)MZbl;6GXD@O0u(@nTFU7z<~JnL64l`=U?{d z7#sR>X+~S|Fa=2s@q+PFTB?X2*?duLSPaU{~!R!xM|du%}B-{pHwe}Uc)$$*5NA!~)mlEMU|YZ%=B{wJEu1_mU1qu}tL8`2 zeu=E5^rvJSfJ)aDb&8La^xx_%PF3a;yNVs>K80amlCp+BW&=hvO7mO?X6yvEjqU$o=5)4>sQT2RblmC{5iLc z{OzEOg|pyrW6G==NP>R6m0$Lvc>i zLjHj}n_YCd7blya-m;)vyw-$*)k)p=Sl3y`kV(nO9UsQeG7Le%dO-*g0zJy#n&)|rh6hbRA5?QytH2A|%7Dh1u72`% zm7Nv%@KB5ry`-}U_L$mI{OmKpH=As{QGqSL#Qiz9S!a}=Oi8JBx@a`qqqbtjkGiF9r#7g?u1HL+ zeWbXqp+L|_Iv6m{J36czF(FHYr+q~}>#;WQ+qhw~qBh+?Y_}%txaFN3b?$sETgYLe znuD|i-J=?c1NFVY^~F$z2s#I{X2D9!V8@qEnqC*3r?+o#q=dX^&(i$zXr;#U zqY%{HjCfR_v^MQUn=4q5Sx?}E=`|OROg6Vif$)+<*dVB{vz;uA6%%<+@tFBhHV8nT zGzG>9p)tu2O^DQWv*5eDRUYPDe76-*g8;^#?!G0^t)=^9pMQ2Z%+bWHQ=@2-)Gwf( zj;dg2EZ+`-F<$qSTB#IjsUN*FH~(zOsipJ#3C6NGr2A6&s?Y|#K3TSqzd5LpHT+E9 z^)nYz*lHY((GC|YR-+h?XQpNG(VMz)J*)kc67DVQWS{W(Ej)H=#qu(VY>W(@ksRIS ztBjN!k@xj-w&a6Ng8bZ8IEZGvIlS;~ByYN&1(UH$Xj>cDy;#HEgOM>8!6DN|yul-_ zqo#ClG~)_IpMV3<{`+#i5x?h8H=voyj*WrYDwr2__=wujzp3`$ny#fW8{S+NVkeaS z@=<7}^eSv0fp(CE476Q-z@38}lx`Z~K!RzqM8gNLjBgKkUkE5xib+0jWGVV#v2mrH9~_YAFjqPxR|!=+CV$mNBXtnVSAu0~Rv5xFxMv~((;!aZ zA%Y@HQcn?fxR0;6>V`s>KJV;$8)&xftl>I5Clq?ubX+&Hc8yDhzKMPECikg~fV0|<&I=oa?0hnF(jGzfeioC8I`|Lt@B2!Vu63)W6qrs()oOuJ0OLtAT8M= z@=-XlSFz~}rTi4xogV|^P~naM>)yI=rwgUc z@SS@Mp#om-Qb07RrAmyOG*TEr!1K6RpT4FfA}`)fhFF$y>_i!N3pHWHb+r;-npzkb zt3h37fyiDXo}WHL_jL3+T{f2DQ5iN45qSNLU+5(zs}`G+U~kFVBh_QxF%Epwx&vcP zxKM(kECdkL7prGX3LS-p1}cruQlgY^v6VPBxVEVSL5(9%0W_AG25t&1XIaEzg3@ua z${ClfV?}O_iRl_G^Q|@4z^;rtztddYAG#uWbsI+%^|7*?6aYG`8mg+@*?^ITUbpjW zkG;(K!SCL>d?z_6;*6eb*xuazsNeW`-;>Ce*UhbY=O5%9--#kOT+%0+hH7X9H(5-G z)WFozcvJG?83G>KjN)BILmpheFFaujQWF|BUDt#c2i;qEnN|XYh6=2P%#b+D9K>4~ z7a_e}#wq%<(;hOi0)Sf{>}k3&6}liyYYxJ3%k#{m(8aL%F&x@Q6Dwb1wR85myYzE` z%^cP}=X)#M>DhILlNAEAb?lBmUtHga{$TVBr>Qu1<@dWf1=H?j}%; z25Ad504QO7Y<~wsnpTJ6VjihXPb(_R+}#`eTV=~=FpcfI6O1lVsfHXpu`ULezxxL9 zpqhJHtTm^xdAw6t9J`b8lGjMvQ-*ydck5KP;@UBF7f}UVVHpW{`USw3F#`0N$Js|# zc6MtXbm0H?3aM9^uG`qIV~sl>9@;KCU9gTtge;ro);?QORdh5GT)Ry^lof@n*wu^^ zxnm%jcAzIwnBcZ$`bm1iy0C03H3{m2SZPIOFOyHy@Jot&Jcbi$ULEVdcVmmAOUeX|K_0 z*kV|-P>_XVqrY)0zW#YT-eig3r^e;rv_0Wy9OUsJl9Qn;wZ@Qdr_PCXYEe{=cFg4- zDCeG|j~J6Ok{U0TJMaY5khPzKGtZ@#q`p-r;nI{^^KUw2Xp}bY?+QLrK9l@@sjF@$ zc)cwAX%Ll)jEu|>Ym1K|sjQB%$DEgnI0Hw4y7II@>oJNJ(2tGm!`4fi=8c{cWA@WI zoVA(pDzv^VdpWzSuKT5$ZY_0Hpfo(#NK-BA4SeN-Ds_fpdMzfvz)XE#Sbg4H0XMqD zyO*Q{IwOp;AYAOVovjse?$f6S_5)eai6BoMs7?v*XvfAL*b7L--;Z%B zb;q;NxR+*c?Me0FYEO-<%?0Eg1D&J8*Im8&yNluhc4GrRp zTOt`~Ep0M__obm6P+5Y<`|&&*+-%&Xo!bnV6GuU6Ym>pM*lgGjShdMyE$0@Ku1!2< zd>P&G&$qUQcp)(&pD*#uz=)vhutl_3w;X+2M~}Fby}*4H%Oia+fJa&^Nx~&w)AeHJ zEa1$skia=6X+1J-)r{7sfob!MqBv1|HgU49X~YFOne=}b`s*J#`u#8qf8SIJhT8k{ zbE1YE_0de#JXu`foFp%Cj;_xxwj@BHk zeyHb3=W%0DAQQobD3o0+&hvE}1nWz8j15N80DiR*cg|HT81j@4%iGbfTC@N}z?He) z#5G!kbbDO5Sx6oobHt^(wF=z}45{aZS|e30#ee? zxet5{?vJywXxddwwwe`^#UWe*7pMiVM8a|S?vTa9tAp$opUcQ^w91(kQ2#|4)tJg| z)D6;s&HIvmxf*)Em9UidnJI^%yJ&XTbDfv0Uv}lHO*C9URWxF)2*z{Vhn)Bfc_Rl( zjRW>e_aRFe6UKXK0y$~9&0A|Lk-WKE;{>N6;ud?(Mq2PdRq?0X`T*TLg}@LIt|fH2 zhn|uQ!BYO!Hbt!wE1j{{d-?hWDF#O4~e%KmkDEd-Bp zr~#r%r19FC5pF{5J=wDST;LMI&#NZM57|XA@Z5MSl;@yxh|IA~TwAkV{K2vhd9$!7 zL(==822K1OoqaEgIvq;3s!q}hy8P0&A{3s3wTW-NY6c7JaWQP&|f_GBxwfdXgQG5Mtlr8&e{I-I#sY5BO1zlYufG@{YK> zDEeW{?yk2Q%!V;uy79_@tij6tHwDkM@WdPeuX8b!&B>y-d8F(4ya>J9fDdedYqFqd zK9vRT0$*nuORi)0R62(XvgyVogtLbsT1j`>XLCV}S;5wzrcpWq; zm53lB029>nU7TKL#jmMBo+)44k+_qiWi>P9fVe!7vqu_z9?oW0d6FX{NHEZX+Q)l% z(SnTcnaddm=(WFiu~~Fwyy302IBWpl4&yfnn#7Eqv$y#6x!F_qrxY|ztE!rBhs7=l z(7;=;*B9)dc2Z|g+Uy!h$r(kDlUT*Ymrcu&&oSq_F%!k+fih)9U%zJZ3Q0b?XOJLu zFXN1jqgW?H&twM3nAk6ev@2;uY49=F?f9zzPT}COE%1Kp(7qoGCFqf| z%>&~X(WrlsYa1rl-k!$w;#FzW^~TSV*Nf4o?Li7msEXilv3wQO)NLHat+>P$ zxp}o4A1vX$(4>a3_@rlOcud{zsI>>P^v}O3?08<+9_pI=q~Z&^DKBtIpIWr-0ffbh zg;U#40k@$x5gb39kBck*7Xd;)L9>BRl(JpRGD@Twj?LQT5NQ!(13!OvM?zLX9>soD zgM5mw;UslL{S=*^q5@g)*w|*Q+yUi}Wq9Tg;|#vw-Ny(Dt}H46wS0|!f>mee+Av^j zy4XBsQop<7_i3-myO8-RRfPLy>4swT^~rxf{Kr;4c|h;{owdJeDpFU9NUq{6wLzfw z{S^MtXn$54F0y-ou-1(rz|FZ33_ zHX8o({4=`a5My(BdhVK|BI!MLT3g%G?KF@6P|^t7l8}?lu3J*cm>EZ0^m9Te6Wuc< zwNLx{1(EF(l5Hh`WSn)8+gr$2k(;v8x6%cxQ#-8b%6z=L5hm^;f(c~e zM3=ggiCycH79Hy!44&D!+5D(-^{N>~6N$BY5GDE9lDs}&o|ZQbXxYXnj%#;3hDoVE z0GDx=Vjz&-{dH^%hC`H-p;mR)>txZlRF4v+#vrp^EQYkPV6ia&KfjsAj8{BRuv?=Zt5WXeTf~aC8p9S>*d?XhN{q z$n%5i=Wt_*wy!}_-CCC$)}EN2@f%KzA6;NTqvO@!u_BKWOXj|5)=yeM(Dx&<;p0-( zP0rV3^EcC}n;f{m8&L3vmGb@SFqB8q-2Y zu>E=#2fU0^BYSRfu@D-$Cfl~g)_6@(d%rc|mag>I2EYh+m>D&pa`@iaSf6LjmFV5Q zV$3Y2$={^&xMI4**Wg*<2eHAD*~80?rq!3YKX*+7L(8Z6e0LrDoUb*H%s!Hj*JcT* zGDwog_))<=R#uz;xLlTzP6=pyqV(@#&4lc#fmZC3TxR?&YmKq-=BP<%WJDkX7)=pf zhiBadbGUzdd-ZSMdSq>FkZMw?QdyO?YVECCwhhF>D)tn5#qFi8#d($689fj|LAR8- zd!dkeZLfKb8OK7xe4)XKoDvdGaMy=TPT93#k=1@J4nB_D4`1BXOJfh3ZA?~vTes_4 z;Ywm-0YR01l^R!BR?PDd{Xo&$(@vM$kVig^G%Rn{IS_@ZucO+>PNpQg2{}q#-WAn) zGRF(l7Ov71c_MDD0S}@--pWeXp9l(6Hq&SR!5Hb`ZamX=*OHektAALJa{-`eLM{KX zsL*Y!Y1Ov^tJl(VqqU)12DD7wsrg_k*RfGw%?0uG&cgG)zb}|-UutKr%ZVeo-Hj3w zziaP8$iVceKC=!Z%5h5)07g;iv0ci5orHL+FwarI`)?E~$7Rk!5h;#FtZ5oD?r#Dg zN`wHP8Q}0blL4d7_MzJ__w^e*CEoufmi&{7ynI zQmQLXd#3m`BNoYDyE=l0(p1yOSMGy1o#zZO%T4RJq`Zr_`_PvRXHjl zTltu4p13?||GmR|)?rz8+D);|CUY|7h*+;l%;CX3)mcq5D|BY`2%yt|&WIjV<4aXt zDsW4`AOtsW=OBYoK-5SEi4cn@)=mNv#TI0C+rh5!&|w409!KwXZLd++Ay3^mFfuBz zPz2eO-+ug`r8)Jj3{t@Wi`YCKlwWHP&e(PTDbL^%5Nuc~ebTXkg}3_tF?|B_Z8a%L zo~^F?A7FnMW3t2{T9~7XzL1Yt)>+$m@2k(4r<;n%7JGm3uFrF#$S2y=%=m#_%XPr4 zgK&%4C~+m!qO9aqcW3*3#nMx-M!eVh0FgJ!SlWW=`%lK~EVn>w)`Zp!8WDl07axVN z_faKfo{P`)iyvV|nRP~V&4N)(mv*uJahn1txrn;R&S&_Wf89{LNniW2w8d;UcYaT2 zb!b>~YyE1cYEX(@9W?4NoY>U>b+<3M;^cNsauwJgXdEo24jeXIL=i=~_N+TQ6$VoG zo^~=K@{Y8zo>3@&KR}hKDbN7hAQ0OY#&MlKR*C$wfq+No& z(~7|>#&LIBp!It%%5Rr2dOd*CkTV>6+r;XdgbZ&C$^Tq$Q#|?dPs@b0s708A{rdV# z`;F+|8%EtP%W>Y+|LKz$KnkwOq3MGRG%td9=}-d+@5pz~u?-^BrSC_tsTNCfe42RO zUr9a|9KH^J3hv{gTKK6w*2goZY-u!K=P&TkEuX!i%Oaaugs~>Z#?_WKczlzjFx|9T zGMBc&1VXfOY*pu4ilYgUX7y9A898@c`Uq-_Yzl>J#>Q%;$Cakp{na5&XuPaV@fQWdz^#2IzVpM5Ia@Z_FhYF8qy~l2Si79$I`oz;i+|URa zc^+eXz5JJ=?l$3X5Mw0{dl!qh5JN^A510q&==2BMQKC$Lj{XUP%9 zV;FBd{F8tLZCvvEPQ;0ZiZa_^^lei*FV85pDw1x%#qGaT_uZ+|mc66DJ?EdFr^I9*i%gEsIsI*ojDW76 z7%^drA%cry7Jt=Y;1bcuD2?0l=@kV zEy#3TDqg7=z5wHd9pzsvecREe&(v7szY{doxXFsL*xBKfe%L)Ypd*9m=epZz?M2#F z-mA0}hKY5=_np#M1d&yx@@V>&oZ+U+KGdz}!`SE7El%+D)4qqRvR7rleeBSO?&YsK)6j|Gi%bC$c_8W}W4I7Ao}97s?7f zm(~Wc8L-UxuNOoB>&cODcYO#|;N`0=Z1!#m>T zZ%wC#HU9(%o^K+sDeqi$mt_KJ4OFj^oM|&=l+vxpNOBlHW+Sr9BBK=L=5FL6gS#2Z zUO1CJIIGXOES4sm#&fPFt|7+xFuWoz>JuLuq(eFbVnmzm7LB+1XrWpTPY+T;etkc| z?0rhSBu(S~GW?hL%IfA150aOAia4*mD>By}OuWH1a|Knt>-vm1WVGyh*Zg+;?+||x zuSjD0PmVasWOoH+Q@?o*)uYDLJ^MTTu7>Ffmj)oIZ$U~+MJ@sFf-mgy*Z1_GSCOc! z!Y}5-5w5m%-?G)OyB1DyV{q3+_6dcxtGtYHA}GH9HKbi~VsJ-8LkmUO+~s4Sy7po6 z5T4_7eqA@;LlR(RI$~^WH=26#S3}P_XNrk#7yW80Nb%R6v$j}+TbieBj(@RdED`2N zMIC9MK3s``P{5aZYM2_-rYHyImO8BF!r^>@)3?lzF2wwDoGx!MQ)!aD%&yZK?HV8> zSzh}DW#t1VJ|>*T*S_)=*5Pn?_2C^K7Brm)cW#bq1HsJR@C*l405!}5jZj>X_y72u zuSNKFUp966Z#pI)p7`%pgm#nsHTjjnP-U*9h@;#wnu2=+;0zecYm3n^WVpK2=s+5Y zTyEb6u?%Vc%9C<)=XIwDQ#NBKPY&!(qmUqbJkPB)K`RUUlmg4W7F+(HmT9(oS=rg> zM%GG7_9QoCX*NEtCArm~C5dQtO%aXS^Sle)lK$x0j<}VT=nUp-mV+5cOj;#08JGY5 zak>^p#LMp^I}MN5PCTH;bv-@=lHCmY;wbY6W|#_pe-IVEps4V$e_t~G;k!J8Z2-SQ z&X?k4>u$(sacEj}?cP&&S=~I4AW0co5=^S*@!>!Qs>R&}3?_Zu5hB_pBbcLjPdYVz z>aacbp{i9Hps)WbNLJ@X`B$cij#pWjCNNTuyk?T$4ncjZp_z+9L-1C2+dlhk-#kNu z=f-Sj#y&-;%VvDDGoeyV!r@x(KQ)NgQVygR@y$2%%K_F;CtoNa`JWY%H#}Efke@q% zH}5A9sSOKQjIIka$uwdc9iz}P2bs9r{T(v1-p1wbr^iAm0yTDYf`lSYpv9-ZWJN}b zK(-XlW;Ftu$;5pxvHQ9RB;AWEnIP=lPnxAPm$OREqb8h9w&6nACVS9TP*E&~)3St< zaxPlA`4Aud7vnqAi6Ktg{r=sGDhQpGEpi%rfJ?U<k2Iqojrm(AqZt+Lt!z&&BG`1T^SBh@Sz>&}KnXJ2k0Jq9Wx&6h z#J@{L1@Px$^VT0fZ*z}@EnTl)LyT2A4vG)WmlaFaTZDh?#MT6kf45(eEDT{Z5-3Wd z)jAqhV-ZRYC;1Yg%4?u;*OWqOXE_CGGZU-IZ`R>s{iACoVZnS~a@abtJRjV}iMQ=R z0i*SW=jLaDh18qdQo?L)uZSIg>buXIKdGr_|(DrC4Y0IZ~$bwThB&3oM^^Win zj_5kv!S3I;LP`I)6mAN^8DMw9>o%$+GAwz;7p;)}d+YWV2deLg52mI&*2>@3=s5iy zqMuM|YZO$N3fuN%{`HcFZ5#)=+XI%BLA>*^-enYV`U;QUD)9_zbJbo;hZ8Md<1-<@ zThAf)?;Y1}nC7L*+KpA?f41I8lb}`Cq1{qXnwwXl7uLmVrHYa~6jzAp!Km6d6t3s{ z?HU6G^7?EyO%bwXk%HGCs`&9ap0GxqO$6ojUu35~W9ehEww|WqMF*>PVFRsh?B0ZZ z@`H62mO;{Gm_P{ncwtA$p){zpX_60C_0>>z^z!@9%`EH83*z3gBf&iDnsnDYQy^&>SGh^s1BJ0lO4 z76;09Pu6}A>S5d$mzj+VzfuPk4!Sqq&wFo*VPMH+s3ezq6`@;fuyzW=z2pM$aZ!54 zm8UK7NWaMuiknRr64H@41JDZA&Dp{@80!@E(%8H)`;mV(Al{MB|513T8=4k#2QJz} zOp3Us9~WITHNqrr^Ona0q2F=>qA`yEfARfMmm+`!BtLUM$0^acC8hX7HmMPD zM8m)_uMt!{px=b0`0toot}Kk0OsJ}DUtwq-jFFPfp`O=4Jg<+WKeOEvaV%A==h1kP`ynq5-?=x&Ybw}!|LLq#UOntHhK{~UcVF?7po^g_ zHyIF7M|<#yEII4hljyL#n5hEJ?Nj?A#v0&nsrV{8Ky?OT^9y;VMO&)a*-3r5rF-0F z=vewyqV^RpNR8eA#9kPfP2oM>+J5h_zJ3EYb(8Y~UJ@=DD&y<)0~2%e2i}U;Ehtf0 zFf-){e3*lNuCk<_%?;Zz+sV07t>hV@`M6c~i2IMUx%v|cp7fen-uSW!^9Mmzv7HG5 zS#d}@UZ0G`k`7x%fPU6{!B)K}@&O&;?Ol-S1mCSX79x87J|?$DbO7v%dJ%+e7hy&9 zJO2+58r(O!(GMKkH#Cu`D6fB2Fnck(`m#3E=4Q=D@1*w9ckWMZ&qDF7n?z5l;390A4z&as%FN%rP@mQS`foB|8*5`bG8zBPRIHqv5HP(e z(>gVj0zxh@AtGi3A4t%_-5d;eJQtO$BaXc?e*Vhp{H09MyPZtirWRHlN1EPv zY4VOB&aP=(l{m;+7_~EW@26h$!p|nXCAh7^>bnvoPs3-^0xad753&rLR53y7Ky!p+ zvO4ktw{QBEb}NF@#lvhz!h@b$e7uG;4f5(In;mgT%AcHe;yYzv6;EyH6S!k#$mYJ6 zV}RWY&+WJ~3YIM4t8?PA(=$x-OMGu+$lcQR=#~4mf3B_~Ua1@@d@}M~^+YMku(5o$ z2LWi4qvo38IvFQ8n#qBvDb1%Dz);=@S=l^Lcs9$!c?l#UH z^YeH%>+sRqDOb4w2g%FqUIU=p@cFp+c^Il%KJ$@s98QWrrN%m@Qu-^d-R_9$ou<*v zV0&;n*jZ=aAZRtB+S|Y95`H{;pEcBj&e9UuwB|o=J9~cEdD@Ciy@!8Qy5;~zj(yiAtMt|`YP_|_~5>_m*3?u1v5pc$@V0D>9 z+EW!Nd*6SyHuI6E-L<|4`qF#dOFJzalirK?Wm|luwLfeQXraRn(LL4fdA0+d&vuok zPZH?b<=9RGL(7(57NvhB{2JsRWp#_py*OgNm$)A=Sp49E8gO9WyNX7mp}~@C4wLL- z+^DA7X5nULtIRWbsrOrJNmos@Y@)jp++}qWdiGhg3-zdVFV@5>ZNMg!$8ydDo=s=p zK3rO8F?(~}d2{LTd)`>qx3pHGTg7(P`wX4`=OyctaN&8I1D-qE?IZ%pV_oDEzY;GY z9cN~fFusHwo6SN?m);IE2|HXP`i|=x+Hsuc;_5n4e`QQoc7ny$oS+QLDmN5fV-(dmVp|*Zv0>9FFHrmE^F>ne^&<18H}ZJE5jf zRN{{}ksr}~!?(G*AP0T+1LlYaBB}#^j&THWoUv)et3NUBW87g02#zN+ZFyHOTeB1S zqs~f=xRbs8_(^KmX}>;5XoNQ*c6~wh#}Xzr7&bB*_gD!Mx0^ahloH6+CF%yi(M~~1 z9p&bod*0_H!Gr3-!CHvzd8C&`g8yFEhSBr#0#Y0xnpk#>Rc7lN(0olNe&)d}=$}g& zL;!#RT+%U735n_rIxULW2q-UP>}+jfuFIW6Fe*8)jj8k3IezAaR0vn4zlR1Ayp=bn z7j8t+CMaFC$vKe2=hD0&K8(NQRsb$v>%aiQ)@efGffB=WLmpk|TctKM$^xWtF!lLl!V3cEEYZ?E2;6VO9H+Iuf5y^DR&9x^Eok5bA*ZBb*_v z&4VHWOHL}*FqKc5Mqa32ZE3!mZBZvJ#^Qv=TS`86+C<3u9mz8LG(d9@9e=-K^EXP9 z+yy*uG@F?i{yfNO?X-qy+ADwOA;O0}4PvH2q{PUiu!KeU zvkEw!x&}ZXFHu!MrotOqYx8Y4T>(KL|H&6S-bN81ZHMFuEc_oJd~vk~w=QF|kje2# zB%sAOwg$SW8&AC%v_?y#!Azy^t+m|Yo7;u;^a{hO42Vm0YCg^b_Bn$+hhJ_7?i*Xe z94hLrW$xdZpIduK!SltqY}unK`Z&7l@5OS6-sj`p!+e4pS%t)dJ0wDi%#Q&H-rhY1U+I*CmTNLeDjDde^dC8$W#^2rDh(# zakqxJmArXcPTCOk5a^jYlNmwF!+4%T5ye`{WmKt^=5d% zM)(O}_q9$ROlN9Wj2_y)*!4CP2qTI#7cWl)i#WG9%L_ah*xntPyo~Md@8c|5@Z2KS z5DVr*@TR|JH6JQ_@Tm8^s5b5Nw^__y>ipgPTOl*a;sq|4q>6H(y z)c*i7WFwmCa?w?3BFH~;TekUjtA@A4-G${{!;QNbNl}f=blY@eBdeBqs0EP_p}1zu z8y8gHZTaCmg1)TBwF1OR5tNyMJ|BRoJ;W)34RtM~9w+8RMgBAe1sQTv6gsQYQQmQ* zVyIqoy1D#OZDiV8n`~@ww-g^8)^q&wpz}AALhaI|dzw)}_yrkNe<8@PA6|Hu$0SuFn^plJZBLz* z->g49N17R65fJlZbZlFNCcBb%A+h#O9Uom+(S&>AZ9T-W!F{kaK#XQj@WWP}t%Qbt zD8^WIe!i~gVkX4 zI*ueB_>Nzs<~R2gwLtCb3ZlP5f}Bv+4K|2I_vHIWXA=_>@=oxK78J#o19`fMIqv~` zt+qu$HRBH8c1APX(I{hXKUpRS!EdlFJBjAe3F16LoP@wQULN74~09k4b zcqq5Y%i9;Wlz^Xm+cNjUi@i4e*X0G-yLC~{Jhrf-zXbNtpbv`3kUtgcm-8n}QA2T%^19sMwk8sb%5$TUPBvK-3%Jq?$F1^N9=j`*gWI@`$q88DCIg`f1T}dQQ^M(v z;&!|l89I+o&a)bN&#=cG^nM*U^jKUdMU~;e%XZqZX$pE+xEM!Tc2LZSP>p~{#{GnBuWDR)(k!z&}5-wSJZ5s z_wN7n!FOothX_*@Y2Ppe;B_l173Xa-0AgX=t>~V6Wv%^_z2jOdvx`4t6{xHjvqCjZ zc=sqPPgJ_!e}LZl07|(a0gI9H1=~ms`6&v%WAhOT9*TrZpPFy93kma3)aCD0Z}-c; z1kY`L_ci=-B;LIqn)Ca2JM}03uu72mf%&kkV1TG~E-a`|31wzTB#_8+VoWo4nR4e9qvP#&2~QLa;ScVZk9m&BAc|Ii z>MosHGx=-ozOPxdCz7A3c4N|Pq<@hJaOylSfMS$Ulb8Acn_$yyL&HgLXmm*Td2i&J z|IA|WLdV{s*KuwAY-ejo-LY>(&NG66zkeVf3cly=x_dE<&e->nOf5Y&!#B^Qnf^Sk z_tk{h>-)PeEq%h2nmyuo?;EAJ`y|*1d{&U8#y`~ZDB@Fg5GnbaGSS$eqldTE&r&ckZQZA>ymNJFkP~gnDyM06hL1+#D6|- z&%SN-W8g2wNfwpg>U+LXOX@NocVxhfOGjXaHHn{EX**eDInV~K_DPH4Cc$_fqqnjy z4DNqI!fa`LM*<2G3|P!IBrE1M#%5n3(V%GLJx9N$G-@^Gg zG-Dkb%2u4&$~Gp$UXBIKbj^|5F(RMIa=ca-wT(*xRr5U-F5j-5YqG%<gZ;FxENsRe7GUY_mhzFrqVA zW-*){Lup;jB)rbx3*qLiiG8K`Tpg-Ab&GDkXpK)C00Xe;h|$AMjG$q2xRYhQ&brDF zJy^+m#mvt6-Lj==kKXCBp#{&5&Mx-HId_eJ2$%J6hj`FLeW9e*gcP!hJ%w@hx3_t* zMV2b+-<285(g*HGwhhFB8MXICV1_$vCk4m0m{jlEP|}mkcx24=mB5L3HDnO#(*E|h zSy@#U*QF&+?`}NYmW_7z)*gpuw!LN13YlFfUp3#acAvuXZQsM)eSGBkJo<*|yCG}k zoqJ)cD*LUiJ>BdFwJ>%{sv2TGtt$gh9@*)5Mt7EYx^siguFf=29SCUd#LORyAUQJJ zGGgV^g=i1*2sJxrNChM>P*8ya8pkkkhV}XRm9b%KMcSF_FppPoliCxHqU77Idjj{Z zgdDT`@H`SRpo5!1d2I~J0G7HW0o$lH= zrnK=R+>dP9)+mr=T5Cernf)^XHw4rRAWaG|~vRAeGyRW;yu8VSARMT4D zuAY{@ogaH&t@;p3sinZlYdcOBNLpx^D{W#?oRTucg*ZHnf&m|R!)bA6d1Ec9mKm)Y zac^#sZja<_CqS&jI{_t5cAlyaGbiuw1h9{@%?9BK^8>{eK>3LoCB8+;uui6##K}$D?RV} zJ8OL}zgsOsV;GW%VQguZ`!#Yl;pou)Ga0o&2Gg| z$sBJBw2HDal^ae#7#tj&0N5F*ql)Kj(nAa)7&2jZl%FXF%jJ`h7#!@*NIOa7;D_F6 ztM+)T<1s6ijy=0wqL;357B;B!3Md~iu0t0>!(D;c?yp!Ru_H+| z(+MVag}kQ+0T&q@WqiWiWR=~L39UJyHrtQ%wYJOey1O@RR7!Gt+it4brmyF2%T0A= zjLYZQBobOI*E1@Fm0zEn36?bZxC$H$1C6I64TUfyZ>NjcNPhTd>p}kn-1^R^`ZvFwn?qQ5@2?W-X2re&@-EIEVFCBp^ zakZc13fSC6GOd=vjBH+Z;lxvE=`EVm*M6?aJ#F~~Nx4eRd%unPX?=Fnb@r9V$#XEi zOZK)BM8S-TtidK`&n3FI7|&z#ZO(R?8qajjzuDGu-F>L6al;sq*>)1en6L$RWrFjJ zji3++Q3$!sq!(~AullEud15WAhZ}(;0tqZY+5-R^8A^?>MUCyuR~OPUn{P9FGKivb z#47GNIlv?{3<8+spO(25{ncma?`?L`-CMg?wUZp(tmEtW-=@!NwW59WBUHG!Q+8mE zH-;Hf6%#NB4hR6b1#`8SC;$zt58fDAU2b&oCFIlG+{oT$(I)8BZOTSoNB;nxip1^7 z8DKgLr;-xwVv_1_vHhkPUO^r=oGCfnP6=QM0A~dK;G+=Ej}#&sbTY)C&E&@)maLBI z6+lH$*e9F;oZ)bCX>(l-Qn!i6wXzi%rrnzB1iTw2KfA}}LPW=X3I!PQ--T6-& zxkiJ{N86l-Y=C^eu0saSHgy>$%sPXKZ+^*lx(LvNbjVp_3`kxVYJ>Bk$lO60!6%#d z*UuKACDq(^aL%#?EX;*4pp}p|@qj)+I82!rsW=R|mTQQioY{WrSX*IReCAAHWMD$E zQX8Hzgl74%T2J39rO|)EE?qvN>IKOjOe-?H zcA1rsx)R88O7C2wW6=u^06|w#io)y6p32_y?98}+Wo9kA_q&jG;OUwp; zuAv=;+=QfU8zWo7$fa@S41>343_A>*F+2=gzO|bA&f4xdn|Xjm50?9*!;Oc2RnKU{ zHbS0HeZ;>b(3_Pa!0D zran?jl|eaRP$>s-1@{m_fI!AM8((pIYZ6+9GWm`a!Wqc`7t5%1;g@-3IVro3OdXZ2 z+WW8b*Zvyr_eN2dysY_?(OXMjwUTewzKn)g*5)L%Z9+1Fb$DPb_*GrZz((8>KJE@k z3fTnpG^mTr_Hpcp;6rn2?nVI(w{(k-ya&ooNX~aQ>@;F|g|u3Hj-P0fBu^t^%usK` z#uP4hs%JZ!Bmgm$;gH-Ji84nlGFwW~%K|_{xWg_^&7lB5qa!MC`*_VbUJ6dl+5Vnh zfB834Tt!B-QunQ8w(Cc29{s*<#MfJ$9NBCu0N#ut-m2in8GN8l#>zJvN6DP@<|v*! z$>D}IwOEymk)k7_#87XMuugH8BxH0=q#O_~YHAl!hDju}j%d(`#nd61Mh-y=a#RDb zoB&ANINO}pVC@-MZG6~+A2Cbpc|Ze`y9ER81a5QCefG|Chb*aH?f(EzU2T5q%`NS_ zElTl=l6rJry)ESR+fOY8wuO?;2@*7t)g%n;MpcK*#Dxj~QUE(Q1t6wROxC)*chtXg zZDa(HtK3Ki><0Nt1t5O!IA9y*7&zb>g%ZLWyT*xPm85?uB@G#6$sq0o43C|T!b4$E zakLUNFPz6E<)y^4NduU!A&MQ*0yeSQtGMJ~7|t*<04F>Pi&3j&ucg+u-_gGMdD#%6 zrv|Tm-%EKvKb4-HidMGSB57zzT0Wlr3@aHc<_tPI;j@$k=Y<61 z9%@ZC_@&L##u`}}-^{tUC7oENcO#sF<7|q(hH?PQD977INx1Y~UfWyJ_rK-Q6xBCL zw3E|UeY)GG*5B=Y_K(O_xZi62UBdqMMMagC5LveElQ~d#WQ-h!A1TgEHkQq7e}5zv zw)atjllf1RBL@M4k~mgq)02!4afZo7xwN=>l2mJ?fU}TPv6&sn;TcHD3m41=ag)&I znLV*s-Qrl8!1G6z#I)OrZ1X-=`9mtMGr=rQ-j#H@+;#ZMX9?-G;)=2!{y%p05_S=;>7&9ZdkuXBNc9Q zVnQvnh)hL`xWp+N62y@zhG1MAC>Z%$89ZUl%}HAKeSUgt@@t~}Ii{m&C{(kZ`e}84 zxi->P{9cz!o7S2}mf-}fftd)L#;}Huu0pFBFpHCrGT`77wwrS+Y5Ji^cUmf_PxNLi zS+^>IwT@U|;NW4-INUY>U82iz71Uu2sT*7|!Hp0o^8$rj{L$gQV8kMm&ImSBOqWn_ zvpmSJ9!$`LLl-PcZO%qRFd*fLCl~~lmYQmP66vb8*IthMzMDH-sL$EEZD-TAmU}4q zG`#iIjE_&cw$vn8)(E6Qv@Nrl9%F(Sgte<&i%o+bp-9@fPpY|=DE}( zhSoOP<<$bLj#QW08|G5X%%J2DI-gfLb9rfdX|mgE_tRfJ(VTgf_U7}|H+{M-zsqe8 F|JfZ&jEDdL diff --git a/tests/test_resize_from_pil.py b/tests/test_resize_from_pil.py deleted file mode 100644 index f1d14aa..0000000 --- a/tests/test_resize_from_pil.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -"""Feed PIL-decoded pixels through C++'s bilinear resize (replicated in Python -with double precision) and compare against torchvision resize. - -This isolates the resize from the JPEG decoder. -""" -import numpy as np -import torch -from PIL import Image -from torchvision.transforms import v2 - -image_path = "tests/test_random.jpg" -img_size = 1008 - -# Load with PIL (same decoder as torchvision) -img = Image.open(image_path).convert("RGB") -src = np.array(img, dtype=np.uint8) # [480, 640, 3] HWC -src_h, src_w = src.shape[:2] - -# ═══ Torch resize (ground truth) ═══ -img_tensor = v2.functional.to_image(img) -tv = v2.Resize(size=(img_size, img_size))(img_tensor) -tv_hwc = tv.permute(1, 2, 0).numpy() - -# ═══ C++ resize replicated (double precision, same source pixels) ═══ -sx = float(src_w) / img_size -sy = float(src_h) / img_size - -cpp_hwc = np.zeros((img_size, img_size, 3), dtype=np.uint8) -for y in range(img_size): - fy = (y + 0.5) * sy - 0.5 - if fy < 0: fy = 0.0 - y0 = int(fy) - y1 = y0 + 1 if y0 < src_h - 1 else y0 - wy = fy - y0 - wy0 = 1.0 - wy - for x in range(img_size): - fx = (x + 0.5) * sx - 0.5 - if fx < 0: fx = 0.0 - x0 = int(fx) - x1 = x0 + 1 if x0 < src_w - 1 else x0 - wx = fx - x0 - wx0 = 1.0 - wx - for c in range(3): - p00 = float(src[y0, x0, c]) - p01 = float(src[y0, x1, c]) - p10 = float(src[y1, x0, c]) - p11 = float(src[y1, x1, c]) - v = wy0 * (wx0 * p00 + wx * p01) + wy * (wx0 * p10 + wx * p11) - iv = int(v + 0.5) - if iv < 0: iv = 0 - if iv > 255: iv = 255 - cpp_hwc[y, x, c] = iv - -# Compare -diff = np.abs(cpp_hwc.astype(np.int16) - tv_hwc.astype(np.int16)) -print("═══ Same source pixels: C++ resize (double) vs torchvision ═══") -print(f" max pixel diff: {diff.max()}") -print(f" mean pixel diff: {diff.mean():.6f}") -print(f" pixels differing: {(diff > 0).sum()} / {diff.size} ({100*(diff>0).sum()/diff.size:.2f}%)") -for t in [0, 1, 2]: - n = (diff > t).sum() - if n > 0: - print(f" pixels with diff > {t}: {n}") - -if diff.max() == 0: - print("\n PERFECT MATCH! The resize algorithms are identical.") - print(" The 8% difference was entirely from the JPEG decoder (stb_image vs libjpeg).") -else: - print(f"\n {(diff > 0).sum()} pixels still differ — residual rounding difference.") diff --git a/tests/test_sam2_backbone_compare.cpp b/tests/test_sam2_backbone_compare.cpp deleted file mode 100644 index 153f3ae..0000000 --- a/tests/test_sam2_backbone_compare.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Test: Compare SAM2 backbone + FPN outputs against Python. -// Set SAM2_DUMP_DIR env var to dump debug intermediates. -// Usage: SAM2_DUMP_DIR=/tmp/debug_sam2_cpp test_sam2_backbone_compare - -#include "sam3.h" -#include -#include -#include -#include - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - mkdir(argv[3], 0755); - - std::ifstream fin(argv[2], std::ios::binary); - fin.seekg(0, std::ios::end); - size_t sz = fin.tellg(); - fin.seekg(0); - std::vector img(sz / 4); - fin.read(reinterpret_cast(img.data()), sz); - - sam3_params p; - p.model_path = argv[1]; - p.n_threads = 8; - p.use_gpu = false; - auto model = sam3_load_model(p); - if (!model) return 1; - auto state = sam3_create_state(*model, p); - if (!state) return 1; - - if (!sam3_encode_image_from_preprocessed(*state, *model, img.data(), 1024)) return 1; - - // Dump FPN outputs via state tensor API - for (int i = 0; i < 3; ++i) { - char name[32]; - snprintf(name, sizeof(name), "neck_trk_%d", i); - std::string path = std::string(argv[3]) + "/" + name; - sam3_dump_state_tensor(*state, name, path); - } - - fprintf(stderr, "Done.\n"); - return 0; -} diff --git a/tests/test_sam2_e2e_all_variants.py b/tests/test_sam2_e2e_all_variants.py deleted file mode 100644 index bdff6fc..0000000 --- a/tests/test_sam2_e2e_all_variants.py +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env python3 -"""End-to-end comparison of all SAM2.1 variants: Python vs C++ ggml. - -This script: -1. Runs Python SAM2 inference for all 4 variants (Tiny, Small, Base+, Large) -2. Runs C++ ggml inference for the same variants -3. Compares binary masks and IoU scores - -Usage: - cd ~/Documents/sam2 - python /path/to/test_sam2_e2e_all_variants.py \ - --image /path/to/test_image.jpg \ - --cpp-dir /path/to/sam3.cpp-sam2 \ - --point-x 600 --point-y 599 - -Requires: SAM2 Python package installed, C++ test binary built. -""" -import argparse, os, sys, subprocess, json, time -import numpy as np - -sys.path.insert(0, os.path.expanduser("~/Documents/sam2")) -import torch -torch.set_default_dtype(torch.float32) -from PIL import Image - -VARIANTS_21 = [ - {"name": "tiny", "config": "configs/sam2.1/sam2.1_hiera_t.yaml", "prefix": "sam2.1"}, - {"name": "small", "config": "configs/sam2.1/sam2.1_hiera_s.yaml", "prefix": "sam2.1"}, - {"name": "base_plus", "config": "configs/sam2.1/sam2.1_hiera_b+.yaml", "prefix": "sam2.1"}, - {"name": "large", "config": "configs/sam2.1/sam2.1_hiera_l.yaml", "prefix": "sam2.1"}, -] - -VARIANTS_20 = [ - {"name": "tiny", "config": "configs/sam2/sam2_hiera_t.yaml", "prefix": "sam2"}, - {"name": "small", "config": "configs/sam2/sam2_hiera_s.yaml", "prefix": "sam2"}, - {"name": "base_plus", "config": "configs/sam2/sam2_hiera_b+.yaml", "prefix": "sam2"}, - {"name": "large", "config": "configs/sam2/sam2_hiera_l.yaml", "prefix": "sam2"}, -] - - -def run_python_variant(variant_name, config, checkpoint, image_path, point_x, point_y, dump_dir): - """Run Python SAM2 inference for one variant.""" - from sam2.build_sam import build_sam2 - from sam2.sam2_image_predictor import SAM2ImagePredictor - - os.makedirs(dump_dir, exist_ok=True) - - print(f"\n{'='*60}") - print(f"Python: {variant_name}") - print(f"{'='*60}") - - t0 = time.time() - model = build_sam2(config, checkpoint, device="cpu") - model = model.float().eval() - predictor = SAM2ImagePredictor(model) - t_load = time.time() - t0 - - image = np.array(Image.open(image_path).convert("RGB")) - img_h, img_w = image.shape[:2] - - t0 = time.time() - with torch.no_grad(): - predictor.set_image(image) - t_encode = time.time() - t0 - - point_coords = np.array([[point_x, point_y]], dtype=np.float32) - point_labels = np.array([1], dtype=np.int32) - - t0 = time.time() - with torch.no_grad(): - masks, scores, logits = predictor.predict( - point_coords=point_coords, - point_labels=point_labels, - multimask_output=True, - normalize_coords=True, - ) - t_predict = time.time() - t0 - - # Save preprocessed image for C++ consumption - transforms = predictor._transforms - img_tensor = torch.from_numpy(image).permute(2, 0, 1).float() / 255.0 - from torchvision.transforms.functional import resize, normalize - img_resized = resize(img_tensor, [model.image_size, model.image_size], antialias=True) - mean = torch.tensor([0.485, 0.456, 0.406]).view(3, 1, 1) - std = torch.tensor([0.229, 0.224, 0.225]).view(3, 1, 1) - img_normalized = (img_resized - mean) / std - preproc_path = os.path.join(dump_dir, "preprocessed.bin") - img_normalized.numpy().astype(np.float32).tofile(preproc_path) - - # Save masks and scores - logits.astype(np.float32).tofile(os.path.join(dump_dir, "logits.bin")) - scores.astype(np.float32).tofile(os.path.join(dump_dir, "scores.bin")) - masks.astype(np.uint8).tofile(os.path.join(dump_dir, "masks.bin")) - - # Save metadata - meta = { - "variant": variant_name, "img_w": img_w, "img_h": img_h, - "point_x": point_x, "point_y": point_y, - "n_masks": int(masks.shape[0]), - "mask_h": int(masks.shape[1]), "mask_w": int(masks.shape[2]), - "logit_h": int(logits.shape[1]), "logit_w": int(logits.shape[2]), - "scores": scores.tolist(), - "t_load": t_load, "t_encode": t_encode, "t_predict": t_predict, - } - with open(os.path.join(dump_dir, "meta.json"), "w") as f: - json.dump(meta, f, indent=2) - - best = int(np.argmax(scores)) - fg_pct = 100 * masks[best].sum() / masks[best].size - print(f" Load: {t_load:.1f}s Encode: {t_encode:.1f}s Predict: {t_predict:.3f}s") - print(f" Scores: {scores}") - print(f" Best mask {best}: {fg_pct:.1f}% foreground") - - return meta - - -def run_cpp_variant(variant_name, model_path, preproc_path, dump_dir, cpp_binary, point_x, point_y, img_w, img_h): - """Run C++ ggml inference for one variant.""" - print(f"\n C++: {variant_name}") - - os.makedirs(dump_dir, exist_ok=True) - - # Build a minimal C++ test that uses sam3_encode_image_from_preprocessed + sam3_segment_pvs - # and writes raw mask logits + IoU scores - env = os.environ.copy() - env["SAM2_DUMP_DIR"] = dump_dir - build_dir = os.path.dirname(cpp_binary) - env["DYLD_LIBRARY_PATH"] = ":".join([ - os.path.join(build_dir, "ggml/src"), - os.path.join(build_dir, "ggml/src/ggml-cpu"), - os.path.join(build_dir, "ggml/src/ggml-metal"), - os.path.join(build_dir, "ggml/src/ggml-blas"), - ]) - - cmd = [cpp_binary, model_path, preproc_path, dump_dir, - str(img_w), str(img_h), str(point_x), str(point_y)] - t0 = time.time() - result = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=600) - t_total = time.time() - t0 - - if result.returncode != 0: - print(f" C++ FAILED (exit {result.returncode}):") - print(result.stderr[-500:] if result.stderr else "no stderr") - return None - - # Parse IoU from stderr - ious = [] - for line in result.stderr.split("\n"): - if "iou=" in line: - # Parse: iou=[0.990, 0.994, 0.397, 0.977] - import re - m = re.search(r'iou=\[([\d., ]+)\]', line) - if m: - ious = [float(x.strip()) for x in m.group(1).split(",")] - - print(f" Time: {t_total:.1f}s IoU: {ious}") - return {"ious": ious, "t_total": t_total} - - -def compare_variant(variant_name, py_dir, cpp_dir, py_meta): - """Compare Python and C++ results for one variant.""" - print(f"\n Comparison: {variant_name}") - - # Load Python masks and scores - py_scores = np.fromfile(os.path.join(py_dir, "scores.bin"), dtype=np.float32) - py_logits = np.fromfile(os.path.join(py_dir, "logits.bin"), dtype=np.float32) - n_masks = py_meta["n_masks"] - logit_h, logit_w = py_meta["logit_h"], py_meta["logit_w"] - py_logits = py_logits.reshape(n_masks, logit_h, logit_w) - - # Load C++ mask logits: [H4*H4, 4, 1, 1] in ggml layout - cpp_mask_path = os.path.join(cpp_dir, "cpp_pvs_masks.bin") - if not os.path.exists(cpp_mask_path): - print(f" SKIP — C++ masks not found") - return None - - cpp_masks_raw = np.fromfile(cpp_mask_path, dtype=np.float32) - # C++ has 4 masks (including single-mask at index 0), Python has 3 (multimask only) - # C++ ggml layout: [H4*H4, 4, 1, 1] where flat = pixel + mask*H4*H4 - # Figure out mask resolution from total size - n_pixels = cpp_masks_raw.size // 4 - mask_hw = int(np.sqrt(n_pixels)) - cpp_masks_4 = cpp_masks_raw.reshape(4, n_pixels) - - # C++ IoU - cpp_iou_path = os.path.join(cpp_dir, "cpp_pvs_iou.bin") - cpp_ious = np.fromfile(cpp_iou_path, dtype=np.float32) if os.path.exists(cpp_iou_path) else np.zeros(4) - - # Compare multimask outputs (indices 1-3 in C++, 0-2 in Python) - results = [] - for m in range(n_masks): - py_m = py_logits[m].flatten() - cpp_m = cpp_masks_4[m + 1] # skip index 0 (single-mask) - - # Resize if needed (Python is logit_h × logit_w, C++ is mask_hw × mask_hw) - if py_m.size != cpp_m.size: - # Reshape and compare at the smaller resolution - print(f" Mask {m}: SIZE MISMATCH py={py_m.size} cpp={cpp_m.size}") - results.append({"mask": m, "status": "SIZE_MISMATCH"}) - continue - - # Cosine similarity of logits - cos = np.dot(py_m, cpp_m) / (np.linalg.norm(py_m) * np.linalg.norm(cpp_m) + 1e-8) - - # Binary mask agreement (threshold at 0) - py_bin = (py_m > 0).astype(int) - cpp_bin = (cpp_m > 0).astype(int) - union = max((py_bin | cpp_bin).sum(), 1) - intersect = (py_bin & cpp_bin).sum() - binary_iou = intersect / union - agree_pct = 100 * (py_bin == cpp_bin).sum() / len(py_bin) - - # IoU score comparison - py_iou = py_scores[m] - cpp_iou = cpp_ious[m + 1] if len(cpp_ious) > m + 1 else 0 - - status = "PASS" if binary_iou > 0.90 else "FAIL" - results.append({ - "mask": m, "status": status, - "logit_cos": float(cos), "binary_iou": float(binary_iou), - "agree_pct": float(agree_pct), - "py_iou_score": float(py_iou), "cpp_iou_score": float(cpp_iou), - "py_fg": int(py_bin.sum()), "cpp_fg": int(cpp_bin.sum()), - }) - - flag = "✓" if status == "PASS" else "✗" - print(f" {flag} Mask {m}: logit_cos={cos:.4f} binary_IoU={binary_iou:.4f} " - f"agree={agree_pct:.1f}% py_iou={py_iou:.3f} cpp_iou={cpp_iou:.3f}") - - return results - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--image", required=True) - parser.add_argument("--cpp-dir", required=True, help="Path to sam3.cpp-sam2 root") - parser.add_argument("--point-x", type=float, default=600) - parser.add_argument("--point-y", type=float, default=599) - parser.add_argument("--variants", default="tiny,small,base_plus,large", - help="Comma-separated list of variants to test") - parser.add_argument("--version", default="2.1", choices=["2.0", "2.1", "both"], - help="SAM2 version to test") - args = parser.parse_args() - - cpp_dir = args.cpp_dir - cpp_binary = os.path.join(cpp_dir, "build", "test_sam2_pvs_compare") - if not os.path.exists(cpp_binary): - print(f"ERROR: C++ binary not found at {cpp_binary}") - print(f"Build it first: cd {cpp_dir}/build && make test_sam2_pvs_compare") - sys.exit(1) - - selected = args.variants.split(",") - base_dump = "/tmp/sam2_e2e_test" - os.makedirs(base_dump, exist_ok=True) - - image = np.array(Image.open(args.image).convert("RGB")) - img_h, img_w = image.shape[:2] - - all_results = {} - - # Select variant list based on version - if args.version == "both": - variant_lists = [("SAM2.0", VARIANTS_20), ("SAM2.1", VARIANTS_21)] - elif args.version == "2.0": - variant_lists = [("SAM2.0", VARIANTS_20)] - else: - variant_lists = [("SAM2.1", VARIANTS_21)] - - for ver_label, variants in variant_lists: - print(f"\n{'#'*70}") - print(f"# {ver_label}") - print(f"{'#'*70}") - - for v in variants: - if v["name"] not in selected: - continue - - name = v["name"] - prefix = v["prefix"] - display_name = f"{prefix}_{name}" - ckpt = os.path.join(cpp_dir, f"weights/{prefix}/{prefix}_hiera_{name}.pt") - ggml = os.path.join(cpp_dir, f"weights/ggml/{prefix}_hiera_{name}_f32.ggml") - config = v["config"] - - if not os.path.exists(ckpt): - print(f"\nSKIP {display_name}: checkpoint not found at {ckpt}") - continue - if not os.path.exists(ggml): - print(f"\nSKIP {display_name}: ggml model not found at {ggml}") - continue - - py_dump = os.path.join(base_dump, f"py_{display_name}") - cpp_dump = os.path.join(base_dump, f"cpp_{display_name}") - - # Run Python - py_meta = run_python_variant( - display_name, config, ckpt, args.image, - args.point_x, args.point_y, py_dump - ) - - # Run C++ - cpp_result = run_cpp_variant( - display_name, ggml, - os.path.join(py_dump, "preprocessed.bin"), - cpp_dump, cpp_binary, - args.point_x, args.point_y, img_w, img_h - ) - - # Compare - if cpp_result is not None: - results = compare_variant(display_name, py_dump, cpp_dump, py_meta) - all_results[display_name] = results - - # Summary - print(f"\n{'='*70}") - print(f"SUMMARY: SAM2.1 End-to-End Comparison (all variants)") - print(f"{'='*70}") - print(f"Image: {args.image} ({img_w}x{img_h})") - print(f"Point: ({args.point_x}, {args.point_y})") - print() - print(f"{'Variant':<12} {'Best Mask IoU':>14} {'Logit Cos':>10} {'Binary IoU':>11} {'Agreement':>10} {'Status':>8}") - print("-" * 70) - - all_pass = True - for name, results in all_results.items(): - if not results: - print(f"{name:<12} {'ERROR':>14}") - all_pass = False - continue - best_idx = max(range(len(results)), key=lambda i: results[i].get("py_iou_score", 0)) - r = results[best_idx] - status = r["status"] - if status != "PASS": - all_pass = False - print(f"{name:<12} {r.get('py_iou_score', 0):>14.4f} {r.get('logit_cos', 0):>10.4f} " - f"{r.get('binary_iou', 0):>11.4f} {r.get('agree_pct', 0):>9.1f}% " - f"{' ✓' if status == 'PASS' else ' ✗':>8}") - - print() - print(f"Overall: {'ALL PASS ✓' if all_pass else 'SOME FAILED ✗'}") - return 0 if all_pass else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_sam2_pe_cache_size.cpp b/tests/test_sam2_pe_cache_size.cpp deleted file mode 100644 index d01672d..0000000 --- a/tests/test_sam2_pe_cache_size.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// Regression test: PE cache must use feat_size() (64 for SAM2), NOT n_img_embd() (73). -// -// The bug: sam3_populate_pe_cache() used n_img_embd() = img_size/patch_size = 1024/14 = 73 -// for the dense PE grid size. For SAM2 (Hiera backbone), the correct size is feat_size() = 64. -// A 73×73 PE grid fed to a 64×64 decoder produces garbage masks. -// -// This test verifies: -// 1. The PE cache dense grids have size D * feat_size^2 (not D * n_img_embd^2) -// 2. The PVS decoder produces a mask where the clicked point is inside the foreground -// 3. The IoU score is above a reasonable threshold -// -// Usage: test_sam2_pe_cache_size - -#include "sam3.h" -#include -#include -#include - -int main(int argc, char** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - sam3_params p; - p.model_path = argv[1]; - p.n_threads = 4; - p.use_gpu = false; - - auto model = sam3_load_model(p); - if (!model) { fprintf(stderr, "FAIL: cannot load model\n"); return 1; } - - // Must be a SAM2 model - auto mt = sam3_get_model_type(*model); - if (mt != SAM3_MODEL_SAM2) { - fprintf(stderr, "SKIP: not a SAM2 model (type=%d)\n", mt); - return 0; - } - - auto state = sam3_create_state(*model, p); - if (!state) { fprintf(stderr, "FAIL: cannot create state\n"); return 1; } - - // Load and encode image - auto image = sam3_load_image(argv[2]); - if (image.data.empty()) { fprintf(stderr, "FAIL: cannot load image\n"); return 1; } - fprintf(stderr, "Image: %dx%d\n", image.width, image.height); - - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "FAIL: image encoding failed\n"); - return 1; - } - - // Run PVS with a point at the center of the image - float cx = image.width / 2.0f; - float cy = image.height / 2.0f; - fprintf(stderr, "Point: (%.0f, %.0f)\n", cx, cy); - - sam3_pvs_params pvs; - pvs.pos_points.push_back({cx, cy}); - pvs.multimask = false; // single mask — same as GUI default - - auto result = sam3_segment_pvs(*state, *model, pvs); - - // ── Test 1: Must produce exactly 1 detection ── - if (result.detections.empty()) { - fprintf(stderr, "FAIL: no detections returned\n"); - return 1; - } - fprintf(stderr, "Detections: %zu\n", result.detections.size()); - - auto& det = result.detections[0]; - fprintf(stderr, "Mask 0: iou=%.4f, mask=%dx%d\n", - det.iou_score, det.mask.width, det.mask.height); - - // ── Test 2: IoU score must be reasonable (> 0.1) ── - // A garbage PE grid produces IoU near 0 or random values - if (det.iou_score < 0.1f) { - fprintf(stderr, "FAIL: IoU score %.4f is too low (PE cache likely wrong size)\n", - det.iou_score); - return 1; - } - - // ── Test 3: The clicked point must be INSIDE the foreground mask ── - // If the PE grid is wrong size, the mask is often inverted or displaced - int px = (int)(cx * det.mask.width / image.width); - int py = (int)(cy * det.mask.height / image.height); - if (px >= det.mask.width) px = det.mask.width - 1; - if (py >= det.mask.height) py = det.mask.height - 1; - int idx = py * det.mask.width + px; - bool point_in_mask = (det.mask.data[idx] > 0); - - if (!point_in_mask) { - fprintf(stderr, "FAIL: clicked point (%d,%d) is NOT inside the predicted mask\n", px, py); - fprintf(stderr, " This indicates the PE cache spatial size is wrong.\n"); - fprintf(stderr, " Expected feat_size()=%d, check sam3_populate_pe_cache().\n", - 64); // SAM2 feat_size - return 1; - } - - // ── Test 4: Foreground coverage must be reasonable (0.1% - 99%) ── - // A wrong PE grid often produces all-foreground or all-background masks - int fg_count = 0; - for (size_t i = 0; i < det.mask.data.size(); ++i) - if (det.mask.data[i] > 0) fg_count++; - float fg_pct = 100.0f * fg_count / det.mask.data.size(); - - if (fg_pct < 0.05f || fg_pct > 99.5f) { - fprintf(stderr, "FAIL: foreground coverage %.2f%% is degenerate (expected 0.05-99.5%%)\n", - fg_pct); - return 1; - } - - fprintf(stderr, "PASS: IoU=%.4f, point_in_mask=true, fg=%.1f%%\n", - det.iou_score, fg_pct); - return 0; -} diff --git a/tests/test_sam2_pvs_compare.cpp b/tests/test_sam2_pvs_compare.cpp deleted file mode 100644 index 93efbd6..0000000 --- a/tests/test_sam2_pvs_compare.cpp +++ /dev/null @@ -1,74 +0,0 @@ -// Test: Compare SAM2 PVS decoder output against Python reference. -// Set SAM2_DUMP_DIR to dump debug tensors. -// Usage: test_sam2_pvs_compare [orig_w orig_h point_x point_y] -// Default: orig 1200x1198, point at (600, 599) - -#include "sam3.h" -#include -#include -#include -#include -#include - -// Hack: set orig dims on state (state is opaque, but we know its layout) -// This is needed because encode_image_from_preprocessed sets orig to img_size -extern void sam3_state_set_orig_dims(sam3_state& state, int w, int h); - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s [orig_w orig_h point_x point_y]\n", argv[0]); - return 1; - } - - int orig_w = (argc > 4) ? atoi(argv[4]) : 1200; - int orig_h = (argc > 5) ? atoi(argv[5]) : 1198; - float point_x = (argc > 6) ? atof(argv[6]) : 600.0f; - float point_y = (argc > 7) ? atof(argv[7]) : 599.0f; - - mkdir(argv[3], 0755); - - std::ifstream fin(argv[2], std::ios::binary); - fin.seekg(0, std::ios::end); - size_t sz = fin.tellg(); - fin.seekg(0); - std::vector img(sz / 4); - fin.read(reinterpret_cast(img.data()), sz); - - sam3_params p; - p.model_path = argv[1]; - p.n_threads = 8; - p.use_gpu = false; - auto model = sam3_load_model(p); - if (!model) return 1; - auto state = sam3_create_state(*model, p); - if (!state) return 1; - - if (!sam3_encode_image_from_preprocessed(*state, *model, img.data(), 1024)) return 1; - - // Override original image dimensions (encode_from_preprocessed sets them to 1024) - sam3_state_set_orig_dims(*state, orig_w, orig_h); - - sam3_pvs_params pvs; - pvs.pos_points.push_back({point_x, point_y}); - pvs.multimask = true; - - auto result = sam3_segment_pvs(*state, *model, pvs); - - fprintf(stderr, "PVS result: %zu detections\n", result.detections.size()); - for (size_t i = 0; i < result.detections.size(); ++i) { - fprintf(stderr, " det %zu: iou=%.4f score=%.4f mask=%dx%d\n", - i, result.detections[i].iou_score, result.detections[i].score, - result.detections[i].mask.width, result.detections[i].mask.height); - } - - if (result.detections.size() > 0) { - for (size_t i = 0; i < result.detections.size(); ++i) { - char path[256]; - snprintf(path, sizeof(path), "%s/mask_%zu.png", argv[3], i); - sam3_save_mask(result.detections[i].mask, path); - } - } - - fprintf(stderr, "Done.\n"); - return 0; -} diff --git a/tests/test_sam2_video_debug.cpp b/tests/test_sam2_video_debug.cpp deleted file mode 100644 index 13917e7..0000000 --- a/tests/test_sam2_video_debug.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/** - * test_sam2_video_debug — Debug SAM2 video tracking pipeline. - * - * Loads bedroom video frames, runs video tracking for 5 frames, - * and dumps intermediate tensors for comparison with Python reference. - * - * Usage: - * test_sam2_video_debug [point_x point_y] - * - * Example: - * test_sam2_video_debug models/sam2.1_hiera_tiny_f32.ggml \ - * ~/Documents/sam2/notebooks/videos/bedroom 210 350 - */ -#include "sam3.h" -#include -#include -#include -#include -#include -#include -#include - -static void dump_tensor_f32(const char* dir, const char* name, - const float* data, int n, - const char* shape_str) { - char path[512]; - snprintf(path, sizeof(path), "%s/%s.bin", dir, name); - FILE* f = fopen(path, "wb"); - if (!f) { fprintf(stderr, "Failed to write %s\n", path); return; } - fwrite(data, sizeof(float), n, f); - fclose(f); - - snprintf(path, sizeof(path), "%s/%s.shape", dir, name); - f = fopen(path, "w"); - if (!f) return; - fprintf(f, "%s", shape_str); - fclose(f); - - // Print stats - float mn = data[0], mx = data[0]; - double sum = 0; - for (int i = 0; i < n; ++i) { - if (data[i] < mn) mn = data[i]; - if (data[i] > mx) mx = data[i]; - sum += data[i]; - } - fprintf(stderr, " [DUMP] %-50s shape=%-20s range=[%.4f, %.4f] mean=%.6f\n", - name, shape_str, mn, mx, sum / n); -} - -int main(int argc, char** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [point_x point_y]\n", argv[0]); - return 1; - } - - const char* model_path = argv[1]; - const char* frames_dir = argv[2]; - float point_x = (argc > 3) ? atof(argv[3]) : 210.0f; - float point_y = (argc > 4) ? atof(argv[4]) : 350.0f; - - const char* dump_dir = "/tmp/debug_sam2_cpp"; - mkdir(dump_dir, 0755); - - const int N_FRAMES = argc > 5 ? atoi(argv[5]) : 5; - int encode_img_size = (argc > 6) ? atoi(argv[6]) : 0; - - // Load model - fprintf(stderr, "Loading model: %s\n", model_path); - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; // CPU for reproducibility - params.encode_img_size = encode_img_size; - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model\n"); - return 1; - } - - // Create state - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "Failed to create state\n"); - return 1; - } - - // Load frames as JPEG images - std::vector frames; - for (int i = 0; i < N_FRAMES + 1; ++i) { - char path[512]; - snprintf(path, sizeof(path), "%s/%05d.jpg", frames_dir, i); - sam3_image img = sam3_load_image(path); - if (img.data.empty()) { - fprintf(stderr, "Failed to load frame %d: %s\n", i, path); - return 1; - } - fprintf(stderr, "Loaded frame %d: %dx%d\n", i, img.width, img.height); - frames.push_back(std::move(img)); - } - - // Create visual tracker - sam3_visual_track_params vtp; - vtp.max_keep_alive = 100; - vtp.recondition_every = 16; - auto tracker = sam3_create_visual_tracker(*model, vtp); - if (!tracker) { - fprintf(stderr, "Failed to create tracker\n"); - return 1; - } - - // ── Frame 0: Encode + add instance with point prompt ────────────────── - fprintf(stderr, "\n=== Frame 0: Encode + Add Instance ===\n"); - - // Encode frame 0 - if (!sam3_encode_image(*state, *model, frames[0])) { - fprintf(stderr, "Failed to encode frame 0\n"); - return 1; - } - - // Dump neck features for frame 0 - // state.neck_trk[0,1,2] are the FPN features - sam3_dump_state_tensor(*state, "neck_trk_0", std::string(dump_dir) + "/f0_neck_trk_0"); - sam3_dump_state_tensor(*state, "neck_trk_1", std::string(dump_dir) + "/f0_neck_trk_1"); - sam3_dump_state_tensor(*state, "neck_trk_2", std::string(dump_dir) + "/f0_neck_trk_2"); - - // Add instance with point prompt - sam3_pvs_params pvs; - pvs.pos_points.push_back({point_x, point_y}); - pvs.multimask = false; - - int inst_id = sam3_tracker_add_instance(*tracker, *state, *model, pvs); - fprintf(stderr, "Added instance: %d\n", inst_id); - if (inst_id < 0) { - fprintf(stderr, "Failed to add instance\n"); - return 1; - } - - // ── Propagate frames 1-4 ────────────────────────────────────────────── - for (int fi = 1; fi <= N_FRAMES; ++fi) { - fprintf(stderr, "\n=== Frame %d: Propagate ===\n", fi); - - auto result = sam3_propagate_frame(*tracker, *state, *model, frames[fi]); - - fprintf(stderr, " Detections: %zu\n", result.detections.size()); - for (size_t d = 0; d < result.detections.size(); ++d) { - const auto& det = result.detections[d]; - int fg = 0; - for (auto v : det.mask.data) if (v > 127) fg++; - float pct = 100.0f * fg / std::max(1, (int)det.mask.data.size()); - fprintf(stderr, " det[%zu] inst=%d score=%.4f fg=%d (%.1f%%) mask=%dx%d\n", - d, det.instance_id, det.score, fg, pct, - det.mask.width, det.mask.height); - } - - // Dump neck features - char buf[128]; - snprintf(buf, sizeof(buf), "f%d_neck_trk_2", fi); - sam3_dump_state_tensor(*state, "neck_trk_2", - std::string(dump_dir) + "/" + buf); - - // Dump output mask - if (!result.detections.empty()) { - const auto& det = result.detections[0]; - std::vector mask_f(det.mask.data.size()); - for (size_t i = 0; i < det.mask.data.size(); ++i) - mask_f[i] = det.mask.data[i] > 127 ? 1.0f : 0.0f; - snprintf(buf, sizeof(buf), "f%d_output_mask_binary", fi); - char shape[64]; - snprintf(shape, sizeof(shape), "%d,%d", det.mask.height, det.mask.width); - dump_tensor_f32(dump_dir, buf, mask_f.data(), (int)mask_f.size(), shape); - } - } - - fprintf(stderr, "\nDone. Tensors dumped to %s/\n", dump_dir); - return 0; -} diff --git a/tests/test_text_enc_cat.cpp b/tests/test_text_enc_cat.cpp deleted file mode 100644 index 760be33..0000000 --- a/tests/test_text_enc_cat.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Quick test: run the text encoder in isolation for "cat" token IDs - * and compare against Python reference. - */ -#include "sam3.h" -#include "test_utils.h" -#include -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string ref_dir = std::string(argv[2]) + "/pcs_ref"; - const std::string out_dir = std::string(argv[2]) + "/pcs_cpp_text_only"; - ensure_dir(out_dir); - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - // Load cat token IDs from Python reference - auto ref_tokens_f = load_ref_f32(ref_dir + "/token_ids"); - if (ref_tokens_f.data.empty()) { - fprintf(stderr, "Failed to load token IDs\n"); - return 1; - } - std::vector token_ids(ref_tokens_f.numel()); - for (int i = 0; i < ref_tokens_f.numel(); ++i) { - token_ids[i] = (int32_t)ref_tokens_f.data[i]; - } - while ((int)token_ids.size() < 32) token_ids.push_back(0); - - fprintf(stderr, "Cat token IDs: [%d, %d, %d, ...]\n", - token_ids[0], token_ids[1], token_ids[2]); - - // Run text encoder in ISOLATION - if (!sam3_test_dump_text_encoder(*model, token_ids, out_dir, params.n_threads)) { - fprintf(stderr, "Failed to dump text encoder\n"); - return 1; - } - - // Compare against Python reference - auto cpp = load_ref_f32(out_dir + "/text_features_2d"); - auto py = load_ref_f32(ref_dir + "/text_features"); - if (cpp.data.empty() || py.data.empty()) { - fprintf(stderr, "Failed to load tensors for comparison\n"); - return 1; - } - - int n = std::min(cpp.numel(), py.numel()); - auto r = compare_tensors(cpp.data.data(), py.data.data(), n, 1e-4f); - - fprintf(stderr, "\n═══ Text Encoder (Isolated): C++ vs Python for 'cat' ═══\n"); - fprintf(stderr, " shape: cpp=[%d,%d] py=[%d,%d]\n", - cpp.shape[0], cpp.shape.size()>1?cpp.shape[1]:0, - py.shape[0], py.shape.size()>1?py.shape[1]:0); - fprintf(stderr, " max_diff: %.6e\n", r.max_diff); - fprintf(stderr, " mean_diff: %.6e\n", r.mean_diff); - fprintf(stderr, " cosine: %.8f\n", r.cosine_sim); - fprintf(stderr, " n_bad (atol=1e-4): %d/%d\n", r.n_bad, r.n_total); - - fprintf(stderr, "\n Result: %s\n", r.cosine_sim > 0.999f ? "PASS" : "FAIL"); - - sam3_free_model(*model); - return r.cosine_sim > 0.999f ? 0 : 1; -} diff --git a/tests/test_text_enc_dump.cpp b/tests/test_text_enc_dump.cpp deleted file mode 100644 index 174d647..0000000 --- a/tests/test_text_enc_dump.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "sam3.h" - -#include -#include -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string output_dir = argv[2]; - - // Create output directory - mkdir(output_dir.c_str(), 0755); - - // Load tokenizer from embedded data in model file - if (!sam3_test_load_tokenizer(model_path)) { - fprintf(stderr, "Failed to load tokenizer from %s\n", model_path.c_str()); - return 1; - } - - // Tokenize "shoe" - auto token_ids = sam3_test_tokenize("shoe"); - fprintf(stderr, "Token IDs: ["); - for (size_t i = 0; i < token_ids.size(); ++i) { - if (i > 0) fprintf(stderr, ", "); - fprintf(stderr, "%d", token_ids[i]); - if (token_ids[i] == 0 && i > 1) { fprintf(stderr, ", ..."); break; } - } - fprintf(stderr, "]\n"); - - // Load model - sam3_params params; - params.model_path = model_path; - params.n_threads = 1; - params.use_gpu = false; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "Failed to load model from %s\n", model_path.c_str()); - return 1; - } - - // Dump text encoder tensors - if (!sam3_test_dump_text_encoder(*model, token_ids, output_dir, params.n_threads)) { - fprintf(stderr, "Failed to dump text encoder tensors\n"); - sam3_free_model(*model); - return 1; - } - - fprintf(stderr, "All tensors dumped to %s/\n", output_dir.c_str()); - sam3_free_model(*model); - return 0; -} diff --git a/tests/test_tokenizer.cpp b/tests/test_tokenizer.cpp deleted file mode 100644 index 223637b..0000000 --- a/tests/test_tokenizer.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include "sam3.h" - -#include -#include -#include -#include -#include - -// ── Test cases with Python-verified expected token IDs ─────────────────────── -// Generated by: python scripts/verify_tokenizer.py raw_weights/ - -struct test_case { - const char * text; - std::vector expected; // non-padding tokens only (SOT + BPE + EOT) -}; - -static const test_case CASES[] = { - { "a", { 49406, 320, 49407 } }, - { "hello world", { 49406, 3306, 1002, 49407 } }, - { "yellow school bus", { 49406, 4481, 1228, 2840, 49407 } }, - { "a photo of a cat", { 49406, 320, 1125, 539, 320, 2368, 49407 } }, - { "don't stop", { 49406, 847, 713, 1691, 49407 } }, - { "SAM 3", { 49406, 3730, 274, 49407 } }, - { "123", { 49406, 272, 273, 274, 49407 } }, - { "", { 49406, 49407 } }, - { "a person riding a bicycle on a sunny day in the park", - { 49406, 320, 2533, 6765, 320, 11652, 525, 320, 5438, 575, 530, 518, 1452, 49407 } }, -}; -static const int N_CASES = sizeof(CASES) / sizeof(CASES[0]); - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s \n", argv[0]); - return 1; - } - - if (!sam3_test_load_tokenizer(argv[1])) { - fprintf(stderr, "FAIL: could not load tokenizer from '%s'\n", argv[1]); - return 1; - } - - int n_pass = 0; - for (int i = 0; i < N_CASES; ++i) { - auto tokens = sam3_test_tokenize(CASES[i].text); - const auto & exp = CASES[i].expected; - bool pass = true; - - // Must be exactly 32 tokens (context length) - if ((int)tokens.size() != 32) { - fprintf(stderr, "FAIL [%d]: expected 32 tokens, got %d\n", i, (int)tokens.size()); - pass = false; - } - - // Check non-padding prefix matches expected exactly - for (size_t j = 0; j < exp.size() && j < tokens.size(); ++j) { - if (tokens[j] != exp[j]) { - fprintf(stderr, "FAIL [%d]: token[%zu] = %d, expected %d\n", - i, j, tokens[j], exp[j]); - pass = false; - } - } - - // Check remaining positions are zero-padded - for (size_t j = exp.size(); j < tokens.size(); ++j) { - if (tokens[j] != 0) { - fprintf(stderr, "FAIL [%d]: expected 0 at position %zu, got %d\n", - i, j, tokens[j]); - pass = false; - break; - } - } - - // Print result - fprintf(stderr, "%s [%d] \"%s\" → [", pass ? "PASS" : "FAIL", i, CASES[i].text); - for (size_t j = 0; j < tokens.size(); ++j) { - if (j > 0) fprintf(stderr, ", "); - fprintf(stderr, "%d", tokens[j]); - if (tokens[j] == 0 && j > 1) { fprintf(stderr, ", ..."); break; } - } - fprintf(stderr, "]\n"); - - if (pass) n_pass++; - } - - fprintf(stderr, "\n%d/%d tests passed\n", n_pass, N_CASES); - return (n_pass == N_CASES) ? 0 : 1; -} diff --git a/tests/test_utils.h b/tests/test_utils.h deleted file mode 100644 index ef5907a..0000000 --- a/tests/test_utils.h +++ /dev/null @@ -1,194 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -struct ref_tensor_f32 { - std::vector data; - std::vector shape; - - int numel() const { - int n = 1; - for (int d : shape) { - n *= d; - } - return n; - } -}; - -struct ref_tensor_i32 { - std::vector data; - std::vector shape; - - int numel() const { - int n = 1; - for (int d : shape) { - n *= d; - } - return n; - } -}; - -struct compare_result { - float max_diff = 0.0f; - float mean_diff = 0.0f; - float cosine_sim = 0.0f; - int n_bad = 0; - int n_total = 0; - int worst_index = -1; - float worst_a = 0.0f; - float worst_b = 0.0f; -}; - -static inline std::vector load_shape_file(const std::string & path) { - std::vector shape; - std::ifstream f(path); - if (!f) { - return shape; - } - - std::string line; - std::getline(f, line); - size_t pos = 0; - while (pos < line.size()) { - size_t end = line.find(',', pos); - if (end == std::string::npos) { - end = line.size(); - } - shape.push_back(std::stoi(line.substr(pos, end - pos))); - pos = end + 1; - } - return shape; -} - -static inline ref_tensor_f32 load_ref_f32(const std::string & path) { - ref_tensor_f32 t; - t.shape = load_shape_file(path + ".shape"); - if (t.shape.empty()) { - return t; - } - - std::ifstream f(path + ".bin", std::ios::binary); - if (!f) { - t.shape.clear(); - return t; - } - - t.data.resize(t.numel()); - f.read(reinterpret_cast(t.data.data()), t.numel() * sizeof(float)); - return t; -} - -static inline ref_tensor_i32 load_ref_i32(const std::string & path) { - ref_tensor_i32 t; - t.shape = load_shape_file(path + ".shape"); - if (t.shape.empty()) { - return t; - } - - std::ifstream f(path + ".bin", std::ios::binary); - if (!f) { - t.shape.clear(); - return t; - } - - t.data.resize(t.numel()); - f.read(reinterpret_cast(t.data.data()), t.numel() * sizeof(int32_t)); - return t; -} - -static inline compare_result compare_tensors(const float * a, - const float * b, - int n, - float atol = 1e-4f) { - compare_result r; - r.n_total = n; - - double sum_diff = 0.0; - double dot_ab = 0.0; - double dot_aa = 0.0; - double dot_bb = 0.0; - - for (int i = 0; i < n; ++i) { - float diff = 0.0f; - if (!std::isfinite(a[i]) || !std::isfinite(b[i])) { - diff = (std::isinf(a[i]) && std::isinf(b[i]) && - ((a[i] > 0.0f) == (b[i] > 0.0f))) ? 0.0f : INFINITY; - } else { - diff = fabsf(a[i] - b[i]); - dot_ab += (double) a[i] * (double) b[i]; - dot_aa += (double) a[i] * (double) a[i]; - dot_bb += (double) b[i] * (double) b[i]; - } - - if (diff > r.max_diff) { - r.max_diff = diff; - r.worst_index = i; - r.worst_a = a[i]; - r.worst_b = b[i]; - } - if (diff > atol) { - r.n_bad++; - } - sum_diff += diff; - } - - r.mean_diff = n > 0 ? (float) (sum_diff / n) : 0.0f; - const double denom = sqrt(dot_aa) * sqrt(dot_bb); - r.cosine_sim = denom > 0.0 ? (float) (dot_ab / denom) : 0.0f; - return r; -} - -static inline int compare_exact_i32(const std::vector & got, - const std::vector & ref) { - const size_t n = got.size() < ref.size() ? got.size() : ref.size(); - int n_bad = got.size() == ref.size() ? 0 : 1; - for (size_t i = 0; i < n; ++i) { - if (got[i] != ref[i]) { - ++n_bad; - } - } - return n_bad; -} - -static inline bool ensure_dir(const std::string & path) { - if (path.empty()) { - return true; - } - - std::string cur; - size_t pos = 0; - if (path[0] == '/') { - cur = "/"; - pos = 1; - } - - while (pos <= path.size()) { - size_t next = path.find('/', pos); - if (next == std::string::npos) { - next = path.size(); - } - - const std::string part = path.substr(pos, next - pos); - if (!part.empty()) { - if (!cur.empty() && cur.back() != '/') { - cur.push_back('/'); - } - cur += part; - if (mkdir(cur.c_str(), 0755) != 0 && errno != EEXIST) { - return false; - } - } - - pos = next + 1; - } - - return true; -} diff --git a/tests/test_v_cont_removal.cpp b/tests/test_v_cont_removal.cpp deleted file mode 100644 index db185c9..0000000 --- a/tests/test_v_cont_removal.cpp +++ /dev/null @@ -1,183 +0,0 @@ -// Validate that removing cont(permute) for V produces identical output. -// Builds the same ViT block twice — once with V cont, once without — on CPU. -// Output must be bit-identical. - -#include "ggml-alloc.h" -#include "ggml-backend.h" -#include "ggml-cpu.h" -#include "ggml.h" - -#ifdef GGML_USE_METAL -#include "ggml-metal.h" -#endif - -#include -#include -#include - -struct block_result { - std::vector output; - int n_nodes; -}; - -static block_result run_vit_block(ggml_backend_t backend, bool use_v_cont) { - const int E = 1024, HD = 64, NH = 16, MLP = 4736; - const int W = 24, H = 24, NP = 9, N = W * H; - const ggml_type WT = GGML_TYPE_F16; - - size_t ctx_size = ggml_tensor_overhead() * 128 + ggml_graph_overhead_custom(128, false); - ggml_init_params params = {ctx_size, nullptr, true}; - auto* ctx = ggml_init(params); - std::vector inputs; - - auto inp = [&](ggml_type type, int64_t d0, int64_t d1=1, int64_t d2=1, int64_t d3=1) { - auto* t = ggml_new_tensor_4d(ctx, type, d0, d1, d2, d3); - ggml_set_input(t); - inputs.push_back(t); - return t; - }; - - auto* x = inp(GGML_TYPE_F32, E, W, H, NP); - auto* norm1_w = inp(GGML_TYPE_F32, E); - auto* norm1_b = inp(GGML_TYPE_F32, E); - auto* xn = ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, x, 1e-6f), norm1_w), norm1_b); - - auto* qkv_w = inp(WT, E, 3*E); - auto* qkv_b = inp(GGML_TYPE_F32, 3*E); - auto* qkv = ggml_add(ctx, ggml_mul_mat(ctx, qkv_w, xn), qkv_b); - - qkv = ggml_reshape_4d(ctx, qkv, E, 3, N, NP); - qkv = ggml_cont(ctx, ggml_permute(ctx, qkv, 0, 3, 1, 2)); - auto* Q = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], 0); - auto* K = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], qkv->nb[3]); - auto* V = ggml_view_3d(ctx, qkv, E, N, NP, qkv->nb[1], qkv->nb[2], 2*qkv->nb[3]); - - // Q and K: always use cont (they need it for RoPE) - Q = ggml_reshape_4d(ctx, Q, HD, NH, N, NP); - Q = ggml_cont(ctx, ggml_permute(ctx, Q, 0, 2, 1, 3)); - Q = ggml_reshape_4d(ctx, Q, HD, N, NH, NP); - - K = ggml_reshape_4d(ctx, K, HD, NH, N, NP); - K = ggml_cont(ctx, ggml_permute(ctx, K, 0, 2, 1, 3)); - K = ggml_reshape_4d(ctx, K, HD, N, NH, NP); - - // V: the part we're testing - if (use_v_cont) { - // OLD path: reshape → permute → cont → reshape_3d → reshape_4d - V = ggml_reshape_4d(ctx, V, HD, NH, N, NP); - V = ggml_cont(ctx, ggml_permute(ctx, V, 0, 2, 1, 3)); - V = ggml_reshape_3d(ctx, V, HD, N, NH * NP); - V = ggml_reshape_4d(ctx, V, HD, N, NH, NP); - } else { - // NEW path: reshape → permute (no cont, no reshape) - V = ggml_reshape_4d(ctx, V, HD, NH, N, NP); - V = ggml_permute(ctx, V, 0, 2, 1, 3); - } - - auto* attn = ggml_flash_attn_ext(ctx, Q, K, V, nullptr, 1.0f/8.0f, 0, 0); - auto* attn_r = ggml_reshape_4d(ctx, attn, E, W, H, NP); - - auto* proj_w = inp(WT, E, E); - auto* proj_b = inp(GGML_TYPE_F32, E); - auto* proj = ggml_add(ctx, ggml_mul_mat(ctx, proj_w, attn_r), proj_b); - auto* res1 = ggml_add(ctx, x, proj); - - auto* norm2_w = inp(GGML_TYPE_F32, E); - auto* norm2_b = inp(GGML_TYPE_F32, E); - auto* xn2 = ggml_add(ctx, ggml_mul(ctx, ggml_norm(ctx, res1, 1e-6f), norm2_w), norm2_b); - - auto* fc1_w = inp(WT, E, MLP); - auto* fc1_b = inp(GGML_TYPE_F32, MLP); - auto* fc2_w = inp(WT, MLP, E); - auto* fc2_b = inp(GGML_TYPE_F32, E); - auto* h = ggml_add(ctx, ggml_mul_mat(ctx, fc1_w, xn2), fc1_b); - h = ggml_gelu_erf(ctx, h); - h = ggml_add(ctx, ggml_mul_mat(ctx, fc2_w, h), fc2_b); - auto* out = ggml_add(ctx, res1, h); - ggml_set_output(out); - - auto* graph = ggml_new_graph_custom(ctx, 128, false); - ggml_build_forward_expand(graph, out); - - auto* galloc = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); - ggml_gallocr_reserve(galloc, graph); - ggml_gallocr_alloc_graph(galloc, graph); - - // Fill with deterministic data - for (auto* t : inputs) { - int64_t nel = ggml_nelements(t); - if (t->type == GGML_TYPE_F32) { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = 0.01f * ((j % 97) - 48); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(float)); - } else { - std::vector d(nel); - for (int64_t j = 0; j < nel; j++) d[j] = ggml_fp32_to_fp16(0.01f * ((j % 97) - 48)); - ggml_backend_tensor_set(t, d.data(), 0, nel * sizeof(uint16_t)); - } - } - - if (ggml_backend_is_cpu(backend)) - ggml_backend_cpu_set_n_threads(backend, 8); - - ggml_backend_graph_compute(backend, graph); - - int64_t nel = ggml_nelements(out); - block_result r; - r.output.resize(nel); - r.n_nodes = ggml_graph_n_nodes(graph); - ggml_backend_tensor_get(out, r.output.data(), 0, nel * sizeof(float)); - - ggml_gallocr_free(galloc); - ggml_free(ctx); - return r; -} - -static bool compare(const std::vector& a, const std::vector& b, - const char* label, float tol) { - float max_diff = 0; - int max_idx = 0; - for (size_t i = 0; i < a.size(); i++) { - float d = std::fabs(a[i] - b[i]); - if (d > max_diff) { max_diff = d; max_idx = (int)i; } - } - bool ok = max_diff <= tol; - fprintf(stderr, " %s [%s]: max_diff=%.8f at [%d] (a=%.6f b=%.6f) (%zu elements)\n", - ok ? "PASS" : "FAIL", label, max_diff, max_idx, - a[max_idx], b[max_idx], a.size()); - return ok; -} - -int main() { - auto* cpu = ggml_backend_cpu_init(); - bool all_pass = true; - - // TEST 1: CPU with V cont vs CPU without V cont — must be IDENTICAL - fprintf(stderr, "\n=== CPU: with V cont vs without V cont ===\n"); - auto cpu_with = run_vit_block(cpu, true); - auto cpu_without = run_vit_block(cpu, false); - fprintf(stderr, " Nodes: with=%d, without=%d\n", cpu_with.n_nodes, cpu_without.n_nodes); - all_pass &= compare(cpu_with.output, cpu_without.output, "CPU with_cont vs without_cont", 0.0f); - -#ifdef GGML_USE_METAL - auto* metal = ggml_backend_metal_init(); - if (metal) { - // TEST 2: Metal with V cont vs Metal without V cont — must be IDENTICAL - fprintf(stderr, "\n=== Metal: with V cont vs without V cont ===\n"); - auto metal_with = run_vit_block(metal, true); - auto metal_without = run_vit_block(metal, false); - fprintf(stderr, " Nodes: with=%d, without=%d\n", metal_with.n_nodes, metal_without.n_nodes); - all_pass &= compare(metal_with.output, metal_without.output, "Metal with_cont vs without_cont", 0.0f); - - // TEST 3: CPU without vs Metal without — F16 precision diff (informational) - fprintf(stderr, "\n=== CPU vs Metal (both without V cont) ===\n"); - compare(cpu_without.output, metal_without.output, "CPU vs Metal (informational)", 50.0f); - - ggml_backend_free(metal); - } -#endif - - ggml_backend_free(cpu); - fprintf(stderr, "\n%s\n", all_pass ? "ALL TESTS PASSED" : "SOME TESTS FAILED"); - return all_pass ? 0 : 1; -} diff --git a/tests/test_video_2frame.cpp b/tests/test_video_2frame.cpp deleted file mode 100644 index 31e397c..0000000 --- a/tests/test_video_2frame.cpp +++ /dev/null @@ -1,107 +0,0 @@ -/** - * Simple 2-frame video tracking test. - * Frame 0: encode image, add instance via point prompt - * Frame 1: track (propagate mask to same image) - * - * Usage: test_video_2frame [x y] - */ -#include "sam3.h" -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [x y]\n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string image_path = argv[2]; - float px = argc > 4 ? atof(argv[3]) : 315.0f; - float py = argc > 4 ? atof(argv[4]) : 250.0f; - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - fprintf(stderr, "Loading model...\n"); - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - // Create tracker (no text prompt → propagation only, no PCS detection) - sam3_video_params vp; - vp.hotstart_delay = 0; // instant confirmation (no warmup) - vp.max_keep_alive = 100; - auto tracker = sam3_create_tracker(*model, vp); - if (!tracker) { fprintf(stderr, "Failed to create tracker\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - // Load image (use same image for both frames for simplicity) - auto image = sam3_load_image(image_path); - if (image.data.empty()) { fprintf(stderr, "Failed to load image\n"); return 1; } - fprintf(stderr, "Image: %dx%d\n", image.width, image.height); - - // ════════════════════════════════════════════════════════════════ - // Frame 0: Encode + add instance via point - // ════════════════════════════════════════════════════════════════ - fprintf(stderr, "\n═══ Frame 0: Encode + Add Instance ═══\n"); - - // Encode image - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "Failed to encode frame 0\n"); return 1; - } - - // Add instance at the clicked point - sam3_pvs_params pvs; - pvs.pos_points.push_back({px, py}); - pvs.multimask = false; - - int inst_id = sam3_tracker_add_instance(*tracker, *state, *model, pvs); - fprintf(stderr, "Added instance %d at (%.1f, %.1f)\n", inst_id, px, py); - - if (inst_id < 0) { - fprintf(stderr, "Failed to add instance\n"); - return 1; - } - - // Check tracker state - fprintf(stderr, "Tracker frame: %d\n", sam3_tracker_frame_index(*tracker)); - - // ════════════════════════════════════════════════════════════════ - // Frame 1: Track (propagate to same image) - // ════════════════════════════════════════════════════════════════ - fprintf(stderr, "\n═══ Frame 1: Track ═══\n"); - - // Re-encode same image as "frame 1" - if (!sam3_encode_image(*state, *model, image)) { - fprintf(stderr, "Failed to encode frame 1\n"); return 1; - } - - // Track - auto result = sam3_track_frame(*tracker, *state, *model, image); - fprintf(stderr, "Track result: %zu detections\n", result.detections.size()); - - for (size_t i = 0; i < result.detections.size(); ++i) { - const auto& d = result.detections[i]; - fprintf(stderr, " det %zu: inst=%d score=%.4f iou=%.4f box=[%.1f,%.1f,%.1f,%.1f]\n", - i, d.instance_id, d.score, d.iou_score, - d.box.x0, d.box.y0, d.box.x1, d.box.y1); - - // Count mask pixels - int n_on = 0; - for (size_t j = 0; j < d.mask.data.size(); ++j) - if (d.mask.data[j] > 0) n_on++; - fprintf(stderr, " mask: %d/%zu pixels (%.1f%%)\n", - n_on, d.mask.data.size(), 100.0f * n_on / d.mask.data.size()); - - std::string mask_path = "/tmp/video_frame1_mask_" + std::to_string(i) + ".png"; - sam3_save_mask(d.mask, mask_path); - fprintf(stderr, " saved: %s\n", mask_path.c_str()); - } - - sam3_free_model(*model); - return result.detections.empty() ? 1 : 0; -} diff --git a/tests/test_video_text.cpp b/tests/test_video_text.cpp deleted file mode 100644 index 6829f2c..0000000 --- a/tests/test_video_text.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Video tracking test with text prompt on first N frames. - * Usage: test_video_text [prompt] [n_frames] - */ -#include "sam3.h" -#include -#include - -int main(int argc, char ** argv) { - if (argc < 3) { - fprintf(stderr, "Usage: %s [prompt] [n_frames]\n", argv[0]); - return 1; - } - - const std::string model_path = argv[1]; - const std::string video_path = argv[2]; - const std::string prompt = argc > 3 ? argv[3] : "car"; - const int n_frames = argc > 4 ? atoi(argv[4]) : 5; - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; - params.n_threads = 4; - - fprintf(stderr, "Loading model...\n"); - auto model = sam3_load_model(params); - if (!model) { fprintf(stderr, "Failed to load model\n"); return 1; } - - auto state = sam3_create_state(*model, params); - if (!state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - auto vi = sam3_get_video_info(video_path); - fprintf(stderr, "Video: %dx%d, %d frames, %.1f fps\n", - vi.width, vi.height, vi.n_frames, vi.fps); - - // Create tracker with text prompt - sam3_video_params vp; - vp.text_prompt = prompt; - vp.score_threshold = 0.3f; - vp.hotstart_delay = 3; // Quick confirmation for testing - auto tracker = sam3_create_tracker(*model, vp); - if (!tracker) { fprintf(stderr, "Failed to create tracker\n"); return 1; } - - int total_tracked = 0; - for (int fi = 0; fi < std::min(n_frames, vi.n_frames); ++fi) { - fprintf(stderr, "\n════════════════════════════════════════\n"); - fprintf(stderr, " Frame %d/%d\n", fi, n_frames); - fprintf(stderr, "════════════════════════════════════════\n"); - - auto frame = sam3_decode_video_frame(video_path, fi); - if (frame.data.empty()) { - fprintf(stderr, "Failed to decode frame %d\n", fi); - continue; - } - - auto result = sam3_track_frame(*tracker, *state, *model, frame); - fprintf(stderr, " Result: %zu detections\n", result.detections.size()); - - for (size_t i = 0; i < result.detections.size(); ++i) { - const auto& d = result.detections[i]; - int n_on = 0; - for (size_t j = 0; j < d.mask.data.size(); ++j) - if (d.mask.data[j] > 0) n_on++; - fprintf(stderr, " det %zu: inst=%d score=%.3f box=[%.0f,%.0f,%.0f,%.0f] mask=%.1f%%\n", - i, d.instance_id, d.score, - d.box.x0, d.box.y0, d.box.x1, d.box.y1, - 100.0f * n_on / d.mask.data.size()); - } - - if (!result.detections.empty()) { - total_tracked++; - // Save first frame's masks - if (fi == 0) { - for (size_t i = 0; i < result.detections.size() && i < 3; ++i) { - std::string path = "/tmp/video_text_f0_mask" + std::to_string(i) + ".png"; - sam3_save_mask(result.detections[i].mask, path); - fprintf(stderr, " saved: %s\n", path.c_str()); - } - } - } - } - - fprintf(stderr, "\n═══ Summary ═══\n"); - fprintf(stderr, " Prompt: '%s'\n", prompt.c_str()); - fprintf(stderr, " Frames with detections: %d/%d\n", total_tracked, n_frames); - - sam3_free_model(*model); - return total_tracked > 0 ? 0 : 1; -} diff --git a/tests/test_visual_only.cpp b/tests/test_visual_only.cpp deleted file mode 100644 index 584b59a..0000000 --- a/tests/test_visual_only.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "sam3.h" -#include - -int main(int argc, char** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [image.jpg]\n", argv[0]); - return 1; - } - - sam3_params params; - params.model_path = argv[1]; - params.n_threads = 4; - - // 1. Load visual-only model - fprintf(stderr, "=== Test 1: Load visual-only model ===\n"); - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, "FAIL: failed to load model\n"); - return 1; - } - fprintf(stderr, "PASS: model loaded\n"); - - // 2. Verify visual_only flag - fprintf(stderr, "\n=== Test 2: sam3_is_visual_only ===\n"); - if (!sam3_is_visual_only(*model)) { - fprintf(stderr, "FAIL: expected visual_only=true\n"); - return 1; - } - fprintf(stderr, "PASS: sam3_is_visual_only() = true\n"); - - // 3. PCS should return empty result (no crash) - fprintf(stderr, "\n=== Test 3: PCS guard ===\n"); - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, "FAIL: create_state\n"); - return 1; - } - sam3_pcs_params pcs; - pcs.text_prompt = "cat"; - auto r = sam3_segment_pcs(*state, *model, pcs); - if (!r.detections.empty()) { - fprintf(stderr, "FAIL: PCS should return empty on visual-only\n"); - return 1; - } - fprintf(stderr, "PASS: PCS returns empty on visual-only model\n"); - - // 4. If image provided, test PVS + visual tracking - if (argc >= 3) { - fprintf(stderr, "\n=== Test 4: PVS on visual-only model ===\n"); - auto img = sam3_load_image(argv[2]); - if (img.data.empty()) { - fprintf(stderr, "FAIL: failed to load image '%s'\n", argv[2]); - return 1; - } - if (!sam3_encode_image(*state, *model, img)) { - fprintf(stderr, "FAIL: encode_image\n"); - return 1; - } - - // PVS with center point - sam3_pvs_params pvs; - pvs.pos_points.push_back({(float)img.width / 2, (float)img.height / 2}); - auto rp = sam3_segment_pvs(*state, *model, pvs); - if (rp.detections.empty()) { - fprintf(stderr, "FAIL: PVS returned no detections\n"); - return 1; - } - fprintf(stderr, "PASS: PVS produced %zu detection(s)\n", rp.detections.size()); - - // 5. Visual tracker: create, add instance, propagate - fprintf(stderr, "\n=== Test 5: Visual tracker ===\n"); - sam3_visual_track_params vtp; - auto tracker = sam3_create_visual_tracker(*model, vtp); - if (!tracker) { - fprintf(stderr, "FAIL: create_visual_tracker\n"); - return 1; - } - - int id = sam3_tracker_add_instance(*tracker, *state, *model, pvs); - if (id < 0) { - fprintf(stderr, "FAIL: tracker_add_instance returned %d\n", id); - return 1; - } - fprintf(stderr, "PASS: tracker_add_instance -> id=%d\n", id); - - // Propagate same frame as a smoke test - auto rr = sam3_propagate_frame(*tracker, *state, *model, img); - fprintf(stderr, "PASS: propagate_frame -> %zu detection(s)\n", rr.detections.size()); - - // 6. track_frame should fail on visual-only - fprintf(stderr, "\n=== Test 6: track_frame guard ===\n"); - auto rt = sam3_track_frame(*tracker, *state, *model, img); - if (!rt.detections.empty()) { - fprintf(stderr, "FAIL: track_frame should return empty on visual-only\n"); - return 1; - } - fprintf(stderr, "PASS: track_frame returns empty on visual-only model\n"); - } - - fprintf(stderr, "\nAll tests passed.\n"); - return 0; -} diff --git a/tests/test_visual_only_compare.cpp b/tests/test_visual_only_compare.cpp deleted file mode 100644 index 7d12c00..0000000 --- a/tests/test_visual_only_compare.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/** - * Compare visual-only model output vs full model output for PVS (points/boxes) - * and video propagation. The tracker-path weights are identical, so outputs - * should match bit-for-bit (same ggml graph, same weights, same inputs). - * - * Usage: ./test_visual_only_compare [video.mp4] - */ -#include "sam3.h" - -#include -#include -#include -#include -#include -#include - -static float mask_iou(const sam3_mask& a, const sam3_mask& b) { - if (a.width != b.width || a.height != b.height) return 0.0f; - int inter = 0, uni = 0; - for (int i = 0; i < (int)a.data.size(); ++i) { - bool fa = a.data[i] > 127; - bool fb = b.data[i] > 127; - if (fa && fb) inter++; - if (fa || fb) uni++; - } - return uni > 0 ? (float)inter / uni : 1.0f; -} - -static int count_fg(const sam3_mask& m) { - int n = 0; - for (auto v : m.data) if (v > 127) n++; - return n; -} - -static bool compare_results(const char* label, - const sam3_result& full_r, - const sam3_result& vis_r) { - fprintf(stderr, "\n--- %s ---\n", label); - if (full_r.detections.size() != vis_r.detections.size()) { - fprintf(stderr, " FAIL: detection count mismatch: full=%zu visual=%zu\n", - full_r.detections.size(), vis_r.detections.size()); - return false; - } - if (full_r.detections.empty()) { - fprintf(stderr, " WARN: both returned 0 detections\n"); - return true; - } - - bool ok = true; - for (size_t i = 0; i < full_r.detections.size(); ++i) { - const auto& fd = full_r.detections[i]; - const auto& vd = vis_r.detections[i]; - - float iou = mask_iou(fd.mask, vd.mask); - int fg_full = count_fg(fd.mask); - int fg_vis = count_fg(vd.mask); - float score_diff = fabsf(fd.score - vd.score); - float iou_diff = fabsf(fd.iou_score - vd.iou_score); - - fprintf(stderr, " det[%zu]: mask_iou=%.6f fg_full=%d fg_vis=%d " - "score_diff=%.6f iou_diff=%.6f\n", - i, iou, fg_full, fg_vis, score_diff, iou_diff); - - if (iou < 0.999f) { - fprintf(stderr, " FAIL: mask IoU %.6f < 0.999 threshold\n", iou); - ok = false; - } - if (score_diff > 1e-4f) { - fprintf(stderr, " FAIL: score diff %.6f > 1e-4\n", score_diff); - ok = false; - } - } - fprintf(stderr, " %s\n", ok ? "PASS" : "FAIL"); - return ok; -} - -int main(int argc, char** argv) { - if (argc < 4) { - fprintf(stderr, "Usage: %s [video.mp4]\n", argv[0]); - return 1; - } - - const std::string full_path = argv[1]; - const std::string vis_path = argv[2]; - const std::string image_path = argv[3]; - const std::string video_path = (argc >= 5) ? argv[4] : ""; - - int n_pass = 0, n_fail = 0; - auto tally = [&](bool ok) { if (ok) n_pass++; else n_fail++; }; - - auto img = sam3_load_image(image_path); - if (img.data.empty()) { - fprintf(stderr, "Failed to load image '%s'\n", image_path.c_str()); - return 1; - } - fprintf(stderr, "Image: %dx%d\n", img.width, img.height); - - // ── Define test prompts ───────────────────────────────────────────── - struct pvs_test { - const char* name; - sam3_pvs_params params; - }; - - pvs_test tests[] = { - {"center_point", {}}, - {"box_prompt", {}}, - {"two_points", {}}, - {"point_and_box", {}}, - }; - - // Center point - tests[0].params.pos_points.push_back({(float)img.width / 2, (float)img.height / 2}); - - // Box prompt - tests[1].params.box = {(float)img.width * 0.1f, (float)img.height * 0.1f, - (float)img.width * 0.9f, (float)img.height * 0.9f}; - tests[1].params.use_box = true; - - // Two points (positive + negative) - tests[2].params.pos_points.push_back({(float)img.width / 2, (float)img.height / 2}); - tests[2].params.neg_points.push_back({10.0f, 10.0f}); - - // Point + box - tests[3].params.pos_points.push_back({(float)img.width / 2, (float)img.height / 2}); - tests[3].params.box = {(float)img.width * 0.2f, (float)img.height * 0.2f, - (float)img.width * 0.8f, (float)img.height * 0.8f}; - tests[3].params.use_box = true; - - // ═══════════════════════════════════════════════════════════════════════ - // Run on full model - // ═══════════════════════════════════════════════════════════════════════ - fprintf(stderr, "\n═══ Loading FULL model ═══\n"); - sam3_params fparams; - fparams.model_path = full_path; - fparams.n_threads = 4; - fparams.use_gpu = false; - - auto full_model = sam3_load_model(fparams); - if (!full_model) { fprintf(stderr, "Failed to load full model\n"); return 1; } - assert(!sam3_is_visual_only(*full_model)); - - auto full_state = sam3_create_state(*full_model, fparams); - if (!full_state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - if (!sam3_encode_image(*full_state, *full_model, img)) { - fprintf(stderr, "Failed to encode image (full)\n"); return 1; - } - - std::vector full_results; - for (auto& t : tests) { - fprintf(stderr, " Running PVS [%s] on full model...\n", t.name); - full_results.push_back(sam3_segment_pvs(*full_state, *full_model, t.params)); - } - - // Video propagation on full model - std::vector full_video_results; - if (!video_path.empty()) { - fprintf(stderr, "\n Running video propagation on full model...\n"); - sam3_visual_track_params vtp; - auto tracker = sam3_create_visual_tracker(*full_model, vtp); - - // Frame 0: encode + add instance - auto f0 = sam3_decode_video_frame(video_path, 0); - if (!f0.data.empty()) { - sam3_encode_image(*full_state, *full_model, f0); - sam3_pvs_params vpvs; - vpvs.pos_points.push_back({(float)f0.width / 2, (float)f0.height / 2}); - sam3_tracker_add_instance(*tracker, *full_state, *full_model, vpvs); - - // Propagate frames 1-3 - for (int fi = 1; fi <= 3; ++fi) { - auto frame = sam3_decode_video_frame(video_path, fi); - if (frame.data.empty()) break; - full_video_results.push_back( - sam3_propagate_frame(*tracker, *full_state, *full_model, frame)); - } - } - } - - // Free full model to save memory - full_state.reset(); - sam3_free_model(*full_model); - full_model.reset(); - - // ═══════════════════════════════════════════════════════════════════════ - // Run on visual-only model - // ═══════════════════════════════════════════════════════════════════════ - fprintf(stderr, "\n═══ Loading VISUAL-ONLY model ═══\n"); - sam3_params vparams; - vparams.model_path = vis_path; - vparams.n_threads = 4; - vparams.use_gpu = false; - - auto vis_model = sam3_load_model(vparams); - if (!vis_model) { fprintf(stderr, "Failed to load visual model\n"); return 1; } - assert(sam3_is_visual_only(*vis_model)); - - auto vis_state = sam3_create_state(*vis_model, vparams); - if (!vis_state) { fprintf(stderr, "Failed to create state\n"); return 1; } - - if (!sam3_encode_image(*vis_state, *vis_model, img)) { - fprintf(stderr, "Failed to encode image (visual)\n"); return 1; - } - - std::vector vis_results; - for (auto& t : tests) { - fprintf(stderr, " Running PVS [%s] on visual-only model...\n", t.name); - vis_results.push_back(sam3_segment_pvs(*vis_state, *vis_model, t.params)); - } - - // Video propagation on visual-only model - std::vector vis_video_results; - if (!video_path.empty()) { - fprintf(stderr, "\n Running video propagation on visual-only model...\n"); - sam3_visual_track_params vtp; - auto tracker = sam3_create_visual_tracker(*vis_model, vtp); - - auto f0 = sam3_decode_video_frame(video_path, 0); - if (!f0.data.empty()) { - sam3_encode_image(*vis_state, *vis_model, f0); - sam3_pvs_params vpvs; - vpvs.pos_points.push_back({(float)f0.width / 2, (float)f0.height / 2}); - sam3_tracker_add_instance(*tracker, *vis_state, *vis_model, vpvs); - - for (int fi = 1; fi <= 3; ++fi) { - auto frame = sam3_decode_video_frame(video_path, fi); - if (frame.data.empty()) break; - vis_video_results.push_back( - sam3_propagate_frame(*tracker, *vis_state, *vis_model, frame)); - } - } - } - - vis_state.reset(); - sam3_free_model(*vis_model); - vis_model.reset(); - - // ═══════════════════════════════════════════════════════════════════════ - // Compare results - // ═══════════════════════════════════════════════════════════════════════ - fprintf(stderr, "\n══════════════════════════════════════════════════\n"); - fprintf(stderr, " COMPARISON: Full model vs Visual-only model\n"); - fprintf(stderr, "══════════════════════════════════════════════════\n"); - - for (size_t i = 0; i < full_results.size(); ++i) { - tally(compare_results(tests[i].name, full_results[i], vis_results[i])); - } - - for (size_t i = 0; i < full_video_results.size() && i < vis_video_results.size(); ++i) { - char label[64]; - snprintf(label, sizeof(label), "video_frame_%zu", i + 1); - tally(compare_results(label, full_video_results[i], vis_video_results[i])); - } - - fprintf(stderr, "\n══════════════════════════════════════════════════\n"); - fprintf(stderr, " TOTAL: %d PASS, %d FAIL\n", n_pass, n_fail); - fprintf(stderr, "══════════════════════════════════════════════════\n\n"); - - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/test_vit.cpp b/tests/test_vit.cpp deleted file mode 100644 index affb76f..0000000 --- a/tests/test_vit.cpp +++ /dev/null @@ -1,465 +0,0 @@ -#include "sam3.h" - -#include -#include -#include -#include -#include -#include -#include - -// ═══════════════════════════════════════════════════════════════════════════════ -// Helpers: load reference tensors -// ═══════════════════════════════════════════════════════════════════════════════ - -struct ref_tensor { - std::vector data; - std::vector shape; - int numel() const { - int n = 1; - for (auto d : shape) n *= d; - return n; - } -}; - -static ref_tensor load_ref(const std::string & path) { - ref_tensor t; - // Load shape - { - std::ifstream f(path + ".shape"); - if (!f) { - fprintf(stderr, " [SKIP] %s.shape not found\n", path.c_str()); - return t; - } - std::string line; - std::getline(f, line); - size_t pos = 0; - while (pos < line.size()) { - size_t end = line.find(',', pos); - if (end == std::string::npos) end = line.size(); - t.shape.push_back(std::stoi(line.substr(pos, end - pos))); - pos = end + 1; - } - } - // Load data - { - std::ifstream f(path + ".bin", std::ios::binary); - if (!f) { - fprintf(stderr, " [SKIP] %s.bin not found\n", path.c_str()); - t.shape.clear(); - return t; - } - t.data.resize(t.numel()); - f.read(reinterpret_cast(t.data.data()), t.numel() * sizeof(float)); - } - return t; -} - -// Compare two float arrays with tolerance. -// Returns max absolute difference. -static float compare_tensors(const float * a, const float * b, int n, - float atol = 1e-4f, float rtol = 1e-3f) { - float max_diff = 0.0f; - int n_bad = 0; - for (int i = 0; i < n; ++i) { - float diff = fabsf(a[i] - b[i]); - float threshold = atol + rtol * fabsf(b[i]); - if (diff > threshold) n_bad++; - if (diff > max_diff) max_diff = diff; - } - return max_diff; -} - -static bool check(const std::string & name, const float * got, const ref_tensor & ref, - float atol = 1e-4f, float rtol = 1e-3f) { - if (ref.data.empty()) { - fprintf(stderr, " [SKIP] %s — no reference data\n", name.c_str()); - return true; - } - float max_diff = compare_tensors(got, ref.data.data(), ref.numel(), atol, rtol); - bool ok = max_diff < atol + rtol; - fprintf(stderr, " %s %s: max_diff=%.6f (atol=%.1e rtol=%.1e)\n", - ok ? "[PASS]" : "[FAIL]", name.c_str(), max_diff, atol, rtol); - return ok; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test: preprocessing -// ═══════════════════════════════════════════════════════════════════════════════ - -static bool test_preprocessing(const std::string & image_path, const std::string & ref_dir) { - fprintf(stderr, "\n=== Test: Preprocessing ===\n"); - - // Load image - auto img = sam3_load_image(image_path); - if (img.data.empty()) { - fprintf(stderr, " Failed to load image: %s\n", image_path.c_str()); - return false; - } - fprintf(stderr, " Loaded %dx%d image\n", img.width, img.height); - - // Load reference - auto ref = load_ref(ref_dir + "/preprocessed"); - if (ref.data.empty()) return true; // skip - - // The reference is [1, 3, 1008, 1008] from PyTorch (NCHW) - // We check a reasonable tolerance since bilinear resize implementations differ slightly - fprintf(stderr, " Reference shape: "); - for (auto d : ref.shape) fprintf(stderr, "%d ", d); - fprintf(stderr, "\n"); - - // Note: we can't directly test preprocessing since it's internal to sam3_encode_image. - // For now, just verify the image loaded correctly. - fprintf(stderr, " [INFO] Preprocessing test requires running encode_image — tested implicitly\n"); - return true; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test: RoPE computation -// ═══════════════════════════════════════════════════════════════════════════════ - -// Forward-declare the internal function (defined in sam3.cpp) -// We'll test it by replicating the computation here. -static bool test_rope(const std::string & ref_dir) { - fprintf(stderr, "\n=== Test: RoPE ===\n"); - - // Test window RoPE (24x24) - auto ref_window = load_ref(ref_dir + "/rope_window_real"); - if (!ref_window.data.empty()) { - // ref is [576, 32, 2] — (cos, sin) pairs - // Our compute_axial_cis produces [N, head_dim] with interleaved (cos, sin) - // But the Python ref stores them as [N, half_head, 2] where [:,:,0]=cos, [:,:,1]=sin - const int N = 576; - const int half_head = 32; - const int dim = 64; - const float theta = 10000.0f; - const float scale_pos = 1.0f; - - std::vector our_rope(N * dim); - - const int half_dim = dim / 4; // 16 - std::vector freqs(half_dim); - for (int i = 0; i < half_dim; ++i) { - freqs[i] = 1.0f / powf(theta, (float)(i * 4) / dim); - } - - for (int idx = 0; idx < N; ++idx) { - float t_x = (float)(idx % 24) * scale_pos; - float t_y = (float)(idx / 24) * scale_pos; - - for (int i = 0; i < half_dim; ++i) { - float angle_x = t_x * freqs[i]; - our_rope[idx * dim + i * 2 + 0] = cosf(angle_x); - our_rope[idx * dim + i * 2 + 1] = sinf(angle_x); - } - for (int i = 0; i < half_dim; ++i) { - float angle_y = t_y * freqs[i]; - our_rope[idx * dim + half_dim * 2 + i * 2 + 0] = cosf(angle_y); - our_rope[idx * dim + half_dim * 2 + i * 2 + 1] = sinf(angle_y); - } - } - - // Compare: ref is [N, 32, 2] = [N, half_head, (cos,sin)] - // Our layout is [N, dim] = [N, 64] with pairs (cos_x, sin_x, cos_x, sin_x, ..., cos_y, sin_y, ...) - // Need to rearrange comparison: - // ref[idx, j, 0] = cos of freq j for position idx - // ref[idx, j, 1] = sin of freq j for position idx - // Python: freqs_cis = cat([freqs_cis_x, freqs_cis_y], dim=-1) → [N, 32] - // freqs_cis_x has 16 complex values, freqs_cis_y has 16 complex values → total 32 - // view_as_real → [N, 32, 2] where [:,i,0]=cos, [:,i,1]=sin - // So ref[idx, j, 0] for j<16 = cos(t_x * freqs[j]) - // ref[idx, j, 1] for j<16 = sin(t_x * freqs[j]) - // ref[idx, j, 0] for j>=16 = cos(t_y * freqs[j-16]) - // ref[idx, j, 1] for j>=16 = sin(t_y * freqs[j-16]) - - // Our layout: [idx * 64 + j*2+0] = cos, [idx * 64 + j*2+1] = sin - // For x: j = 0..15 → our[idx*64 + j*2], ref[idx*64 + j*2] - // For y: j = 16..31 → our[idx*64 + 32 + (j-16)*2], ref[idx*64 + (j)*2] - - // Actually both are the same layout! ref is stored flat as [N*32*2] = [N*64] - // and our_rope is [N*64] with the same interleaving. - float max_diff = compare_tensors(our_rope.data(), ref_window.data.data(), N * dim); - bool ok = max_diff < 1e-5f; - fprintf(stderr, " %s window RoPE (24x24): max_diff=%.6e\n", ok ? "[PASS]" : "[FAIL]", max_diff); - } - - // Test global RoPE (72x72) - auto ref_global = load_ref(ref_dir + "/rope_global_real"); - if (!ref_global.data.empty()) { - const int N = 5184; - const int dim = 64; - const float theta = 10000.0f; - const float scale_pos = 24.0f / 72.0f; // 1/3 - - std::vector our_rope(N * dim); - const int half_dim = dim / 4; - std::vector freqs(half_dim); - for (int i = 0; i < half_dim; ++i) { - freqs[i] = 1.0f / powf(theta, (float)(i * 4) / dim); - } - - for (int idx = 0; idx < N; ++idx) { - float t_x = (float)(idx % 72) * scale_pos; - float t_y = (float)(idx / 72) * scale_pos; - - for (int i = 0; i < half_dim; ++i) { - float angle_x = t_x * freqs[i]; - our_rope[idx * dim + i * 2 + 0] = cosf(angle_x); - our_rope[idx * dim + i * 2 + 1] = sinf(angle_x); - } - for (int i = 0; i < half_dim; ++i) { - float angle_y = t_y * freqs[i]; - our_rope[idx * dim + half_dim * 2 + i * 2 + 0] = cosf(angle_y); - our_rope[idx * dim + half_dim * 2 + i * 2 + 1] = sinf(angle_y); - } - } - - float max_diff = compare_tensors(our_rope.data(), ref_global.data.data(), N * dim); - bool ok = max_diff < 1e-5f; - fprintf(stderr, " %s global RoPE (72x72): max_diff=%.6e\n", ok ? "[PASS]" : "[FAIL]", max_diff); - } - - return true; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test: sinusoidal PE -// ═══════════════════════════════════════════════════════════════════════════════ - -// Forward-declare for testing -extern "C" { - // We can't easily call static functions from sam3.cpp, so we'll replicate the logic -} - -static bool test_sinusoidal_pe(const std::string & ref_dir) { - fprintf(stderr, "\n=== Test: Sinusoidal PE ===\n"); - - struct { int H; int W; const char * name; } cases[] = { - {288, 288, "pe_288"}, - {144, 144, "pe_144"}, - { 72, 72, "pe_72"}, - { 36, 36, "pe_36"}, - }; - - for (auto & tc : cases) { - auto ref = load_ref(ref_dir + "/" + tc.name); - if (ref.data.empty()) continue; - - // Compute our sinusoidal PE - const int H = tc.H, W = tc.W, d_model = 256; - const int half = d_model / 2; - const float scale = 2.0f * (float)M_PI; - const float temperature = 10000.0f; - - // ref is [1, 256, H, W] in NCHW from PyTorch - std::vector our_pe(d_model * H * W); - - for (int y = 0; y < H; ++y) { - for (int x = 0; x < W; ++x) { - float pos_y = ((float)(y + 1) / (float)(H)) * scale; - float pos_x = ((float)(x + 1) / (float)(W)) * scale; - - for (int i = 0; i < half; ++i) { - float dim_t = powf(temperature, 2.0f * (float)(i / 2) / (float)half); - - float val_x, val_y; - if (i % 2 == 0) { - val_x = sinf(pos_x / dim_t); - val_y = sinf(pos_y / dim_t); - } else { - val_x = cosf(pos_x / dim_t); - val_y = cosf(pos_y / dim_t); - } - - // PyTorch output is [1, 256, H, W] = [N, C, H, W] - // Channel layout: first half is pos_y, second half is pos_x - // Index: [0, c, y, x] → c * H * W + y * W + x - our_pe[i * H * W + y * W + x] = val_y; - our_pe[(i + half) * H * W + y * W + x] = val_x; - } - } - } - - float max_diff = compare_tensors(our_pe.data(), ref.data.data(), d_model * H * W); - bool ok = max_diff < 1e-5f; - fprintf(stderr, " %s %s (%dx%d): max_diff=%.6e\n", - ok ? "[PASS]" : "[FAIL]", tc.name, tc.H, tc.W, max_diff); - } - - return true; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Test: full image encoding -// ═══════════════════════════════════════════════════════════════════════════════ - -static bool test_encode_image(const std::string & model_path, - const std::string & image_path, - const std::string & ref_dir) { - fprintf(stderr, "\n=== Test: Full Image Encoding ===\n"); - - sam3_params params; - params.model_path = model_path; - params.use_gpu = false; // CPU for determinism - params.n_threads = 4; - - auto model = sam3_load_model(params); - if (!model) { - fprintf(stderr, " Failed to load model\n"); - return false; - } - - auto state = sam3_create_state(*model, params); - if (!state) { - fprintf(stderr, " Failed to create state\n"); - return false; - } - - auto img = sam3_load_image(image_path); - if (img.data.empty()) { - fprintf(stderr, " Failed to load image\n"); - return false; - } - - bool ok = sam3_encode_image(*state, *model, img); - if (!ok) { - fprintf(stderr, " sam3_encode_image failed\n"); - return false; - } - - fprintf(stderr, " Image encoding completed\n"); - - int n_fail_enc = 0; - std::string dump_dir = ref_dir + "/cpp_out"; - { - // Create dump dir - std::string cmd = "mkdir -p " + dump_dir; - (void)system(cmd.c_str()); - } - - // Helper to compare intermediate tensors. - // ggml layout [E, W, H], Python ref is [1, H, W, E] (NHWC) - auto compare_nhwc = [&](const std::string & cpp_name, const std::string & ref_name, float tol) { - sam3_dump_state_tensor(*state, cpp_name, dump_dir + "/" + cpp_name); - auto ref_t = load_ref(ref_dir + "/" + ref_name); - auto cpp_t = load_ref(dump_dir + "/" + cpp_name); - if (ref_t.data.empty() || cpp_t.data.empty()) return; - int E = cpp_t.shape[0], nW = cpp_t.shape[1], nH = cpp_t.shape.size() > 2 ? cpp_t.shape[2] : 1; - std::vector transposed(cpp_t.numel()); - for (int e = 0; e < E; ++e) - for (int w = 0; w < nW; ++w) - for (int h = 0; h < nH; ++h) - transposed[h * nW * E + w * E + e] = cpp_t.data[e + w * E + h * E * nW]; - float md = compare_tensors(transposed.data(), ref_t.data.data(), - std::min((int)transposed.size(), ref_t.numel())); - bool ok = md < tol; - fprintf(stderr, " %s %s: max_diff=%.4f\n", ok ? "[PASS]" : "[FAIL]", ref_name.c_str(), md); - if (!ok) n_fail_enc++; - }; - - // Compare intermediate outputs - compare_nhwc("dbg_patch_embed", "patch_embed", 0.05f); - compare_nhwc("dbg_after_ln_pre", "after_ln_pre", 0.1f); - compare_nhwc("dbg_block_0_out", "block_0_out", 0.5f); - - sam3_dump_state_tensor(*state, "vit_output", dump_dir + "/vit_output"); - - // Compare ViT output against Python reference. - // ggml output: [ne0=E, ne1=W, ne2=H] stored as e + w*E + h*E*W - // Python ref: [1, C, H, W] stored as c*H*W + h*W + w (NCHW) - // Need to transpose ggml data to match PyTorch NCHW layout. - auto ref_vit = load_ref(ref_dir + "/vit_output_bchw"); - auto cpp_vit = load_ref(dump_dir + "/vit_output"); - - if (!ref_vit.data.empty() && !cpp_vit.data.empty()) { - // Transpose ggml [E=1024, W=72, H=72] → PyTorch [C=1024, H=72, W=72] - int E = cpp_vit.shape[0], W = cpp_vit.shape[1], H = (cpp_vit.shape.size() > 2) ? cpp_vit.shape[2] : 1; - std::vector transposed(cpp_vit.numel()); - for (int e = 0; e < E; ++e) - for (int h = 0; h < H; ++h) - for (int w = 0; w < W; ++w) - // ggml: e + w*E + h*E*W → pytorch: e*H*W + h*W + w - transposed[e * H * W + h * W + w] = cpp_vit.data[e + w * E + h * E * W]; - - float max_diff = compare_tensors(transposed.data(), ref_vit.data.data(), - std::min((int)transposed.size(), ref_vit.numel())); - bool ok = max_diff < 1.0f; - fprintf(stderr, " %s ViT output vs Python ref: max_diff=%.4f (E=%d W=%d H=%d)\n", - ok ? "[PASS]" : "[FAIL]", max_diff, E, W, H); - if (!ok) n_fail_enc++; - } - - // Dump and compare neck outputs. - // Same layout issue: ggml [C, W, H] vs PyTorch [1, C, H, W] - for (int i = 0; i < 3; ++i) { - std::string name = "neck_det_" + std::to_string(i); - sam3_dump_state_tensor(*state, name, dump_dir + "/" + name); - - auto ref_neck = load_ref(ref_dir + "/" + name); - auto cpp_neck = load_ref(dump_dir + "/" + name); - if (!ref_neck.data.empty() && !cpp_neck.data.empty()) { - int C = cpp_neck.shape[0], nW = cpp_neck.shape[1], nH = (cpp_neck.shape.size() > 2) ? cpp_neck.shape[2] : 1; - std::vector transposed(cpp_neck.numel()); - for (int c = 0; c < C; ++c) - for (int h = 0; h < nH; ++h) - for (int w = 0; w < nW; ++w) - transposed[c * nH * nW + h * nW + w] = cpp_neck.data[c + w * C + h * C * nW]; - - float max_diff = compare_tensors(transposed.data(), ref_neck.data.data(), - std::min((int)transposed.size(), ref_neck.numel())); - bool ok = max_diff < 2.0f; - fprintf(stderr, " %s %s vs Python ref: max_diff=%.4f\n", - ok ? "[PASS]" : "[FAIL]", name.c_str(), max_diff); - } - } - - state.reset(); - sam3_free_model(*model); - model.reset(); - - return true; -} - -// ═══════════════════════════════════════════════════════════════════════════════ -// Main -// ═══════════════════════════════════════════════════════════════════════════════ - -int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s [model.ggml] [image.jpg]\n", argv[0]); - fprintf(stderr, "\nRun tests/dump_reference.py first to generate reference data.\n"); - return 1; - } - - std::string ref_dir = argv[1]; - std::string model_path = (argc > 2) ? argv[2] : ""; - std::string image_path = (argc > 3) ? argv[3] : ""; - - int n_pass = 0, n_fail = 0, n_skip = 0; - - // Test RoPE (no model needed) - if (test_rope(ref_dir)) n_pass++; else n_fail++; - - // Test sinusoidal PE (no model needed) - if (test_sinusoidal_pe(ref_dir)) n_pass++; else n_fail++; - - // Test preprocessing (needs image) - if (!image_path.empty()) { - if (test_preprocessing(image_path, ref_dir)) n_pass++; else n_fail++; - } else { - fprintf(stderr, "\n[SKIP] Preprocessing test — no image provided\n"); - n_skip++; - } - - // Test full encoding (needs model + image) - if (!model_path.empty() && !image_path.empty()) { - if (test_encode_image(model_path, image_path, ref_dir)) n_pass++; else n_fail++; - } else { - fprintf(stderr, "\n[SKIP] Full encoding test — no model/image provided\n"); - n_skip++; - } - - fprintf(stderr, "\n═══════════════════════════════════════════\n"); - fprintf(stderr, "Results: %d passed, %d failed, %d skipped\n", n_pass, n_fail, n_skip); - return n_fail > 0 ? 1 : 0; -} diff --git a/tests/trace_single_pixel.py b/tests/trace_single_pixel.py deleted file mode 100644 index e498d6e..0000000 --- a/tests/trace_single_pixel.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -"""Trace a single pixel through both C++ and torch bilinear resize.""" -import numpy as np -import torch -from PIL import Image -from torchvision.transforms import v2 - -# Target pixel: (y=36, x=816, c=2) where cpp=110, py=113 - -image_path = "tests/test_random.jpg" -img = Image.open(image_path).convert("RGB") -src = np.array(img, dtype=np.uint8) # [480, 640, 3] HWC -src_h, src_w = src.shape[:2] -img_size = 1008 -c = 2 - -sx = src_w / img_size -sy = src_h / img_size - -y, x = 36, 816 - -# ═══ C++ path (double precision, exact) ═══ -fy = (y + 0.5) * sy - 0.5 -if fy < 0: fy = 0.0 -y0 = int(fy) -y1 = y0 + 1 if y0 < src_h - 1 else y0 -wy = fy - y0 -wy0 = 1.0 - wy - -fx = (x + 0.5) * sx - 0.5 -if fx < 0: fx = 0.0 -x0 = int(fx) -x1 = x0 + 1 if x0 < src_w - 1 else x0 -wx = fx - x0 -wx0 = 1.0 - wx - -p00 = float(src[y0, x0, c]) -p01 = float(src[y0, x1, c]) -p10 = float(src[y1, x0, c]) -p11 = float(src[y1, x1, c]) - -v_double = wy0 * (wx0 * p00 + wx * p01) + wy * (wx0 * p10 + wx * p11) -result_cpp = int(v_double + 0.5) - -print("═══ Pixel Trace: (y=36, x=816, c=2) ═══") -print(f"\nSource image: {src_w}x{src_h}") -print(f"sx={sx:.15f}, sy={sy:.15f}") -print(f"fy={fy:.15f}, fx={fx:.15f}") -print(f"y0={y0}, y1={y1}, wy={wy:.15f}, wy0={wy0:.15f}") -print(f"x0={x0}, x1={x1}, wx={wx:.15f}, wx0={wx0:.15f}") -print(f"p00={p00}, p01={p01}, p10={p10}, p11={p11}") -print(f"v(double)={v_double:.15f}") -print(f"result_cpp = int({v_double:.15f} + 0.5) = {result_cpp}") - -# ═══ torch path ═══ -img_tensor = v2.functional.to_image(img) # uint8 CHW -float_input = img_tensor.float() # [3, 480, 640] - -# Check: what float values does torch see at the source positions? -print(f"\ntorch float input at (c={c}, y0={y0}, x0={x0}): {float_input[c, y0, x0].item()}") -print(f"torch float input at (c={c}, y0={y0}, x1={x1}): {float_input[c, y0, x1].item()}") -print(f"torch float input at (c={c}, y1={y1}, x0={x0}): {float_input[c, y1, x0].item()}") -print(f"torch float input at (c={c}, y1={y1}, x1={x1}): {float_input[c, y1, x1].item()}") - -# Run torch F.interpolate -fi = torch.nn.functional.interpolate( - float_input.unsqueeze(0), size=(img_size, img_size), - mode='bilinear', align_corners=False, antialias=False -).squeeze(0) - -torch_float_result = fi[c, y, x].item() -torch_uint8_result = fi[c, y, x].clamp(0, 255).round().to(torch.uint8).item() - -print(f"\ntorch float result: {torch_float_result:.15f}") -print(f"torch uint8 result: {torch_uint8_result}") -print(f"C++ uint8 result: {result_cpp}") -print(f"difference: {torch_uint8_result - result_cpp}") - -# ═══ Manual torch replication (float32 step by step) ═══ -# Replicate torch's exact computation in float32 -rheight = np.float64(src_h) / np.float64(img_size) -rwidth = np.float64(src_w) / np.float64(img_size) - -h1r = rheight * (y + 0.5) - 0.5 -if h1r < 0: h1r = 0.0 -h1 = int(h1r) -h1p = 1 if h1 < src_h - 1 else 0 -h1lambda = np.float32(h1r - h1) -h0lambda = np.float32(1.0) - h1lambda - -w1r = rwidth * (x + 0.5) - 0.5 -if w1r < 0: w1r = 0.0 -w1 = int(w1r) -w1p = 1 if w1 < src_w - 1 else 0 -w1lambda = np.float32(w1r - w1) -w0lambda = np.float32(1.0) - w1lambda - -# Read as float32 (matching torch's float conversion) -pos1_0 = np.float32(src[h1, w1, c]) -pos1_wp = np.float32(src[h1, w1 + w1p, c]) -pos2_0 = np.float32(src[h1 + h1p, w1, c]) -pos2_wp = np.float32(src[h1 + h1p, w1 + w1p, c]) - -# torch formula: h0lambda * (w0lambda * pos1[0] + w1lambda * pos1[wp]) + h1lambda * (w0lambda * pos2[0] + w1lambda * pos2[wp]) -manual_val = np.float32(h0lambda * (w0lambda * pos1_0 + w1lambda * pos1_wp) + - h1lambda * (w0lambda * pos2_0 + w1lambda * pos2_wp)) - -print(f"\n═══ Manual float32 replication ═══") -print(f"h0lambda={h0lambda:.10f}, h1lambda={h1lambda:.10f}") -print(f"w0lambda={w0lambda:.10f}, w1lambda={w1lambda:.10f}") -print(f"pos: {pos1_0}, {pos1_wp}, {pos2_0}, {pos2_wp}") -print(f"manual float32 result: {manual_val:.10f}") -print(f"manual uint8: {int(round(float(manual_val)))}") - -# ═══ Step by step float32 with explicit intermediate values ═══ -print(f"\n═══ Step-by-step float32 ═══") -t1 = np.float32(w0lambda * pos1_0) -t2 = np.float32(w1lambda * pos1_wp) -row0 = np.float32(t1 + t2) -t3 = np.float32(w0lambda * pos2_0) -t4 = np.float32(w1lambda * pos2_wp) -row1 = np.float32(t3 + t4) -t5 = np.float32(h0lambda * row0) -t6 = np.float32(h1lambda * row1) -result_f32 = np.float32(t5 + t6) -print(f"w0*p00={t1}, w1*p01={t2}, row0={row0}") -print(f"w0*p10={t3}, w1*p11={t4}, row1={row1}") -print(f"h0*row0={t5}, h1*row1={t6}, result={result_f32}") -print(f"uint8(round): {int(round(float(result_f32)))}") -print(f"uint8(+0.5): {int(result_f32 + np.float32(0.5))}") diff --git a/tests/trace_torchvision_resize.py b/tests/trace_torchvision_resize.py deleted file mode 100644 index 1240f65..0000000 --- a/tests/trace_torchvision_resize.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -"""Trace exactly what torchvision v2.Resize does on uint8 tensor input. - -The goal: figure out the exact algorithm so C++ can replicate it identically. -""" -import numpy as np -import torch -from torchvision.transforms import v2 -from PIL import Image - -image_path = "tests/test_random.jpg" -img_size = 1008 - -# Load image -img = Image.open(image_path).convert("RGB") -img_tensor = v2.functional.to_image(img) # uint8 CHW - -# ═══ Test 1: Does v2.Resize on uint8 go through PIL or torch? ═══ -# Create a unique pattern that would reveal the algorithm -print("═══ Test 1: PIL vs torch path detection ═══") - -# Check if v2.Resize produces the same output as PIL resize -step1 = v2.ToDtype(torch.uint8, scale=True)(img_tensor) -tv_out = v2.Resize(size=(img_size, img_size))(step1) # uint8 output - -# PIL path: convert to PIL Image, resize, convert back -pil_img = Image.fromarray(step1.permute(1, 2, 0).numpy(), "RGB") -pil_resized = pil_img.resize((img_size, img_size), Image.BILINEAR) -pil_out = torch.from_numpy(np.array(pil_resized)).permute(2, 0, 1) - -diff = (tv_out.to(torch.int16) - pil_out.to(torch.int16)).abs() -print(f" v2.Resize vs PIL.resize(BILINEAR): max_diff={diff.max().item()}, n_diff={(diff>0).sum().item()}") - -# ═══ Test 2: Does v2.Resize use float internally? ═══ -# Resize through float path manually: uint8 → float → interpolate → round → uint8 -print("\n═══ Test 2: Manual float interpolation ═══") -float_input = step1.float() # [3, 480, 640] float32 -float_resized = torch.nn.functional.interpolate( - float_input.unsqueeze(0), # [1, 3, 480, 640] - size=(img_size, img_size), - mode='bilinear', - align_corners=False, - antialias=False, -).squeeze(0) # [3, 1008, 1008] - -# Method A: clamp and truncate -manual_uint8_trunc = float_resized.clamp(0, 255).to(torch.uint8) -diff_a = (tv_out.to(torch.int16) - manual_uint8_trunc.to(torch.int16)).abs() -print(f" vs float→interpolate→clamp→uint8(trunc): max={diff_a.max().item()}, n_diff={(diff_a>0).sum().item()}") - -# Method B: clamp, round, then truncate -manual_uint8_round = float_resized.clamp(0, 255).round().to(torch.uint8) -diff_b = (tv_out.to(torch.int16) - manual_uint8_round.to(torch.int16)).abs() -print(f" vs float→interpolate→clamp→round→uint8: max={diff_b.max().item()}, n_diff={(diff_b>0).sum().item()}") - -# Method C: with antialias=True -float_resized_aa = torch.nn.functional.interpolate( - float_input.unsqueeze(0), - size=(img_size, img_size), - mode='bilinear', - align_corners=False, - antialias=True, -).squeeze(0) -manual_uint8_aa = float_resized_aa.clamp(0, 255).round().to(torch.uint8) -diff_c = (tv_out.to(torch.int16) - manual_uint8_aa.to(torch.int16)).abs() -print(f" vs float→interpolate(aa)→clamp→round→uint8: max={diff_c.max().item()}, n_diff={(diff_c>0).sum().item()}") - -# ═══ Test 3: Check exact matching path ═══ -print("\n═══ Test 3: Finding exact match ═══") - -# Try: PIL → resize directly -# torchvision v2 for uint8 on CPU typically dispatches to PIL -from torchvision.transforms.functional import resize as F_resize -from torchvision.transforms import InterpolationMode - -# F.resize with different options -for aa in [True, False]: - out = F_resize(step1, [img_size, img_size], - interpolation=InterpolationMode.BILINEAR, antialias=aa) - diff = (tv_out.to(torch.int16) - out.to(torch.int16)).abs() - print(f" F.resize(antialias={aa}): max={diff.max().item()}, n_diff={(diff>0).sum().item()}") - -# ═══ Test 4: Save the float interpolation result (before uint8 conversion) ═══ -# This tells us the "ground truth" sub-pixel values -print("\n═══ Test 4: Float interpolation values ═══") -print(f" float_resized range: [{float_resized.min():.4f}, {float_resized.max():.4f}]") -print(f" Sample values at (0,0): {float_resized[:, 0, 0].tolist()}") -print(f" tv_out values at (0,0): {tv_out[:, 0, 0].tolist()}") -print(f" float rounded at (0,0): {float_resized[:, 0, 0].round().tolist()}") - -# ═══ Test 5: What rounding does tv_out use? ═══ -# Compare tv_out against the float values to determine rounding -print("\n═══ Test 5: Rounding analysis ═══") -float_vals = float_resized.flatten().numpy() -tv_vals = tv_out.flatten().numpy().astype(np.float32) - -# For each pixel, check: is tv_val == floor(float_val) or round(float_val)? -n_floor = 0 -n_round = 0 -n_ceil = 0 -n_other = 0 -for i in range(min(len(float_vals), 100000)): - f = float_vals[i] - t = tv_vals[i] - if t == np.floor(f): - n_floor += 1 - elif t == np.round(f): - n_round += 1 - elif t == np.ceil(f): - n_ceil += 1 - else: - n_other += 1 - -print(f" floor: {n_floor}, round: {n_round}, ceil: {n_ceil}, other: {n_other}") -print(f" (sampled first 100000 pixels)") - -# Most likely: v2.Resize on uint8 goes through PIL which uses truncation (floor) -# Let's verify by computing floor explicitly -manual_floor = np.floor(float_vals).astype(np.uint8) -tv_flat = tv_out.flatten().numpy() -floor_match = (manual_floor == tv_flat).sum() -round_match = (np.round(float_vals).astype(np.uint8) == tv_flat).sum() -print(f"\n Global floor match: {floor_match}/{len(tv_flat)} ({100*floor_match/len(tv_flat):.2f}%)") -print(f" Global round match: {round_match}/{len(tv_flat)} ({100*round_match/len(tv_flat):.2f}%)") diff --git a/tests/visualize_cpp_masks.py b/tests/visualize_cpp_masks.py deleted file mode 100644 index 257e207..0000000 --- a/tests/visualize_cpp_masks.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate overlay images from C++ SAM2 tracking masks. -Also compares with Python masks side-by-side. -""" -import numpy as np -import os, sys -from PIL import Image - -sam2_root = os.path.expanduser("~/Documents/sam2") -video_dir = os.path.join(sam2_root, "notebooks/videos/bedroom") -py_dir = "/tmp/debug_sam2_pipeline" -cpp_dir = "/tmp/debug_sam2_cpp" -out_dir = os.path.join(os.path.dirname(__file__), "debug_pipeline") -os.makedirs(out_dir, exist_ok=True) - -def load_tensor(path): - data = np.fromfile(path + ".bin", dtype=np.float32) - if os.path.exists(path + ".shape"): - with open(path + ".shape") as f: - shape = [int(x) for x in f.read().strip().split(",") if x] - return data.reshape(shape) if shape else data - return data - -for fi in [1, 2, 5, 10, 20, 30, 40, 50]: - frame_path = os.path.join(video_dir, f"{fi:05d}.jpg") - if not os.path.exists(frame_path): - continue - img = np.array(Image.open(frame_path)) - - # C++ binary mask - cpp_path = os.path.join(cpp_dir, f"f{fi}_output_mask_binary") - if os.path.exists(cpp_path + ".bin"): - cpp_mask = load_tensor(cpp_path) - cpp_binary = cpp_mask > 0.5 - if cpp_binary.shape[0] == img.shape[0] and cpp_binary.shape[1] == img.shape[1]: - overlay_cpp = img.copy() - overlay_cpp[cpp_binary] = (overlay_cpp[cpp_binary] * 0.5 + np.array([255, 0, 0]) * 0.5).astype(np.uint8) - Image.fromarray(overlay_cpp).save(os.path.join(out_dir, f"cpp_frame{fi:03d}.png")) - fg = cpp_binary.sum() - print(f"Frame {fi:3d}: C++ fg={fg:6d} ({100.0*fg/cpp_binary.size:.1f}%)") - - # Python mask (logits) - py_path = os.path.join(py_dir, f"f{fi}_output_mask") - if os.path.exists(py_path + ".bin"): - py_logits = load_tensor(py_path).squeeze() - py_binary = py_logits > 0.0 - if py_binary.shape[0] == img.shape[0] and py_binary.shape[1] == img.shape[1]: - overlay_py = img.copy() - overlay_py[py_binary] = (overlay_py[py_binary] * 0.5 + np.array([0, 0, 255]) * 0.5).astype(np.uint8) - Image.fromarray(overlay_py).save(os.path.join(out_dir, f"py_frame{fi:03d}.png")) - - # IoU comparison - if os.path.exists(cpp_path + ".bin") and os.path.exists(py_path + ".bin"): - cpp_mask2 = load_tensor(cpp_path) - py_logits2 = load_tensor(py_path).squeeze() - c = cpp_mask2.flatten() > 0.5 - p = py_logits2.flatten() > 0.0 - if c.size == p.size: - inter = np.logical_and(c, p).sum() - union = np.logical_or(c, p).sum() - iou = inter / max(union, 1) - print(f" IoU(py,cpp)={iou:.4f}") - -print(f"\nOverlays saved to {out_dir}/")