From 66e23528f55c933ac777ffebf57f60f413fa8e98 Mon Sep 17 00:00:00 2001 From: fszontagh Date: Thu, 27 Aug 2026 12:19:54 +0200 Subject: [PATCH 1/2] fix: validate vision projector output dim against LLM hidden size --- ggml | 2 +- src/model/te/llm.hpp | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ggml b/ggml index 8e800cef2..032b6997d 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit 8e800cef2948046cc47f9db6090491c6128ca42c +Subproject commit 032b6997db4c9c75dc85d8d2bb2beec77b1231b0 diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index 092c68e48..a8e614f56 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -73,6 +73,7 @@ namespace LLM { int num_heads = 16; int64_t in_channels = 3; int64_t out_hidden_size = 3584; + bool out_hidden_size_detected = false; int temporal_patch_size = 2; int patch_size = 14; int spatial_merge_size = 2; @@ -238,7 +239,8 @@ namespace LLM { } if (contains(name, "visual.merger.linear_fc2.weight") || contains(name, "visual.merger.mlp.2.weight")) { - config.vision.out_hidden_size = tensor_storage.ne[1]; + config.vision.out_hidden_size = tensor_storage.ne[1]; + config.vision.out_hidden_size_detected = true; } continue; } @@ -1768,6 +1770,15 @@ namespace LLM { LOG_WARN("no vision weights detected, vision disabled"); enable_vision = false; } + // The default would reject valid models, so only compare a detected dim. + if (enable_vision && config.vision.out_hidden_size_detected && + config.vision.out_hidden_size != config.hidden_size) { + LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), " + "the vision weights (mmproj) likely belong to a different LLM variant, vision disabled", + config.vision.out_hidden_size, + config.hidden_size); + enable_vision = false; + } if (enable_vision) { LOG_DEBUG("enable llm vision"); if (config.llama_cpp_style) { From fb1c85941f32459273bb00fd677a23b42693b0b1 Mon Sep 17 00:00:00 2001 From: leejet Date: Sun, 13 Sep 2026 23:35:42 +0800 Subject: [PATCH 2/2] refactor: move LLM vision validation into config detection --- src/model/te/llm.hpp | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index 835ae2c3e..19f1fec3c 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -75,7 +75,6 @@ namespace LLM { int num_heads = 16; int64_t in_channels = 3; int64_t out_hidden_size = 3584; - bool out_hidden_size_detected = false; int temporal_patch_size = 2; int patch_size = 14; int spatial_merge_size = 2; @@ -134,7 +133,8 @@ namespace LLM { static LLMConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, const std::string& prefix, - LLMArch arch) { + LLMArch arch, + bool& enable_vision) { LLMConfig config; config.arch = arch; if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) { @@ -225,8 +225,9 @@ namespace LLM { config.num_experts_per_tok = 4; } - config.num_layers = 0; - int detected_vision_layers = 0; + config.num_layers = 0; + int detected_vision_layers = 0; + bool out_hidden_size_detected = false; for (const auto& [name, tensor_storage] : tensor_storage_map) { if (!starts_with(name, prefix)) { continue; @@ -271,8 +272,8 @@ namespace LLM { } if (ends_with(name, "visual.merger.linear_fc2.weight") || ends_with(name, "visual.merger.mlp.2.weight")) { - config.vision.out_hidden_size = tensor_storage.ne[1]; - config.vision.out_hidden_size_detected = true; + config.vision.out_hidden_size = tensor_storage.ne[1]; + out_hidden_size_detected = true; } continue; } @@ -326,6 +327,19 @@ namespace LLM { config.vocab_size, config.hidden_size, config.intermediate_size); + if (enable_vision && !config.have_vision_weight) { + LOG_WARN("no vision weights detected, vision disabled"); + enable_vision = false; + } + // The default would reject valid models, so only compare a detected dim. + if (enable_vision && out_hidden_size_detected && + config.vision.out_hidden_size != config.hidden_size) { + LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), " + "the vision weights (mmproj) likely belong to a different LLM variant, vision disabled", + config.vision.out_hidden_size, + config.hidden_size); + enable_vision = false; + } return config; } }; @@ -1882,21 +1896,8 @@ namespace LLM { bool enable_vision_ = false, std::shared_ptr weight_manager = nullptr) : GGMLRunner(backend, weight_manager), - config(LLMConfig::detect_from_weights(tensor_storage_map, prefix, arch)), + config(LLMConfig::detect_from_weights(tensor_storage_map, prefix, arch, enable_vision_)), enable_vision(enable_vision_) { - if (enable_vision && !config.have_vision_weight) { - LOG_WARN("no vision weights detected, vision disabled"); - enable_vision = false; - } - // The default would reject valid models, so only compare a detected dim. - if (enable_vision && config.vision.out_hidden_size_detected && - config.vision.out_hidden_size != config.hidden_size) { - LOG_ERROR("vision projector output size (%" PRId64 ") does not match LLM hidden size (%" PRId64 "), " - "the vision weights (mmproj) likely belong to a different LLM variant, vision disabled", - config.vision.out_hidden_size, - config.hidden_size); - enable_vision = false; - } if (enable_vision) { LOG_VERBOSE("enable llm vision"); if (config.llama_cpp_style) {