From d4a89d7744be9a9312d68cf319ec942e20d7c65c Mon Sep 17 00:00:00 2001 From: erdmann Date: Sat, 10 May 2025 16:08:37 +0200 Subject: [PATCH] Add 'gemma-3-vision' as chat_format for OpenAI-compatible server. --- llama_cpp/server/model.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/llama_cpp/server/model.py b/llama_cpp/server/model.py index c6716f9196..0cc0e714ee 100644 --- a/llama_cpp/server/model.py +++ b/llama_cpp/server/model.py @@ -157,6 +157,20 @@ def load_llama_from_model_settings(settings: ModelSettings) -> llama_cpp.Llama: chat_handler = llama_cpp.llama_chat_format.Llama3VisionAlpha( clip_model_path=settings.clip_model_path, verbose=settings.verbose ) + elif settings.chat_format == "gemma-3-vision": + assert settings.clip_model_path is not None, "clip model not found" + if settings.hf_model_repo_id is not None: + chat_handler = ( + llama_cpp.llama_chat_format.Gemma3ChatHandler.from_pretrained( + repo_id=settings.hf_model_repo_id, + filename=settings.clip_model_path, + verbose=settings.verbose, + ) + ) + else: + chat_handler = llama_cpp.llama_chat_format.Gemma3ChatHandler( + clip_model_path=settings.clip_model_path, verbose=settings.verbose + ) elif settings.chat_format == "minicpm-v-2.6": assert settings.clip_model_path is not None, "clip model not found" if settings.hf_model_repo_id is not None: